<commit_before>package task

import (
	"context"

	"v2ray.com/core/common/signal/semaphore"
)

type Task func() error

type executionContext struct {
	ctx       context.Context
	task      Task
	onSuccess Task
	onFailure Task
}

func (c *executionContext) executeTask() error {
	if c.ctx == nil && c.task == nil {
		return nil
	}

	if c.ctx == nil {
		return c.task()
	}

	if c.task == nil {
		<-c.ctx.Done()
		return c.ctx.Err()
	}

	return executeParallel(func() error {
		<-c.ctx.Done()
		return c.ctx.Err()
	}, c.task)
}

func (c *executionContext) run() error {
	err := c.executeTask()
	if err == nil && c.onSuccess != nil {
		return c.onSuccess()
	}
	if err != nil && c.onFailure != nil {
		return c.onFailure()
	}
	return err
}

type ExecutionOption func(*executionContext)

func WithContext(ctx context.Context) ExecutionOption {
	return func(c *executionContext) {
		c.ctx = ctx
	}
}

func Parallel(tasks ...Task) ExecutionOption {
	return func(c *executionContext) {
		c.task = func() error {
			return executeParallel(tasks...)
		}
	}
}

func Sequential(tasks ...Task) ExecutionOption {
	return func(c *executionContext) {
		c.task = func() error {
			return execute(tasks...)
		}
	}
}

func OnSuccess(task Task) ExecutionOption {
	return func(c *executionContext) {
		c.onSuccess = task
	}
}

func OnFailure(task Task) ExecutionOption {
	return func(c *executionContext) {
		c.onFailure = task
	}
}

func Single(task Task, opts ExecutionOption) Task {
	return Run(append([]ExecutionOption{Sequential(task)}, opts)...)
}

func Run(opts ...ExecutionOption) Task {
	var c executionContext
	for _, opt := range opts {
		opt(&c)
	}
	return func() error {
		return c.run()
	}
}

// execute runs a list of tasks sequentially, returns the first error encountered or nil if all tasks pass.
func execute(tasks ...Task) error {
	for _, task := range tasks {
		if err := task(); err != nil {
			return err
		}
	}
	return nil
}

// executeParallel executes a list of tasks asynchronously, returns the first error encountered or nil if all tasks pass.
func executeParallel(tasks ...Task) error {
	n := len(tasks)
	s := semaphore.New(n)
	done := make(chan error, 1)

	for _, task := range tasks {
		<-s.Wait()
		go func(f func() error) {
			if err := f(); err != nil {
				select {
				case done <- err:
				default:
				}
			}
			s.Signal()
		}(task)
	}

	for i := 0; i < n; i++ {
		select {
		case err := <-done:
			return err
		case <-s.Wait():
		}
	}

	return nil
}
<commit_msg>fix #1131<commit_after>package task

import (
	"context"

	"v2ray.com/core/common/signal/semaphore"
)

type Task func() error

type executionContext struct {
	ctx       context.Context
	tasks     []Task
	onSuccess Task
	onFailure Task
}

func (c *executionContext) executeTask() error {
	if len(c.tasks) == 0 {
		return nil
	}

	ctx := context.Background()

	if c.ctx != nil {
		ctx = c.ctx
	}

	return executeParallel(ctx, c.tasks)
}

func (c *executionContext) run() error {
	err := c.executeTask()
	if err == nil && c.onSuccess != nil {
		return c.onSuccess()
	}
	if err != nil && c.onFailure != nil {
		return c.onFailure()
	}
	return err
}

type ExecutionOption func(*executionContext)

func WithContext(ctx context.Context) ExecutionOption {
	return func(c *executionContext) {
		c.ctx = ctx
	}
}

func Parallel(tasks ...Task) ExecutionOption {
	return func(c *executionContext) {
		c.tasks = append(c.tasks, tasks...)
	}
}

func Sequential(tasks ...Task) ExecutionOption {
	return func(c *executionContext) {
		c.tasks = append(c.tasks, func() error {
			return execute(tasks...)
		})
	}
}

func OnSuccess(task Task) ExecutionOption {
	return func(c *executionContext) {
		c.onSuccess = task
	}
}

func OnFailure(task Task) ExecutionOption {
	return func(c *executionContext) {
		c.onFailure = task
	}
}

func Single(task Task, opts ExecutionOption) Task {
	return Run(append([]ExecutionOption{Sequential(task)}, opts)...)
}

func Run(opts ...ExecutionOption) Task {
	var c executionContext
	for _, opt := range opts {
		opt(&c)
	}
	return func() error {
		return c.run()
	}
}

// execute runs a list of tasks sequentially, returns the first error encountered or nil if all tasks pass.
func execute(tasks ...Task) error {
	for _, task := range tasks {
		if err := task(); err != nil {
			return err
		}
	}
	return nil
}

// executeParallel executes a list of tasks asynchronously, returns the first error encountered or nil if all tasks pass.
func executeParallel(ctx context.Context, tasks []Task) error {
	n := len(tasks)
	s := semaphore.New(n)
	done := make(chan error, 1)

	for _, task := range tasks {
		<-s.Wait()
		go func(f func() error) {
			if err := f(); err != nil {
				select {
				case done <- err:
				default:
				}
			}
			s.Signal()
		}(task)
	}

	for i := 0; i < n; i++ {
		select {
		case err := <-done:
			return err
		case <-ctx.Done():
			return ctx.Err()
		case <-s.Wait():
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package trace

import (
	"fmt"
	"os"
)

const debugEnabled = false

func debug(format string, args ...interface{}) {
	if !debugEnabled {
		return
	}
	fmt.Fprintf(os.Stderr, format+"\n", args...)
}
<commit_msg>[#347] package trace: envconst-configurable debug mode<commit_after>package trace

import (
	"fmt"
	"os"

	"github.com/zrepl/zrepl/util/envconst"
)

var debugEnabled = envconst.Bool("ZREPL_TRACE_DEBUG_ENABLED", false)

func debug(format string, args ...interface{}) {
	if !debugEnabled {
		return
	}
	fmt.Fprintf(os.Stderr, format+"\n", args...)
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package zoekt

import (
	"bytes"
	"log"
	"sort"
)

var _ = log.Println

type docIterator struct {
	query *SubstringQuery

	leftPad  uint32
	rightPad uint32
	distance uint32
	first    []uint32
	last     []uint32

	fileIdx int
	ends    []uint32
}

type candidateMatch struct {
	query *SubstringQuery

	substrBytes []byte
	lowered     []byte
	caseMask    [][]byte
	caseBits    [][]byte

	file   uint32
	offset uint32
}

func (m *candidateMatch) caseMatches(fileCaseBits []byte) bool {
	if !m.query.CaseSensitive {
		return true
	}
	patLen := len(m.substrBytes)
	startExtend := m.offset % 8
	patEnd := m.offset + uint32(patLen)
	endExtend := (8 - (patEnd % 8)) % 8

	start := m.offset - startExtend
	end := m.offset + uint32(patLen) + endExtend

	fileBits := append([]byte{}, fileCaseBits[start/8:end/8]...)
	mask := m.caseMask[startExtend]
	bits := m.caseBits[startExtend]

	for i := range fileBits {
		if fileBits[i]&mask[i] != bits[i] {
			return false
		}
	}

	return true
}

func (m *candidateMatch) matchContent(content []byte) bool {
	return bytes.Compare(content[m.offset:m.offset+uint32(len(m.lowered))], m.lowered) == 0
}

func (m *candidateMatch) line(newlines []uint32, content []byte, caseBits []byte) (lineNum, lineOff int, lineContent []byte) {
	idx := sort.Search(len(newlines), func(n int) bool {
		return newlines[n] >= m.offset
	})

	end := len(content)
	if idx < len(newlines) {
		end = int(newlines[idx])
	}

	start := 0
	if idx > 0 {
		start = int(newlines[idx-1] + 1)
	}

	return idx + 1, int(m.offset) - start, toOriginal(content, caseBits, start, end)
}

func (s *docIterator) next() []*candidateMatch {
	patBytes := []byte(s.query.Pattern)
	lowerPatBytes := toLower(patBytes)

	caseMasks, caseBits := findCaseMasks(patBytes)
	var candidates []*candidateMatch
	for {
		if len(s.first) == 0 || len(s.last) == 0 {
			break
		}
		p1 := s.first[0]
		p2 := s.last[0]

		for s.fileIdx < len(s.ends) && s.ends[s.fileIdx] <= p1 {
			s.fileIdx++
		}

		if p1+s.distance < p2 {
			s.first = s.first[1:]
		} else if p1+s.distance > p2 {
			s.last = s.last[1:]
		} else {
			s.first = s.first[1:]
			s.last = s.last[1:]

			var fileStart uint32
			if s.fileIdx > 0 {
				fileStart = s.ends[s.fileIdx-1]
			}
			if p1 < s.leftPad+fileStart || p1+s.distance+ngramSize+s.rightPad > s.ends[s.fileIdx] {
				continue
			}

			candidates = append(candidates,
				&candidateMatch{
					caseMask:    caseMasks,
					caseBits:    caseBits,
					query:       s.query,
					substrBytes: patBytes,
					lowered:     lowerPatBytes,
					file:        uint32(s.fileIdx),
					offset:      p1 - fileStart - s.leftPad,
				})
		}
	}

	return candidates
}
<commit_msg>Optimization: avoid allocation in case comparison.<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package zoekt

import (
	"bytes"
	"log"
	"sort"
)

var _ = log.Println

type docIterator struct {
	query *SubstringQuery

	leftPad  uint32
	rightPad uint32
	distance uint32
	first    []uint32
	last     []uint32

	fileIdx int
	ends    []uint32
}

type candidateMatch struct {
	query *SubstringQuery

	substrBytes []byte
	lowered     []byte
	caseMask    [][]byte
	caseBits    [][]byte

	file   uint32
	offset uint32
}

func (m *candidateMatch) caseMatches(fileCaseBits []byte) bool {
	if !m.query.CaseSensitive {
		return true
	}
	patLen := len(m.substrBytes)
	startExtend := m.offset % 8
	patEnd := m.offset + uint32(patLen)
	endExtend := (8 - (patEnd % 8)) % 8

	start := m.offset - startExtend
	end := m.offset + uint32(patLen) + endExtend

	fileBits := fileCaseBits[start/8 : end/8]
	mask := m.caseMask[startExtend]
	bits := m.caseBits[startExtend]

	for i := range fileBits {
		if fileBits[i]&mask[i] != bits[i] {
			return false
		}
	}

	return true
}

func (m *candidateMatch) matchContent(content []byte) bool {
	return bytes.Compare(content[m.offset:m.offset+uint32(len(m.lowered))], m.lowered) == 0
}

func (m *candidateMatch) line(newlines []uint32, content []byte, caseBits []byte) (lineNum, lineOff int, lineContent []byte) {
	idx := sort.Search(len(newlines), func(n int) bool {
		return newlines[n] >= m.offset
	})

	end := len(content)
	if idx < len(newlines) {
		end = int(newlines[idx])
	}

	start := 0
	if idx > 0 {
		start = int(newlines[idx-1] + 1)
	}

	return idx + 1, int(m.offset) - start, toOriginal(content, caseBits, start, end)
}

func (s *docIterator) next() []*candidateMatch {
	patBytes := []byte(s.query.Pattern)
	lowerPatBytes := toLower(patBytes)

	caseMasks, caseBits := findCaseMasks(patBytes)
	var candidates []*candidateMatch
	for {
		if len(s.first) == 0 || len(s.last) == 0 {
			break
		}
		p1 := s.first[0]
		p2 := s.last[0]

		for s.fileIdx < len(s.ends) && s.ends[s.fileIdx] <= p1 {
			s.fileIdx++
		}

		if p1+s.distance < p2 {
			s.first = s.first[1:]
		} else if p1+s.distance > p2 {
			s.last = s.last[1:]
		} else {
			s.first = s.first[1:]
			s.last = s.last[1:]

			var fileStart uint32
			if s.fileIdx > 0 {
				fileStart = s.ends[s.fileIdx-1]
			}
			if p1 < s.leftPad+fileStart || p1+s.distance+ngramSize+s.rightPad > s.ends[s.fileIdx] {
				continue
			}

			candidates = append(candidates,
				&candidateMatch{
					caseMask:    caseMasks,
					caseBits:    caseBits,
					query:       s.query,
					substrBytes: patBytes,
					lowered:     lowerPatBytes,
					file:        uint32(s.fileIdx),
					offset:      p1 - fileStart - s.leftPad,
				})
		}
	}

	return candidates
}
<|endoftext|>


<commit_before>package handlers

import (
	"encoding/json"
	"net/http"
	"time"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/memcache"
	"google.golang.org/appengine/user"

	"github.com/icco/natnatnat/models"
	"github.com/pilu/traffic"
)

type ArchiveData struct {
	Years   *map[int]Year
	Posts   *[]models.Entry
	IsAdmin bool
}

// TODO(icco): Rewrite to fix map iteration problems.
type Year map[time.Month]Month
type Month []Day
type Day []int64

var months = [12]time.Month{
	time.January,
	time.February,
	time.March,
	time.April,
	time.May,
	time.June,
	time.July,
	time.August,
	time.September,
	time.October,
	time.November,
	time.December,
}

func ArchiveTaskHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	entries, err := models.AllPosts(c)
	if err != nil {
		log.Errorf(c, err.Error())
		http.Error(w, err.Error(), 500)
		return
	}
	log.Infof(c, "Retrieved data: %d.", len(*entries))

	years := make(map[int]Year)

	oldest := (*entries)[len(*entries)-1].Datetime
	newest := (*entries)[0].Datetime

	log.Infof(c, "Oldest: %v, Newest: %v", oldest, newest)

	for year := oldest.Year(); year <= newest.Year(); year += 1 {
		years[year] = make(Year)
		log.Infof(c, "Adding %d.", year)
		for _, month := range months {
			if year < newest.Year() || (year == newest.Year() && month <= newest.Month()) {
				years[year][month] = make([]Day, daysIn(month, year))
				log.Debugf(c, "Adding %d/%d - %d days.", year, month, len(years[year][month]))
			}
		}
	}

	q := models.ArchivePageQuery()
	t := q.Run(c)
	for {
		var p models.Entry
		_, err := t.Next(&p)
		if err == datastore.Done {
			break // No further entities match the query.
		}

		if err != nil {
			log.Errorf(c, "Error fetching next Entry: %v", err)
			break
		}

		year := p.Datetime.Year()
		month := p.Datetime.Month()
		day := p.Datetime.Day()
		log.Infof(c, "Trying post id %d", p.Id)

		if years[year] == nil {
			years[year] = make(Year)
			log.Errorf(c, "%d isn't a valid year.", year)
		}

		if years[year][month] == nil {
			log.Errorf(c, "%d/%d isn't a valid month.", year, month)
			years[year][month] = make([]Day, daysIn(month, year))
		}

		if years[year][month][day] == nil {
			log.Infof(c, "Making %d/%d/%d", year, month, day)
			years[year][month][day] = make(Day, 0)
		}

		// log.Infof(c, "Appending %d/%d/%d: %+v", year, month, day, years[year][month][day])
		years[year][month][day] = append(years[year][month][day], p.Id)
	}
	log.Infof(c, "Added posts.")

	// https://blog.golang.org/json-and-go
	b, err := json.Marshal(m)
	if err != nil {
		log.Errorf(c, err.Error())
		http.Error(w, err.Error(), 500)
	}

	item := &memcache.Item{
		Key:   "archive_data",
		Value: b,
	}

	// Set the item, unconditionally
	if err := memcache.Set(c, item); err != nil {
		log.Errorf(log, "error setting item: %v", err)
	}
}

func ArchiveHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	entries, err := models.AllPosts(c)
	if err != nil {
		log.Errorf(c, err.Error())
		http.Error(w, err.Error(), 500)
		return
	}
	log.Infof(c, "Retrieved data: %d.", len(*entries))

	// Get the item from the memcache
	var years map[int]Year
	if year_data, err := memcache.Get(c, "archive_data"); err == memcache.ErrCacheMiss {
		log.Infof(c, "item not in the cache")
	} else if err != nil {
		log.Errorf(c, "error getting item: %v", err)
	} else {
		err := json.Unmarshal(year_data, &years)
		if err != nil {
			log.Errorf(c, err.Error())
			http.Error(w, err.Error(), 500)
			return
		}
	}

	data := &ArchiveData{Years: years, IsAdmin: user.IsAdmin(c), Posts: entries}
	w.Render("archive", data)
}

// daysIn returns the number of days in a month for a given year.
func daysIn(m time.Month, year int) int {
	// This is equivalent to time.daysIn(m, year).
	return time.Date(year, m+1, 0, 0, 0, 0, 0, time.UTC).Day()
}
<commit_msg>more archive tweaks<commit_after>package handlers

import (
	"encoding/json"
	"net/http"
	"time"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/memcache"
	"google.golang.org/appengine/user"

	"github.com/icco/natnatnat/models"
	"github.com/pilu/traffic"
)

type ArchiveData struct {
	Years   *map[int]Year
	Posts   *[]models.Entry
	IsAdmin bool
}

// TODO(icco): Rewrite to fix map iteration problems.
type Year map[time.Month]Month
type Month []Day
type Day []int64

var months = [12]time.Month{
	time.January,
	time.February,
	time.March,
	time.April,
	time.May,
	time.June,
	time.July,
	time.August,
	time.September,
	time.October,
	time.November,
	time.December,
}

func ArchiveTaskHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	entries, err := models.AllPosts(c)
	if err != nil {
		log.Errorf(c, err.Error())
		http.Error(w, err.Error(), 500)
		return
	}
	log.Infof(c, "Retrieved data: %d.", len(*entries))

	years := make(map[int]Year)

	oldest := (*entries)[len(*entries)-1].Datetime
	newest := (*entries)[0].Datetime

	log.Infof(c, "Oldest: %v, Newest: %v", oldest, newest)

	for year := oldest.Year(); year <= newest.Year(); year += 1 {
		years[year] = make(Year)
		log.Infof(c, "Adding %d.", year)
		for _, month := range months {
			if year < newest.Year() || (year == newest.Year() && month <= newest.Month()) {
				years[year][month] = make([]Day, daysIn(month, year))
				log.Debugf(c, "Adding %d/%d - %d days.", year, month, len(years[year][month]))
			}
		}
	}

	q := models.ArchivePageQuery()
	t := q.Run(c)
	for {
		var p models.Entry
		_, err := t.Next(&p)
		if err == datastore.Done {
			break // No further entities match the query.
		}

		if err != nil {
			log.Errorf(c, "Error fetching next Entry: %v", err)
			break
		}

		year := p.Datetime.Year()
		month := p.Datetime.Month()
		day := p.Datetime.Day()
		log.Infof(c, "Trying post id %d", p.Id)

		if years[year] == nil {
			years[year] = make(Year)
			log.Errorf(c, "%d isn't a valid year.", year)
		}

		if years[year][month] == nil {
			log.Errorf(c, "%d/%d isn't a valid month.", year, month)
			years[year][month] = make([]Day, daysIn(month, year))
		}

		if years[year][month][day] == nil {
			log.Infof(c, "Making %d/%d/%d", year, month, day)
			years[year][month][day] = make(Day, 0)
		}

		// log.Infof(c, "Appending %d/%d/%d: %+v", year, month, day, years[year][month][day])
		years[year][month][day] = append(years[year][month][day], p.Id)
	}
	log.Infof(c, "Added posts.")

	// https://blog.golang.org/json-and-go
	b, err := json.Marshal(years)
	if err != nil {
		log.Errorf(c, err.Error())
		http.Error(w, err.Error(), 500)
	}

	item := &memcache.Item{
		Key:   "archive_data",
		Value: b,
	}

	// Set the item, unconditionally
	if err := memcache.Set(c, item); err != nil {
		log.Errorf(c, "error setting item: %v", err)
	}
}

func ArchiveHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	entries, err := models.AllPosts(c)
	if err != nil {
		log.Errorf(c, err.Error())
		http.Error(w, err.Error(), 500)
		return
	}
	log.Infof(c, "Retrieved data: %d.", len(*entries))

	// Get the item from the memcache
	var years map[int]Year
	if year_data, err := memcache.Get(c, "archive_data"); err == memcache.ErrCacheMiss {
		log.Infof(c, "item not in the cache")
	} else if err != nil {
		log.Errorf(c, "error getting item: %v", err)
	} else {
		err := json.Unmarshal(year_data.Value, &years)
		if err != nil {
			log.Errorf(c, err.Error())
			http.Error(w, err.Error(), 500)
			return
		}
	}

	data := &ArchiveData{Years: &years, IsAdmin: user.IsAdmin(c), Posts: entries}
	w.Render("archive", data)
}

// daysIn returns the number of days in a month for a given year.
func daysIn(m time.Month, year int) int {
	// This is equivalent to time.daysIn(m, year).
	return time.Date(year, m+1, 0, 0, 0, 0, 0, time.UTC).Day()
}
<|endoftext|>


<commit_before>package hawserclient

import (
	"encoding/json"
	"github.com/bmizerany/assert"
	"github.com/hawser/git-hawser/hawser"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"testing"
)

func TestGet(t *testing.T) {
	mux := http.NewServeMux()
	server := httptest.NewServer(mux)
	tmp := tempdir(t)
	defer server.Close()
	defer os.RemoveAll(tmp)

	mux.HandleFunc("/media/objects/oid", func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" {
			head := w.Header()
			head.Set("Content-Type", "application/octet-stream")
			head.Set("Content-Length", "4")
			w.WriteHeader(200)
			w.Write([]byte("test"))
			return
		}

		w.WriteHeader(405)
	})

	hawser.Config.SetConfig("hawser.url", server.URL+"/media")
	reader, size, wErr := Get("whatever/oid")
	if wErr != nil {
		t.Fatalf("unexpected error: %s", wErr)
	}
	defer reader.Close()

	if size != 4 {
		t.Errorf("unexpected size: %d", size)
	}

	by, err := ioutil.ReadAll(reader)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}

	if body := string(by); body != "test" {
		t.Errorf("unexpected body: %s", body)
	}
}

type putRequest struct {
	Oid    string
	Size   int
	Status int
	Body   string
}

func TestExternalPut(t *testing.T) {
	mux := http.NewServeMux()
	server := httptest.NewServer(mux)
	tmp := tempdir(t)
	defer server.Close()
	defer os.RemoveAll(tmp)

	hawser.Config.SetConfig("hawser.url", server.URL+"/media")
	oidPath := filepath.Join(tmp, "oid")
	if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil {
		t.Fatalf("Unable to write oid file: %s", err)
	}

	uploaded := false
	verified := false

	mux.HandleFunc("/media/objects/oid", func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "PUT" {
			w.WriteHeader(405)
			return
		}

		if value := r.Header.Get("Content-Length"); value != "4" {
			t.Errorf("bad 'Content-Length' header: %v", value)
		}

		if value := r.Header.Get("a"); value != "1" {
			t.Errorf("bad 'a' header: %v", value)
		}

		by, err := ioutil.ReadAll(r.Body)
		r.Body.Close()

		if err != nil {
			t.Errorf("Error reading uploaded body: %s", err)
		}

		if string(by) != "test" {
			t.Errorf("bad body sent: %s", string(by))
		}

		uploaded = true
		w.WriteHeader(201)
		w.Write([]byte("yup"))
	})

	mux.HandleFunc("/media/objects/callback", func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			w.WriteHeader(405)
			return
		}

		if value := r.Header.Get("b"); value != "2" {
			t.Errorf("bad 'b' header: %v", value)
		}

		putReq := &putRequest{}
		if err := json.NewDecoder(r.Body).Decode(putReq); err != nil {
			t.Errorf("error decoding callback request json: %s", err)
		}

		if putReq.Oid != "oid" {
			t.Errorf("bad oid: %s", putReq.Oid)
		}

		if putReq.Size != 4 {
			t.Errorf("bad size: %s", putReq.Oid)
		}

		if putReq.Status != 201 {
			t.Errorf("bad status: %s", putReq.Oid)
		}

		if putReq.Body != "yup" {
			t.Errorf("bad body: %s", putReq.Oid)
		}

		verified = true
		w.WriteHeader(200)
	})

	link := &linkMeta{
		Links: map[string]*link{
			"upload": {
				Href:   server.URL + "/media/objects/oid",
				Header: map[string]string{"a": "1"},
			},
			"callback": {
				Href:   server.URL + "/media/objects/callback",
				Header: map[string]string{"b": "2"},
			},
		},
	}

	if err := ExternalPut(oidPath, "", link, nil); err != nil {
		t.Error(err)
	}

	if !uploaded {
		t.Error("upload request never called")
	}

	if !verified {
		t.Error("callback never called")
	}
}

type postRequest struct {
	Oid  string
	Size int
}

func TestPost(t *testing.T) {
	mux := http.NewServeMux()
	server := httptest.NewServer(mux)
	tmp := tempdir(t)
	defer server.Close()
	defer os.RemoveAll(tmp)

	mux.HandleFunc("/media/objects", func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "POST" {
			postReq := &postRequest{}
			if err := json.NewDecoder(r.Body).Decode(postReq); err != nil {
				t.Errorf("Error parsing json: %s", err)
			}
			r.Body.Close()

			if postReq.Size != 4 {
				t.Errorf("Unexpected size: %d", postReq.Size)
			}

			if postReq.Oid != "oid" {
				t.Errorf("unexpected oid: %s", postReq.Oid)
			}

			w.Header().Set("Content-Type", "application/json")
			w.WriteHeader(201)
			w.Write([]byte(`{
				"_links": {
					"abc": {
						"href": "def",
						"header": {
							"a": "1",
							"b": "2"
						}
					}
				}
			}`))
			return
		}

		w.WriteHeader(405)
	})

	hawser.Config.SetConfig("hawser.url", server.URL+"/media")
	oidPath := filepath.Join(tmp, "oid")
	if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil {
		t.Fatalf("Unable to write oid file: %s", err)
	}

	link, status, err := Post(oidPath, "")
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}

	if status != 201 {
		t.Errorf("unexpected status: %d", status)
	}

	if link == nil || link.Links == nil {
		t.Error("expected a link object, got none")
	}

	for key, rel := range link.Links {
		t.Logf("%s: %v", key, rel)
	}

	if len(link.Links) != 1 {
		t.Error("wrong number of link relations")
	}

	linkRel, ok := link.Links["abc"]
	if !ok {
		t.Error("no 'abc' rel")
	}

	if linkRel.Href != "def" {
		t.Errorf("bad href: %s", linkRel.Href)
	}

	if linkRel.Header["a"] != "1" {
		t.Errorf("bad 'a': %s", linkRel.Header["a"])
	}

	if linkRel.Header["b"] != "2" {
		t.Errorf("bad 'b': %s", linkRel.Header["b"])
	}
}

func TestPut(t *testing.T) {
	mux := http.NewServeMux()
	server := httptest.NewServer(mux)
	tmp := tempdir(t)
	defer server.Close()
	defer os.RemoveAll(tmp)

	mux.HandleFunc("/media/objects/oid", func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "PUT" {
			by, err := ioutil.ReadAll(r.Body)
			if err != nil {
				t.Errorf("Error reading request body: %s", err)
			}

			r.Body.Close()
			if body := string(by); body != "test" {
				t.Errorf("Unexpected body: %s", body)
			}

			w.WriteHeader(200)
			return
		}

		w.WriteHeader(405)
	})

	hawser.Config.SetConfig("hawser.url", server.URL+"/media")
	oidPath := filepath.Join(tmp, "oid")
	if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil {
		t.Fatalf("Unable to write oid file: %s", err)
	}

	if err := Put(oidPath, "", nil); err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
}

func TestOptions(t *testing.T) {
	mux := http.NewServeMux()
	server := httptest.NewServer(mux)
	tmp := tempdir(t)
	defer server.Close()
	defer os.RemoveAll(tmp)

	mux.HandleFunc("/media/objects/oid", func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "OPTIONS" {
			w.WriteHeader(200)
		} else {
			w.WriteHeader(405)
		}
	})

	hawser.Config.SetConfig("hawser.url", server.URL+"/media")
	oidPath := filepath.Join(tmp, "oid")
	if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil {
		t.Fatalf("Unable to write oid file: %s", err)
	}

	status, err := Options(oidPath)
	if err != nil {
		t.Errorf("Unexpected error: %s", err)
	}

	if status != 200 {
		t.Errorf("unexpected status: %d", status)
	}
}

func TestOptionsWithoutExistingObject(t *testing.T) {
	status, err := Options("/this/better/not/work")
	if err == nil {
		t.Errorf("expected an error to be returned")
	}

	if status != 0 {
		t.Errorf("unexpected status: %d", status)
	}
}

func TestObjectUrl(t *testing.T) {
	oid := "oid"
	tests := map[string]string{
		"http://example.com":      "http://example.com/objects/oid",
		"http://example.com/":     "http://example.com/objects/oid",
		"http://example.com/foo":  "http://example.com/foo/objects/oid",
		"http://example.com/foo/": "http://example.com/foo/objects/oid",
	}

	config := hawser.Config
	for endpoint, expected := range tests {
		config.SetConfig("hawser.url", endpoint)
		assert.Equal(t, expected, ObjectUrl(oid).String())
	}
}

func init() {
	execCreds = func(input Creds, subCommand string) (credentialFetcher, error) {
		return &testCredentialFetcher{input}, nil
	}
}

func tempdir(t *testing.T) string {
	dir, err := ioutil.TempDir("", "hawser-test-hawserclient")
	if err != nil {
		t.Fatalf("Error getting temp dir: %s", err)
	}
	return dir
}

type testCredentialFetcher struct {
	Creds Creds
}

func (c *testCredentialFetcher) Credentials() Creds {
	return c.Creds
}
<commit_msg>ンンー ンンンン ンーンン<commit_after>package hawserclient

import (
	"encoding/json"
	"github.com/bmizerany/assert"
	"github.com/hawser/git-hawser/hawser"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"testing"
)

func TestGet(t *testing.T) {
	mux := http.NewServeMux()
	server := httptest.NewServer(mux)
	tmp := tempdir(t)
	defer server.Close()
	defer os.RemoveAll(tmp)

	mux.HandleFunc("/media/objects/oid", func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "GET" {
			w.WriteHeader(405)
			return
		}

		head := w.Header()
		head.Set("Content-Type", "application/octet-stream")
		head.Set("Content-Length", "4")
		w.WriteHeader(200)
		w.Write([]byte("test"))
	})

	hawser.Config.SetConfig("hawser.url", server.URL+"/media")
	reader, size, wErr := Get("whatever/oid")
	if wErr != nil {
		t.Fatalf("unexpected error: %s", wErr)
	}
	defer reader.Close()

	if size != 4 {
		t.Errorf("unexpected size: %d", size)
	}

	by, err := ioutil.ReadAll(reader)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}

	if body := string(by); body != "test" {
		t.Errorf("unexpected body: %s", body)
	}
}

type putRequest struct {
	Oid    string
	Size   int
	Status int
	Body   string
}

func TestExternalPut(t *testing.T) {
	mux := http.NewServeMux()
	server := httptest.NewServer(mux)
	tmp := tempdir(t)
	defer server.Close()
	defer os.RemoveAll(tmp)

	hawser.Config.SetConfig("hawser.url", server.URL+"/media")
	oidPath := filepath.Join(tmp, "oid")
	if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil {
		t.Fatalf("Unable to write oid file: %s", err)
	}

	uploaded := false
	verified := false

	mux.HandleFunc("/media/objects/oid", func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "PUT" {
			w.WriteHeader(405)
			return
		}

		if value := r.Header.Get("Content-Length"); value != "4" {
			t.Errorf("bad 'Content-Length' header: %v", value)
		}

		if value := r.Header.Get("a"); value != "1" {
			t.Errorf("bad 'a' header: %v", value)
		}

		by, err := ioutil.ReadAll(r.Body)
		r.Body.Close()

		if err != nil {
			t.Errorf("Error reading uploaded body: %s", err)
		}

		if string(by) != "test" {
			t.Errorf("bad body sent: %s", string(by))
		}

		uploaded = true
		w.WriteHeader(201)
		w.Write([]byte("yup"))
	})

	mux.HandleFunc("/media/objects/callback", func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			w.WriteHeader(405)
			return
		}

		if value := r.Header.Get("b"); value != "2" {
			t.Errorf("bad 'b' header: %v", value)
		}

		putReq := &putRequest{}
		if err := json.NewDecoder(r.Body).Decode(putReq); err != nil {
			t.Errorf("error decoding callback request json: %s", err)
		}

		if putReq.Oid != "oid" {
			t.Errorf("bad oid: %s", putReq.Oid)
		}

		if putReq.Size != 4 {
			t.Errorf("bad size: %s", putReq.Oid)
		}

		if putReq.Status != 201 {
			t.Errorf("bad status: %s", putReq.Oid)
		}

		if putReq.Body != "yup" {
			t.Errorf("bad body: %s", putReq.Oid)
		}

		verified = true
		w.WriteHeader(200)
	})

	link := &linkMeta{
		Links: map[string]*link{
			"upload": {
				Href:   server.URL + "/media/objects/oid",
				Header: map[string]string{"a": "1"},
			},
			"callback": {
				Href:   server.URL + "/media/objects/callback",
				Header: map[string]string{"b": "2"},
			},
		},
	}

	if err := ExternalPut(oidPath, "", link, nil); err != nil {
		t.Error(err)
	}

	if !uploaded {
		t.Error("upload request never called")
	}

	if !verified {
		t.Error("callback never called")
	}
}

type postRequest struct {
	Oid  string
	Size int
}

func TestPost(t *testing.T) {
	mux := http.NewServeMux()
	server := httptest.NewServer(mux)
	tmp := tempdir(t)
	defer server.Close()
	defer os.RemoveAll(tmp)

	mux.HandleFunc("/media/objects", func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			w.WriteHeader(405)
			return
		}

		postReq := &postRequest{}
		if err := json.NewDecoder(r.Body).Decode(postReq); err != nil {
			t.Errorf("Error parsing json: %s", err)
		}
		r.Body.Close()

		if postReq.Size != 4 {
			t.Errorf("Unexpected size: %d", postReq.Size)
		}

		if postReq.Oid != "oid" {
			t.Errorf("unexpected oid: %s", postReq.Oid)
		}

		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(201)
		w.Write([]byte(`{
				"_links": {
					"abc": {
						"href": "def",
						"header": {
							"a": "1",
							"b": "2"
						}
					}
				}
			}`))
	})

	hawser.Config.SetConfig("hawser.url", server.URL+"/media")
	oidPath := filepath.Join(tmp, "oid")
	if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil {
		t.Fatalf("Unable to write oid file: %s", err)
	}

	link, status, err := Post(oidPath, "")
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	}

	if status != 201 {
		t.Errorf("unexpected status: %d", status)
	}

	if link == nil || link.Links == nil {
		t.Error("expected a link object, got none")
	}

	for key, rel := range link.Links {
		t.Logf("%s: %v", key, rel)
	}

	if len(link.Links) != 1 {
		t.Error("wrong number of link relations")
	}

	linkRel, ok := link.Links["abc"]
	if !ok {
		t.Error("no 'abc' rel")
	}

	if linkRel.Href != "def" {
		t.Errorf("bad href: %s", linkRel.Href)
	}

	if linkRel.Header["a"] != "1" {
		t.Errorf("bad 'a': %s", linkRel.Header["a"])
	}

	if linkRel.Header["b"] != "2" {
		t.Errorf("bad 'b': %s", linkRel.Header["b"])
	}
}

func TestPut(t *testing.T) {
	mux := http.NewServeMux()
	server := httptest.NewServer(mux)
	tmp := tempdir(t)
	defer server.Close()
	defer os.RemoveAll(tmp)

	mux.HandleFunc("/media/objects/oid", func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "PUT" {
			w.WriteHeader(405)
			return
		}

		by, err := ioutil.ReadAll(r.Body)
		if err != nil {
			t.Errorf("Error reading request body: %s", err)
		}

		r.Body.Close()
		if body := string(by); body != "test" {
			t.Errorf("Unexpected body: %s", body)
		}

		w.WriteHeader(200)
	})

	hawser.Config.SetConfig("hawser.url", server.URL+"/media")
	oidPath := filepath.Join(tmp, "oid")
	if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil {
		t.Fatalf("Unable to write oid file: %s", err)
	}

	if err := Put(oidPath, "", nil); err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
}

func TestOptions(t *testing.T) {
	mux := http.NewServeMux()
	server := httptest.NewServer(mux)
	tmp := tempdir(t)
	defer server.Close()
	defer os.RemoveAll(tmp)

	mux.HandleFunc("/media/objects/oid", func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "OPTIONS" {
			w.WriteHeader(405)
			return
		}

		w.WriteHeader(200)
	})

	hawser.Config.SetConfig("hawser.url", server.URL+"/media")
	oidPath := filepath.Join(tmp, "oid")
	if err := ioutil.WriteFile(oidPath, []byte("test"), 0744); err != nil {
		t.Fatalf("Unable to write oid file: %s", err)
	}

	status, err := Options(oidPath)
	if err != nil {
		t.Errorf("Unexpected error: %s", err)
	}

	if status != 200 {
		t.Errorf("unexpected status: %d", status)
	}
}

func TestOptionsWithoutExistingObject(t *testing.T) {
	status, err := Options("/this/better/not/work")
	if err == nil {
		t.Errorf("expected an error to be returned")
	}

	if status != 0 {
		t.Errorf("unexpected status: %d", status)
	}
}

func TestObjectUrl(t *testing.T) {
	oid := "oid"
	tests := map[string]string{
		"http://example.com":      "http://example.com/objects/oid",
		"http://example.com/":     "http://example.com/objects/oid",
		"http://example.com/foo":  "http://example.com/foo/objects/oid",
		"http://example.com/foo/": "http://example.com/foo/objects/oid",
	}

	config := hawser.Config
	for endpoint, expected := range tests {
		config.SetConfig("hawser.url", endpoint)
		assert.Equal(t, expected, ObjectUrl(oid).String())
	}
}

func init() {
	execCreds = func(input Creds, subCommand string) (credentialFetcher, error) {
		return &testCredentialFetcher{input}, nil
	}
}

func tempdir(t *testing.T) string {
	dir, err := ioutil.TempDir("", "hawser-test-hawserclient")
	if err != nil {
		t.Fatalf("Error getting temp dir: %s", err)
	}
	return dir
}

type testCredentialFetcher struct {
	Creds Creds
}

func (c *testCredentialFetcher) Credentials() Creds {
	return c.Creds
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"exec"
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/scanner"
	"go/token"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime/pprof"
	"strings"
)

var (
	// main operation modes
	list        = flag.Bool("l", false, "list files whose formatting differs from gofmt's")
	write       = flag.Bool("w", false, "write result to (source) file instead of stdout")
	rewriteRule = flag.String("r", "", "rewrite rule (e.g., 'α[β:len(α)] -> α[β:]')")
	simplifyAST = flag.Bool("s", false, "simplify code")
	doDiff      = flag.Bool("d", false, "display diffs instead of rewriting files")

	// layout control
	comments  = flag.Bool("comments", true, "print comments")
	tabWidth  = flag.Int("tabwidth", 8, "tab width")
	tabIndent = flag.Bool("tabindent", true, "indent with tabs independent of -spaces")
	useSpaces = flag.Bool("spaces", true, "align with spaces instead of tabs")

	// debugging
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to this file")
)

var (
	fset        = token.NewFileSet()
	exitCode    = 0
	rewrite     func(*ast.File) *ast.File
	parserMode  uint
	printerMode uint
)

func report(err os.Error) {
	scanner.PrintError(os.Stderr, err)
	exitCode = 2
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: gofmt [flags] [path ...]\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func initParserMode() {
	parserMode = uint(0)
	if *comments {
		parserMode |= parser.ParseComments
	}
}

func initPrinterMode() {
	printerMode = uint(0)
	if *tabIndent {
		printerMode |= printer.TabIndent
	}
	if *useSpaces {
		printerMode |= printer.UseSpaces
	}
}

func isGoFile(f *os.FileInfo) bool {
	// ignore non-Go files
	return f.IsRegular() && !strings.HasPrefix(f.Name, ".") && strings.HasSuffix(f.Name, ".go")
}

// If in == nil, the source is the contents of the file with the given filename.
func processFile(filename string, in io.Reader, out io.Writer) os.Error {
	if in == nil {
		f, err := os.Open(filename)
		if err != nil {
			return err
		}
		defer f.Close()
		in = f
	}

	src, err := ioutil.ReadAll(in)
	if err != nil {
		return err
	}

	file, err := parser.ParseFile(fset, filename, src, parserMode)
	if err != nil {
		return err
	}

	if rewrite != nil {
		file = rewrite(file)
	}

	if *simplifyAST {
		simplify(file)
	}

	var buf bytes.Buffer
	_, err = (&printer.Config{printerMode, *tabWidth}).Fprint(&buf, fset, file)
	if err != nil {
		return err
	}
	res := buf.Bytes()

	if !bytes.Equal(src, res) {
		// formatting has changed
		if *list {
			fmt.Fprintln(out, filename)
		}
		if *write {
			err = ioutil.WriteFile(filename, res, 0)
			if err != nil {
				return err
			}
		}
		if *doDiff {
			data, err := diff(src, res)
			if err != nil {
				return fmt.Errorf("computing diff: %s", err)
			}
			fmt.Printf("diff %s gofmt/%s\n", filename, filename)
			out.Write(data)
		}
	}

	if !*list && !*write && !*doDiff {
		_, err = out.Write(res)
	}

	return err
}

type fileVisitor chan os.Error

func (v fileVisitor) VisitDir(path string, f *os.FileInfo) bool {
	return true
}

func (v fileVisitor) VisitFile(path string, f *os.FileInfo) {
	if isGoFile(f) {
		v <- nil // synchronize error handler
		if err := processFile(path, nil, os.Stdout); err != nil {
			v <- err
		}
	}
}

func walkDir(path string) {
	v := make(fileVisitor)
	go func() {
		filepath.Walk(path, v, v)
		close(v)
	}()
	for err := range v {
		if err != nil {
			report(err)
		}
	}
}

func main() {
	// call gofmtMain in a separate function
	// so that it can use defer and have them
	// run before the exit.
	gofmtMain()
	os.Exit(exitCode)
}

func gofmtMain() {
	flag.Usage = usage
	flag.Parse()
	if *tabWidth < 0 {
		fmt.Fprintf(os.Stderr, "negative tabwidth %d\n", *tabWidth)
		exitCode = 2
		return
	}

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "creating cpu profile: %s\n", err)
			exitCode = 2
			return
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	initParserMode()
	initPrinterMode()
	initRewrite()

	if flag.NArg() == 0 {
		if err := processFile("<standard input>", os.Stdin, os.Stdout); err != nil {
			report(err)
		}
		return
	}

	for i := 0; i < flag.NArg(); i++ {
		path := flag.Arg(i)
		switch dir, err := os.Stat(path); {
		case err != nil:
			report(err)
		case dir.IsRegular():
			if err := processFile(path, nil, os.Stdout); err != nil {
				report(err)
			}
		case dir.IsDirectory():
			walkDir(path)
		}
	}
}

func diff(b1, b2 []byte) (data []byte, err os.Error) {
	f1, err := ioutil.TempFile("", "gofmt")
	if err != nil {
		return
	}
	defer os.Remove(f1.Name())
	defer f1.Close()

	f2, err := ioutil.TempFile("", "gofmt")
	if err != nil {
		return
	}
	defer os.Remove(f2.Name())
	defer f2.Close()

	f1.Write(b1)
	f2.Write(b2)

	return exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput()
}
<commit_msg>gofmt: fix -d regression from exec change<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"exec"
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/scanner"
	"go/token"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime/pprof"
	"strings"
)

var (
	// main operation modes
	list        = flag.Bool("l", false, "list files whose formatting differs from gofmt's")
	write       = flag.Bool("w", false, "write result to (source) file instead of stdout")
	rewriteRule = flag.String("r", "", "rewrite rule (e.g., 'α[β:len(α)] -> α[β:]')")
	simplifyAST = flag.Bool("s", false, "simplify code")
	doDiff      = flag.Bool("d", false, "display diffs instead of rewriting files")

	// layout control
	comments  = flag.Bool("comments", true, "print comments")
	tabWidth  = flag.Int("tabwidth", 8, "tab width")
	tabIndent = flag.Bool("tabindent", true, "indent with tabs independent of -spaces")
	useSpaces = flag.Bool("spaces", true, "align with spaces instead of tabs")

	// debugging
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to this file")
)

var (
	fset        = token.NewFileSet()
	exitCode    = 0
	rewrite     func(*ast.File) *ast.File
	parserMode  uint
	printerMode uint
)

func report(err os.Error) {
	scanner.PrintError(os.Stderr, err)
	exitCode = 2
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: gofmt [flags] [path ...]\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func initParserMode() {
	parserMode = uint(0)
	if *comments {
		parserMode |= parser.ParseComments
	}
}

func initPrinterMode() {
	printerMode = uint(0)
	if *tabIndent {
		printerMode |= printer.TabIndent
	}
	if *useSpaces {
		printerMode |= printer.UseSpaces
	}
}

func isGoFile(f *os.FileInfo) bool {
	// ignore non-Go files
	return f.IsRegular() && !strings.HasPrefix(f.Name, ".") && strings.HasSuffix(f.Name, ".go")
}

// If in == nil, the source is the contents of the file with the given filename.
func processFile(filename string, in io.Reader, out io.Writer) os.Error {
	if in == nil {
		f, err := os.Open(filename)
		if err != nil {
			return err
		}
		defer f.Close()
		in = f
	}

	src, err := ioutil.ReadAll(in)
	if err != nil {
		return err
	}

	file, err := parser.ParseFile(fset, filename, src, parserMode)
	if err != nil {
		return err
	}

	if rewrite != nil {
		file = rewrite(file)
	}

	if *simplifyAST {
		simplify(file)
	}

	var buf bytes.Buffer
	_, err = (&printer.Config{printerMode, *tabWidth}).Fprint(&buf, fset, file)
	if err != nil {
		return err
	}
	res := buf.Bytes()

	if !bytes.Equal(src, res) {
		// formatting has changed
		if *list {
			fmt.Fprintln(out, filename)
		}
		if *write {
			err = ioutil.WriteFile(filename, res, 0)
			if err != nil {
				return err
			}
		}
		if *doDiff {
			data, err := diff(src, res)
			if err != nil {
				return fmt.Errorf("computing diff: %s", err)
			}
			fmt.Printf("diff %s gofmt/%s\n", filename, filename)
			out.Write(data)
		}
	}

	if !*list && !*write && !*doDiff {
		_, err = out.Write(res)
	}

	return err
}

type fileVisitor chan os.Error

func (v fileVisitor) VisitDir(path string, f *os.FileInfo) bool {
	return true
}

func (v fileVisitor) VisitFile(path string, f *os.FileInfo) {
	if isGoFile(f) {
		v <- nil // synchronize error handler
		if err := processFile(path, nil, os.Stdout); err != nil {
			v <- err
		}
	}
}

func walkDir(path string) {
	v := make(fileVisitor)
	go func() {
		filepath.Walk(path, v, v)
		close(v)
	}()
	for err := range v {
		if err != nil {
			report(err)
		}
	}
}

func main() {
	// call gofmtMain in a separate function
	// so that it can use defer and have them
	// run before the exit.
	gofmtMain()
	os.Exit(exitCode)
}

func gofmtMain() {
	flag.Usage = usage
	flag.Parse()
	if *tabWidth < 0 {
		fmt.Fprintf(os.Stderr, "negative tabwidth %d\n", *tabWidth)
		exitCode = 2
		return
	}

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "creating cpu profile: %s\n", err)
			exitCode = 2
			return
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	initParserMode()
	initPrinterMode()
	initRewrite()

	if flag.NArg() == 0 {
		if err := processFile("<standard input>", os.Stdin, os.Stdout); err != nil {
			report(err)
		}
		return
	}

	for i := 0; i < flag.NArg(); i++ {
		path := flag.Arg(i)
		switch dir, err := os.Stat(path); {
		case err != nil:
			report(err)
		case dir.IsRegular():
			if err := processFile(path, nil, os.Stdout); err != nil {
				report(err)
			}
		case dir.IsDirectory():
			walkDir(path)
		}
	}
}

func diff(b1, b2 []byte) (data []byte, err os.Error) {
	f1, err := ioutil.TempFile("", "gofmt")
	if err != nil {
		return
	}
	defer os.Remove(f1.Name())
	defer f1.Close()

	f2, err := ioutil.TempFile("", "gofmt")
	if err != nil {
		return
	}
	defer os.Remove(f2.Name())
	defer f2.Close()

	f1.Write(b1)
	f2.Write(b2)

	data, err = exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput()
	if len(data) > 0 {
		// diff exits with a non-zero status when the files don't match.
		// Ignore that failure as long as we get output.
		err = nil
	}
	return

}
<|endoftext|>


<commit_before>package neurgo

import (
	"testing"
	"github.com/couchbaselabs/go.assert"
	"sync"
	"time"
)

func TestConnectBidirectional(t *testing.T) {

	// create nodes
	neuron := &Neuron{}
	sensor := &Sensor{}

	// give names
	neuron.Name = "neuron"
	sensor.Name = "sensor"

	// make connection
	weights := []float64{20,20,20,20,20}
	sensor.ConnectBidirectionalWeighted(neuron, weights)

	// assert that it worked
	assert.Equals(t, len(sensor.outbound), 1)
	assert.Equals(t, len(neuron.inbound), 1)
	assert.True(t, neuron.inbound[0].channel != nil)
	assert.True(t, sensor.outbound[0].channel != nil)
	assert.Equals(t, len(neuron.inbound[0].weights), len(weights))
	assert.Equals(t, neuron.inbound[0].weights[0], weights[0])

	// make a new node and connect it
	actuator := &Actuator{}
	neuron.ConnectBidirectional(actuator)

	// assert that it worked
	assert.Equals(t, len(neuron.outbound), 1)
	assert.Equals(t, len(actuator.inbound), 1)
	assert.Equals(t, len(actuator.inbound[0].weights), 0)

}

func TestRemoveConnection(t *testing.T) {

	// create network nodes
	neuron1 := &Neuron{Bias: 10, ActivationFunction: identity_activation}
	neuron2 := &Neuron{Bias: 10, ActivationFunction: identity_activation}
	sensor := &Sensor{}

	// give names to nodes
	neuron1.Name = "neuron1"
	neuron2.Name = "neuron2"
	sensor.Name = "sensor"

	// connect nodes together
	weights := []float64{20,20,20,20,20}
	sensor.ConnectBidirectionalWeighted(neuron1, weights)
	sensor.ConnectBidirectionalWeighted(neuron2, weights)

	// remove connections
	neuron1.inbound = removeConnection(neuron1.inbound, 0)
	sensor.outbound = removeConnection(sensor.outbound, 0)

	// assert that it worked
	assert.Equals(t, len(neuron1.inbound), 0)
	assert.Equals(t, len(neuron2.inbound), 1)
	assert.Equals(t, len(sensor.outbound), 1)
	assert.Equals(t, sensor.outbound[0].channel, neuron2.inbound[0].channel)

}

func TestRemoveConnectionFromRunningNode(t *testing.T) {

	// create nodes
	sensor1 := &Sensor{}
	sensor2 := &Sensor{}
	neuron := &Neuron{Bias: 10, ActivationFunction: identity_activation}

	// give names to nodes
	neuron.Name = "neuron"
	sensor1.Name = "sensor1"
	sensor2.Name = "sensor2"

	// connect nodes together
	weights := []float64{20}
	sensor1.ConnectBidirectionalWeighted(neuron, weights)
	sensor2.ConnectBidirectionalWeighted(neuron, weights)

	// TODO
	// close other channel
	// call weightedInputs and get zero results

	// basic sanity check, send two inputs to neuron inbound channels
	// and verify that weightedInputs() returns both inputs
	go func() {
		sensor1.outbound[0].channel <- []float64{0}
	}()
	go func() {
		sensor2.outbound[0].channel <- []float64{0}
	}()
	weightedInputs := neuron.weightedInputs()
	assert.Equals(t, len(weightedInputs), 2)

	// close one channel while a neuron is reading from
	// both inbound connections, make sure it returns one value

	var wg sync.WaitGroup
	wg.Add(1)
	wg.Add(1)

	go func() {
		weightedInputs := neuron.weightedInputs()
		assert.Equals(t, len(weightedInputs), 1)
		wg.Done()
	}()

	go func() {

		// need to sleep so that we can be sure that the other go-routine
		// is blocked on the channel read of its inbound channels
		time.Sleep(0.1 * 1e9)

		sensor1.DisconnectBidirectional(neuron)
		sensor2.outbound[0].channel <- []float64{0}
		wg.Done()
	}()

	wg.Wait()

}

func identity_activation(x float64) float64 {
	return x
}
<commit_msg>add test that shows that actuators don't handle input connections being removed<commit_after>package neurgo

import (
	"testing"
	"github.com/couchbaselabs/go.assert"
	"sync"
	"time"
)

func TestConnectBidirectional(t *testing.T) {

	// create nodes
	neuron := &Neuron{}
	sensor := &Sensor{}

	// give names
	neuron.Name = "neuron"
	sensor.Name = "sensor"

	// make connection
	weights := []float64{20,20,20,20,20}
	sensor.ConnectBidirectionalWeighted(neuron, weights)

	// assert that it worked
	assert.Equals(t, len(sensor.outbound), 1)
	assert.Equals(t, len(neuron.inbound), 1)
	assert.True(t, neuron.inbound[0].channel != nil)
	assert.True(t, sensor.outbound[0].channel != nil)
	assert.Equals(t, len(neuron.inbound[0].weights), len(weights))
	assert.Equals(t, neuron.inbound[0].weights[0], weights[0])

	// make a new node and connect it
	actuator := &Actuator{}
	neuron.ConnectBidirectional(actuator)

	// assert that it worked
	assert.Equals(t, len(neuron.outbound), 1)
	assert.Equals(t, len(actuator.inbound), 1)
	assert.Equals(t, len(actuator.inbound[0].weights), 0)

}

func TestRemoveConnection(t *testing.T) {

	// create network nodes
	neuron1 := &Neuron{Bias: 10, ActivationFunction: identity_activation}
	neuron2 := &Neuron{Bias: 10, ActivationFunction: identity_activation}
	sensor := &Sensor{}

	// give names to nodes
	neuron1.Name = "neuron1"
	neuron2.Name = "neuron2"
	sensor.Name = "sensor"

	// connect nodes together
	weights := []float64{20,20,20,20,20}
	sensor.ConnectBidirectionalWeighted(neuron1, weights)
	sensor.ConnectBidirectionalWeighted(neuron2, weights)

	// remove connections
	neuron1.inbound = removeConnection(neuron1.inbound, 0)
	sensor.outbound = removeConnection(sensor.outbound, 0)

	// assert that it worked
	assert.Equals(t, len(neuron1.inbound), 0)
	assert.Equals(t, len(neuron2.inbound), 1)
	assert.Equals(t, len(sensor.outbound), 1)
	assert.Equals(t, sensor.outbound[0].channel, neuron2.inbound[0].channel)

}

func TestRemoveConnectionFromRunningNeuron(t *testing.T) {

	// create nodes
	sensor1 := &Sensor{}
	sensor2 := &Sensor{}
	neuron := &Neuron{Bias: 10, ActivationFunction: identity_activation}

	// give names to nodes
	neuron.Name = "neuron"
	sensor1.Name = "sensor1"
	sensor2.Name = "sensor2"

	// connect nodes together
	weights := []float64{20}
	sensor1.ConnectBidirectionalWeighted(neuron, weights)
	sensor2.ConnectBidirectionalWeighted(neuron, weights)

	// basic sanity check, send two inputs to neuron inbound channels
	// and verify that weightedInputs() returns both inputs
	go func() {
		sensor1.outbound[0].channel <- []float64{0}
	}()
	go func() {
		sensor2.outbound[0].channel <- []float64{0}
	}()
	weightedInputs := neuron.weightedInputs()
	assert.Equals(t, len(weightedInputs), 2)

	// close one channel while a neuron is reading from
	// both inbound connections, make sure it returns one value
	var wg sync.WaitGroup
	wg.Add(1)
	wg.Add(1)

	go func() {
		weightedInputs := neuron.weightedInputs()
		assert.Equals(t, len(weightedInputs), 1)
		wg.Done()
	}()

	go func() {

		// need to sleep so that we can be sure that the other go-routine
		// is blocked on the channel read of its inbound channels
		time.Sleep(0.1 * 1e9)

		sensor1.DisconnectBidirectional(neuron)
		sensor2.outbound[0].channel <- []float64{0}
		wg.Done()
	}()

	wg.Wait()

}

func TestRemoveConnectionFromRunningActuator(t *testing.T) {

	// create nodes
	neuron1 := &Neuron{Bias: 10, ActivationFunction: identity_activation}
	neuron2 := &Neuron{Bias: 10, ActivationFunction: identity_activation}
	actuator := &Actuator{}

	// give names to nodes
	neuron1.Name = "neuron1"
	neuron2.Name = "neuron2"
	actuator.Name = "actuator"

	// connect nodes together
	neuron1.ConnectBidirectional(actuator)
	neuron2.ConnectBidirectional(actuator)

	var wg sync.WaitGroup
	wg.Add(1)
	wg.Add(1)

	go func() {
		inputs := actuator.gatherInputs()
		assert.Equals(t, len(inputs), 1)
		wg.Done()
	}()

	go func() {

		// need to sleep so that we can be sure that the other go-routine
		// is blocked on the channel read of its inbound channels
		time.Sleep(0.1 * 1e9)

		neuron1.DisconnectBidirectional(actuator)
		neuron2.outbound[0].channel <- []float64{0}
		wg.Done()
	}()

	wg.Wait()

}

func identity_activation(x float64) float64 {
	return x
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vsphere

import (
	"sync"

	"k8s.io/api/core/v1"
	clientv1 "k8s.io/client-go/listers/core/v1"
	"k8s.io/cloud-provider-vsphere/pkg/vclib"
	"k8s.io/kubernetes/pkg/cloudprovider"
)

// VSphere is an implementation of cloud provider Interface for VSphere.
type VSphere struct {
	// client        *godo.Client
	// cfg                *Config
	vsphereInstanceMap map[string]*VSphereInstance
	nodeManager        *NodeManager
	instances          cloudprovider.Instances
}

// Config is used to read and store information from the cloud configuration file
type Config struct {
	Global struct {
		// vCenter username.
		User string `gcfg:"user"`
		// vCenter password in clear text.
		Password string `gcfg:"password"`
		// Deprecated. Use VirtualCenter to specify multiple vCenter Servers.
		// vCenter IP.
		VCenterIP string `gcfg:"server"`
		// vCenter port.
		VCenterPort string `gcfg:"port"`
		// True if vCenter uses self-signed cert.
		InsecureFlag bool `gcfg:"insecure-flag"`
		// Datacenter in which VMs are located.
		Datacenters string `gcfg:"datacenters"`
		// Soap round tripper count (retries = RoundTripper - 1)
		RoundTripperCount uint `gcfg:"soap-roundtrip-count"`
		// Name of the secret were vCenter credentials are present.
		SecretName string `gcfg:"secret-name"`
		// Secret Namespace where secret will be present that has vCenter credentials.
		SecretNamespace string `gcfg:"secret-namespace"`
	}
	VirtualCenter map[string]*VirtualCenterConfig

	Network struct {
		// PublicNetwork is name of the network the VMs are joined to.
		PublicNetwork string `gcfg:"public-network"`
	}
}

// Represents a vSphere instance where one or more kubernetes nodes are running.
type VSphereInstance struct {
	conn *vclib.VSphereConnection
	cfg  *VirtualCenterConfig
}

// Structure that represents Virtual Center configuration
type VirtualCenterConfig struct {
	// vCenter username.
	User string `gcfg:"user"`
	// vCenter password in clear text.
	Password string `gcfg:"password"`
	// vCenter port.
	VCenterPort string `gcfg:"port"`
	// Datacenter in which VMs are located.
	Datacenters string `gcfg:"datacenters"`
	// Soap round tripper count (retries = RoundTripper - 1)
	RoundTripperCount uint `gcfg:"soap-roundtrip-count"`
	// PublicNetwork is name of the network the VMs are joined to.
	PublicNetwork string `gcfg:"public-network"`
}

// Stores info about the kubernetes node
type NodeInfo struct {
	dataCenter    *vclib.Datacenter
	vm            *vclib.VirtualMachine
	vcServer      string
	UUID          string
	NodeName      string
	NodeAddresses []v1.NodeAddress
}

type NodeManager struct {
	// TODO: replace map with concurrent map when k8s supports go v1.9

	// Maps the VC server to VSphereInstance
	vsphereInstanceMap map[string]*VSphereInstance
	// Maps node name to node info.
	nodeInfoMap map[string]*NodeInfo
	//CredentialsManager
	credentialManager *SecretCredentialManager

	// Mutexes
	nodeInfoLock          sync.RWMutex
	credentialManagerLock sync.Mutex
}

type NodeDetails struct {
	NodeName string
	vm       *vclib.VirtualMachine
	UUID     string
}

type SecretCache struct {
	cacheLock     sync.Mutex
	VirtualCenter map[string]*Credential
	Secret        *v1.Secret
}

type Credential struct {
	User     string `gcfg:"user"`
	Password string `gcfg:"password"`
}

type SecretCredentialManager struct {
	SecretName      string
	SecretNamespace string
	SecretLister    clientv1.SecretLister
	Cache           *SecretCache
}

type instances struct {
	nodeManager *NodeManager
}
<commit_msg>Removing dead code<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vsphere

import (
	"sync"

	"k8s.io/api/core/v1"
	clientv1 "k8s.io/client-go/listers/core/v1"
	"k8s.io/cloud-provider-vsphere/pkg/vclib"
	"k8s.io/kubernetes/pkg/cloudprovider"
)

// VSphere is an implementation of cloud provider Interface for VSphere.
type VSphere struct {
	// client        *godo.Client
	// cfg                *Config
	vsphereInstanceMap map[string]*VSphereInstance
	nodeManager        *NodeManager
	instances          cloudprovider.Instances
}

// Config is used to read and store information from the cloud configuration file
type Config struct {
	Global struct {
		// vCenter username.
		User string `gcfg:"user"`
		// vCenter password in clear text.
		Password string `gcfg:"password"`
		// Deprecated. Use VirtualCenter to specify multiple vCenter Servers.
		// vCenter IP.
		VCenterIP string `gcfg:"server"`
		// vCenter port.
		VCenterPort string `gcfg:"port"`
		// True if vCenter uses self-signed cert.
		InsecureFlag bool `gcfg:"insecure-flag"`
		// Datacenter in which VMs are located.
		Datacenters string `gcfg:"datacenters"`
		// Soap round tripper count (retries = RoundTripper - 1)
		RoundTripperCount uint `gcfg:"soap-roundtrip-count"`
		// Name of the secret were vCenter credentials are present.
		SecretName string `gcfg:"secret-name"`
		// Secret Namespace where secret will be present that has vCenter credentials.
		SecretNamespace string `gcfg:"secret-namespace"`
	}
	VirtualCenter map[string]*VirtualCenterConfig

	Network struct {
		// PublicNetwork is name of the network the VMs are joined to.
		PublicNetwork string `gcfg:"public-network"`
	}
}

// Represents a vSphere instance where one or more kubernetes nodes are running.
type VSphereInstance struct {
	conn *vclib.VSphereConnection
	cfg  *VirtualCenterConfig
}

// Structure that represents Virtual Center configuration
type VirtualCenterConfig struct {
	// vCenter username.
	User string `gcfg:"user"`
	// vCenter password in clear text.
	Password string `gcfg:"password"`
	// vCenter port.
	VCenterPort string `gcfg:"port"`
	// Datacenter in which VMs are located.
	Datacenters string `gcfg:"datacenters"`
	// Soap round tripper count (retries = RoundTripper - 1)
	RoundTripperCount uint `gcfg:"soap-roundtrip-count"`
	// PublicNetwork is name of the network the VMs are joined to.
	PublicNetwork string `gcfg:"public-network"`
}

// Stores info about the kubernetes node
type NodeInfo struct {
	dataCenter    *vclib.Datacenter
	vm            *vclib.VirtualMachine
	vcServer      string
	UUID          string
	NodeName      string
	NodeAddresses []v1.NodeAddress
}

type NodeManager struct {
	// TODO: replace map with concurrent map when k8s supports go v1.9

	// Maps the VC server to VSphereInstance
	vsphereInstanceMap map[string]*VSphereInstance
	// Maps node name to node info.
	nodeInfoMap map[string]*NodeInfo
	//CredentialsManager
	credentialManager *SecretCredentialManager

	// Mutexes
	nodeInfoLock          sync.RWMutex
	credentialManagerLock sync.Mutex
}

type NodeDetails struct {
	NodeName string
	UUID     string
}

type SecretCache struct {
	cacheLock     sync.Mutex
	VirtualCenter map[string]*Credential
	Secret        *v1.Secret
}

type Credential struct {
	User     string `gcfg:"user"`
	Password string `gcfg:"password"`
}

type SecretCredentialManager struct {
	SecretName      string
	SecretNamespace string
	SecretLister    clientv1.SecretLister
	Cache           *SecretCache
}

type instances struct {
	nodeManager *NodeManager
}
<|endoftext|>


<commit_before>// Copyright 2015-present Oursky Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logging

import (
	"fmt"
	"regexp"

	"github.com/sirupsen/logrus"
)

const (
	maskedResult                 = "********"
	maskJWTAccessTokenPatternStr = "[A-Za-z0-9-_=]+\\.[A-Za-z0-9-_=]+\\.?[A-Za-z0-9-_.+/=]*"
)

var MaskKeyPattern *regexp.Regexp
var MaskJWTAccessTokenPattern *regexp.Regexp

func init() {
	// pattern for JWT access token
	MaskJWTAccessTokenPattern, _ = regexp.Compile(maskJWTAccessTokenPatternStr)
}

func MakeMaskPattern(input string) (*regexp.Regexp, error) {
	p := fmt.Sprintf("\\b%s\\b", input)
	return regexp.Compile(p)
}

type MaskFormatter struct {
	MaskPatterns     []*regexp.Regexp
	DefaultFormatter logrus.Formatter
}

func (f *MaskFormatter) Format(entry *logrus.Entry) ([]byte, error) {
	fields := entry.Data
	for k, v := range entry.Data {
		switch m := v.(type) {
		case string:
			fields[k] = f.mask(m)
		}
	}
	// set masked message back to entry
	entry.Message = f.mask(entry.Message)
	return f.DefaultFormatter.Format(entry)
}

func (f *MaskFormatter) mask(src string) (output string) {
	output = src
	for _, p := range f.MaskPatterns {
		output = p.ReplaceAllString(output, maskedResult)
	}
	return output
}
<commit_msg>Mark the high entropy sting as save by ignore gosec<commit_after>// Copyright 2015-present Oursky Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logging

import (
	"fmt"
	"regexp"

	"github.com/sirupsen/logrus"
)

// nolint: gosec
const (
	maskedResult                 = "********"
	maskJWTAccessTokenPatternStr = "[A-Za-z0-9-_=]+\\.[A-Za-z0-9-_=]+\\.?[A-Za-z0-9-_.+/=]*"
)

var MaskKeyPattern *regexp.Regexp
var MaskJWTAccessTokenPattern *regexp.Regexp

func init() {
	// pattern for JWT access token
	MaskJWTAccessTokenPattern, _ = regexp.Compile(maskJWTAccessTokenPatternStr)
}

func MakeMaskPattern(input string) (*regexp.Regexp, error) {
	p := fmt.Sprintf("\\b%s\\b", input)
	return regexp.Compile(p)
}

type MaskFormatter struct {
	MaskPatterns     []*regexp.Regexp
	DefaultFormatter logrus.Formatter
}

func (f *MaskFormatter) Format(entry *logrus.Entry) ([]byte, error) {
	fields := entry.Data
	for k, v := range entry.Data {
		switch m := v.(type) {
		case string:
			fields[k] = f.mask(m)
		}
	}
	// set masked message back to entry
	entry.Message = f.mask(entry.Message)
	return f.DefaultFormatter.Format(entry)
}

func (f *MaskFormatter) mask(src string) (output string) {
	output = src
	for _, p := range f.MaskPatterns {
		output = p.ReplaceAllString(output, maskedResult)
	}
	return output
}
<|endoftext|>


<commit_before>package sqlstore

import (
	"bytes"
	"fmt"
	"time"

	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/metrics"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/services/search"
)

func init() {
	bus.AddHandler("sql", SaveDashboard)
	bus.AddHandler("sql", GetDashboard)
	bus.AddHandler("sql", GetDashboards)
	bus.AddHandler("sql", DeleteDashboard)
	bus.AddHandler("sql", SearchDashboards)
	bus.AddHandler("sql", GetDashboardTags)
	bus.AddHandler("sql", GetDashboardSlugById)
	bus.AddHandler("sql", GetDashboardsByPluginId)
}

func SaveDashboard(cmd *m.SaveDashboardCommand) error {
	return inTransaction(func(sess *DBSession) error {
		dash := cmd.GetDashboardModel()

		// try get existing dashboard
		var existing, sameTitle m.Dashboard

		if dash.Id > 0 {
			dashWithIdExists, err := sess.Where("id=? AND org_id=?", dash.Id, dash.OrgId).Get(&existing)
			if err != nil {
				return err
			}
			if !dashWithIdExists {
				return m.ErrDashboardNotFound
			}

			// check for is someone else has written in between
			if dash.Version != existing.Version {
				if cmd.Overwrite {
					dash.Version = existing.Version
				} else {
					return m.ErrDashboardVersionMismatch
				}
			}

			// do not allow plugin dashboard updates without overwrite flag
			if existing.PluginId != "" && cmd.Overwrite == false {
				return m.UpdatePluginDashboardError{PluginId: existing.PluginId}
			}
		}

		sameTitleExists, err := sess.Where("org_id=? AND slug=?", dash.OrgId, dash.Slug).Get(&sameTitle)
		if err != nil {
			return err
		}

		if sameTitleExists {
			// another dashboard with same name
			if dash.Id != sameTitle.Id {
				if cmd.Overwrite {
					dash.Id = sameTitle.Id
				} else {
					return m.ErrDashboardWithSameNameExists
				}
			}
		}

		parentVersion := dash.Version
		affectedRows := int64(0)

		if dash.Id == 0 {
			metrics.M_Models_Dashboard_Insert.Inc(1)
			dash.Data.Set("version", dash.Version)
			affectedRows, err = sess.Insert(dash)
		} else {
			dash.Version += 1
			dash.Data.Set("version", dash.Version)
			affectedRows, err = sess.Id(dash.Id).Update(dash)
		}

		if err != nil {
			return err
		}

		if affectedRows == 0 {
			return m.ErrDashboardNotFound
		}

		dashVersion := &m.DashboardVersion{
			DashboardId:   dash.Id,
			ParentVersion: parentVersion,
			RestoredFrom:  cmd.RestoredFrom,
			Version:       dash.Version,
			Created:       time.Now(),
			CreatedBy:     dash.UpdatedBy,
			Message:       cmd.Message,
			Data:          dash.Data,
		}

		// insert version entry
		if affectedRows, err = sess.Insert(dashVersion); err != nil {
			return err
		} else if affectedRows == 0 {
			return m.ErrDashboardNotFound
		}

		// delete existing tabs
		_, err = sess.Exec("DELETE FROM dashboard_tag WHERE dashboard_id=?", dash.Id)
		if err != nil {
			return err
		}

		// insert new tags
		tags := dash.GetTags()
		if len(tags) > 0 {
			for _, tag := range tags {
				if _, err := sess.Insert(&DashboardTag{DashboardId: dash.Id, Term: tag}); err != nil {
					return err
				}
			}
		}

		cmd.Result = dash

		return err
	})
}

func GetDashboard(query *m.GetDashboardQuery) error {
	dashboard := m.Dashboard{Slug: query.Slug, OrgId: query.OrgId, Id: query.Id}
	has, err := x.Get(&dashboard)

	if err != nil {
		return err
	} else if has == false {
		return m.ErrDashboardNotFound
	}

	dashboard.Data.Set("id", dashboard.Id)
	query.Result = &dashboard
	return nil
}

type DashboardSearchProjection struct {
	Id    int64
	Title string
	Slug  string
	Term  string
}

func SearchDashboards(query *search.FindPersistedDashboardsQuery) error {
	var sql bytes.Buffer
	params := make([]interface{}, 0)

	sql.WriteString(`SELECT
					  dashboard.id,
					  dashboard.title,
					  dashboard.slug,
					  dashboard_tag.term
					FROM dashboard
					LEFT OUTER JOIN dashboard_tag on dashboard_tag.dashboard_id = dashboard.id`)

	if query.IsStarred {
		sql.WriteString(" INNER JOIN star on star.dashboard_id = dashboard.id")
	}

	sql.WriteString(` WHERE dashboard.org_id=?`)

	params = append(params, query.OrgId)

	if query.IsStarred {
		sql.WriteString(` AND star.user_id=?`)
		params = append(params, query.UserId)
	}

	if len(query.DashboardIds) > 0 {
		sql.WriteString(" AND (")
		for i, dashboardId := range query.DashboardIds {
			if i != 0 {
				sql.WriteString(" OR")
			}

			sql.WriteString(" dashboard.id = ?")
			params = append(params, dashboardId)
		}
		sql.WriteString(")")
	}

	if len(query.Title) > 0 {
		sql.WriteString(" AND dashboard.title " + dialect.LikeStr() + " ?")
		params = append(params, "%"+query.Title+"%")
	}

	sql.WriteString(fmt.Sprintf(" ORDER BY dashboard.title ASC LIMIT 1000"))

	var res []DashboardSearchProjection

	err := x.Sql(sql.String(), params...).Find(&res)
	if err != nil {
		return err
	}

	query.Result = make([]*search.Hit, 0)
	hits := make(map[int64]*search.Hit)

	for _, item := range res {
		hit, exists := hits[item.Id]
		if !exists {
			hit = &search.Hit{
				Id:    item.Id,
				Title: item.Title,
				Uri:   "db/" + item.Slug,
				Type:  search.DashHitDB,
				Tags:  []string{},
			}
			query.Result = append(query.Result, hit)
			hits[item.Id] = hit
		}
		if len(item.Term) > 0 {
			hit.Tags = append(hit.Tags, item.Term)
		}
	}

	return err
}

func GetDashboardTags(query *m.GetDashboardTagsQuery) error {
	sql := `SELECT
					  COUNT(*) as count,
						term
					FROM dashboard
					INNER JOIN dashboard_tag on dashboard_tag.dashboard_id = dashboard.id
					WHERE dashboard.org_id=?
					GROUP BY term`

	query.Result = make([]*m.DashboardTagCloudItem, 0)
	sess := x.Sql(sql, query.OrgId)
	err := sess.Find(&query.Result)
	return err
}

func DeleteDashboard(cmd *m.DeleteDashboardCommand) error {
	return inTransaction(func(sess *DBSession) error {
		dashboard := m.Dashboard{Slug: cmd.Slug, OrgId: cmd.OrgId}
		has, err := sess.Get(&dashboard)
		if err != nil {
			return err
		} else if has == false {
			return m.ErrDashboardNotFound
		}

		deletes := []string{
			"DELETE FROM dashboard_tag WHERE dashboard_id = ? ",
			"DELETE FROM star WHERE dashboard_id = ? ",
			"DELETE FROM dashboard WHERE id = ?",
			"DELETE FROM playlist_item WHERE type = 'dashboard_by_id' AND value = ?",
			"DELETE FROM dashboard_version WHERE dashboard_id = ?",
		}

		for _, sql := range deletes {
			_, err := sess.Exec(sql, dashboard.Id)
			if err != nil {
				return err
			}
		}

		if err := DeleteAlertDefinition(dashboard.Id, sess); err != nil {
			return nil
		}

		return nil
	})
}

func GetDashboards(query *m.GetDashboardsQuery) error {
	if len(query.DashboardIds) == 0 {
		return m.ErrCommandValidationFailed
	}

	var dashboards = make([]*m.Dashboard, 0)

	err := x.In("id", query.DashboardIds).Find(&dashboards)
	query.Result = dashboards

	if err != nil {
		return err
	}

	return nil
}

func GetDashboardsByPluginId(query *m.GetDashboardsByPluginIdQuery) error {
	var dashboards = make([]*m.Dashboard, 0)

	err := x.Where("org_id=? AND plugin_id=?", query.OrgId, query.PluginId).Find(&dashboards)
	query.Result = dashboards

	if err != nil {
		return err
	}

	return nil
}

type DashboardSlugDTO struct {
	Slug string
}

func GetDashboardSlugById(query *m.GetDashboardSlugByIdQuery) error {
	var rawSql = `SELECT slug from dashboard WHERE Id=?`
	var slug = DashboardSlugDTO{}

	exists, err := x.Sql(rawSql, query.Id).Get(&slug)

	if err != nil {
		return err
	} else if exists == false {
		return m.ErrDashboardNotFound
	}

	query.Result = slug.Slug
	return nil
}
<commit_msg>dashboard_history: fixed issue with save as & overwrite<commit_after>package sqlstore

import (
	"bytes"
	"fmt"
	"time"

	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/metrics"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/services/search"
)

func init() {
	bus.AddHandler("sql", SaveDashboard)
	bus.AddHandler("sql", GetDashboard)
	bus.AddHandler("sql", GetDashboards)
	bus.AddHandler("sql", DeleteDashboard)
	bus.AddHandler("sql", SearchDashboards)
	bus.AddHandler("sql", GetDashboardTags)
	bus.AddHandler("sql", GetDashboardSlugById)
	bus.AddHandler("sql", GetDashboardsByPluginId)
}

func SaveDashboard(cmd *m.SaveDashboardCommand) error {
	return inTransaction(func(sess *DBSession) error {
		dash := cmd.GetDashboardModel()

		// try get existing dashboard
		var existing, sameTitle m.Dashboard

		if dash.Id > 0 {
			dashWithIdExists, err := sess.Where("id=? AND org_id=?", dash.Id, dash.OrgId).Get(&existing)
			if err != nil {
				return err
			}
			if !dashWithIdExists {
				return m.ErrDashboardNotFound
			}

			// check for is someone else has written in between
			if dash.Version != existing.Version {
				if cmd.Overwrite {
					dash.Version = existing.Version
				} else {
					return m.ErrDashboardVersionMismatch
				}
			}

			// do not allow plugin dashboard updates without overwrite flag
			if existing.PluginId != "" && cmd.Overwrite == false {
				return m.UpdatePluginDashboardError{PluginId: existing.PluginId}
			}
		}

		sameTitleExists, err := sess.Where("org_id=? AND slug=?", dash.OrgId, dash.Slug).Get(&sameTitle)
		if err != nil {
			return err
		}

		if sameTitleExists {
			// another dashboard with same name
			if dash.Id != sameTitle.Id {
				if cmd.Overwrite {
					dash.Id = sameTitle.Id
					dash.Version = sameTitle.Version
				} else {
					return m.ErrDashboardWithSameNameExists
				}
			}
		}

		parentVersion := dash.Version
		affectedRows := int64(0)

		if dash.Id == 0 {
			metrics.M_Models_Dashboard_Insert.Inc(1)
			dash.Data.Set("version", dash.Version)
			affectedRows, err = sess.Insert(dash)
		} else {
			dash.Version += 1
			dash.Data.Set("version", dash.Version)
			affectedRows, err = sess.Id(dash.Id).Update(dash)
		}

		if err != nil {
			return err
		}

		if affectedRows == 0 {
			return m.ErrDashboardNotFound
		}

		dashVersion := &m.DashboardVersion{
			DashboardId:   dash.Id,
			ParentVersion: parentVersion,
			RestoredFrom:  cmd.RestoredFrom,
			Version:       dash.Version,
			Created:       time.Now(),
			CreatedBy:     dash.UpdatedBy,
			Message:       cmd.Message,
			Data:          dash.Data,
		}

		// insert version entry
		if affectedRows, err = sess.Insert(dashVersion); err != nil {
			return err
		} else if affectedRows == 0 {
			return m.ErrDashboardNotFound
		}

		// delete existing tabs
		_, err = sess.Exec("DELETE FROM dashboard_tag WHERE dashboard_id=?", dash.Id)
		if err != nil {
			return err
		}

		// insert new tags
		tags := dash.GetTags()
		if len(tags) > 0 {
			for _, tag := range tags {
				if _, err := sess.Insert(&DashboardTag{DashboardId: dash.Id, Term: tag}); err != nil {
					return err
				}
			}
		}

		cmd.Result = dash

		return err
	})
}

func GetDashboard(query *m.GetDashboardQuery) error {
	dashboard := m.Dashboard{Slug: query.Slug, OrgId: query.OrgId, Id: query.Id}
	has, err := x.Get(&dashboard)

	if err != nil {
		return err
	} else if has == false {
		return m.ErrDashboardNotFound
	}

	dashboard.Data.Set("id", dashboard.Id)
	query.Result = &dashboard
	return nil
}

type DashboardSearchProjection struct {
	Id    int64
	Title string
	Slug  string
	Term  string
}

func SearchDashboards(query *search.FindPersistedDashboardsQuery) error {
	var sql bytes.Buffer
	params := make([]interface{}, 0)

	sql.WriteString(`SELECT
					  dashboard.id,
					  dashboard.title,
					  dashboard.slug,
					  dashboard_tag.term
					FROM dashboard
					LEFT OUTER JOIN dashboard_tag on dashboard_tag.dashboard_id = dashboard.id`)

	if query.IsStarred {
		sql.WriteString(" INNER JOIN star on star.dashboard_id = dashboard.id")
	}

	sql.WriteString(` WHERE dashboard.org_id=?`)

	params = append(params, query.OrgId)

	if query.IsStarred {
		sql.WriteString(` AND star.user_id=?`)
		params = append(params, query.UserId)
	}

	if len(query.DashboardIds) > 0 {
		sql.WriteString(" AND (")
		for i, dashboardId := range query.DashboardIds {
			if i != 0 {
				sql.WriteString(" OR")
			}

			sql.WriteString(" dashboard.id = ?")
			params = append(params, dashboardId)
		}
		sql.WriteString(")")
	}

	if len(query.Title) > 0 {
		sql.WriteString(" AND dashboard.title " + dialect.LikeStr() + " ?")
		params = append(params, "%"+query.Title+"%")
	}

	sql.WriteString(fmt.Sprintf(" ORDER BY dashboard.title ASC LIMIT 1000"))

	var res []DashboardSearchProjection

	err := x.Sql(sql.String(), params...).Find(&res)
	if err != nil {
		return err
	}

	query.Result = make([]*search.Hit, 0)
	hits := make(map[int64]*search.Hit)

	for _, item := range res {
		hit, exists := hits[item.Id]
		if !exists {
			hit = &search.Hit{
				Id:    item.Id,
				Title: item.Title,
				Uri:   "db/" + item.Slug,
				Type:  search.DashHitDB,
				Tags:  []string{},
			}
			query.Result = append(query.Result, hit)
			hits[item.Id] = hit
		}
		if len(item.Term) > 0 {
			hit.Tags = append(hit.Tags, item.Term)
		}
	}

	return err
}

func GetDashboardTags(query *m.GetDashboardTagsQuery) error {
	sql := `SELECT
					  COUNT(*) as count,
						term
					FROM dashboard
					INNER JOIN dashboard_tag on dashboard_tag.dashboard_id = dashboard.id
					WHERE dashboard.org_id=?
					GROUP BY term`

	query.Result = make([]*m.DashboardTagCloudItem, 0)
	sess := x.Sql(sql, query.OrgId)
	err := sess.Find(&query.Result)
	return err
}

func DeleteDashboard(cmd *m.DeleteDashboardCommand) error {
	return inTransaction(func(sess *DBSession) error {
		dashboard := m.Dashboard{Slug: cmd.Slug, OrgId: cmd.OrgId}
		has, err := sess.Get(&dashboard)
		if err != nil {
			return err
		} else if has == false {
			return m.ErrDashboardNotFound
		}

		deletes := []string{
			"DELETE FROM dashboard_tag WHERE dashboard_id = ? ",
			"DELETE FROM star WHERE dashboard_id = ? ",
			"DELETE FROM dashboard WHERE id = ?",
			"DELETE FROM playlist_item WHERE type = 'dashboard_by_id' AND value = ?",
			"DELETE FROM dashboard_version WHERE dashboard_id = ?",
		}

		for _, sql := range deletes {
			_, err := sess.Exec(sql, dashboard.Id)
			if err != nil {
				return err
			}
		}

		if err := DeleteAlertDefinition(dashboard.Id, sess); err != nil {
			return nil
		}

		return nil
	})
}

func GetDashboards(query *m.GetDashboardsQuery) error {
	if len(query.DashboardIds) == 0 {
		return m.ErrCommandValidationFailed
	}

	var dashboards = make([]*m.Dashboard, 0)

	err := x.In("id", query.DashboardIds).Find(&dashboards)
	query.Result = dashboards

	if err != nil {
		return err
	}

	return nil
}

func GetDashboardsByPluginId(query *m.GetDashboardsByPluginIdQuery) error {
	var dashboards = make([]*m.Dashboard, 0)

	err := x.Where("org_id=? AND plugin_id=?", query.OrgId, query.PluginId).Find(&dashboards)
	query.Result = dashboards

	if err != nil {
		return err
	}

	return nil
}

type DashboardSlugDTO struct {
	Slug string
}

func GetDashboardSlugById(query *m.GetDashboardSlugByIdQuery) error {
	var rawSql = `SELECT slug from dashboard WHERE Id=?`
	var slug = DashboardSlugDTO{}

	exists, err := x.Sql(rawSql, query.Id).Get(&slug)

	if err != nil {
		return err
	} else if exists == false {
		return m.ErrDashboardNotFound
	}

	query.Result = slug.Slug
	return nil
}
<|endoftext|>


<commit_before>package discover

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/kardianos/osext"
)

const (
	nomadExe = "nomad"
)

// Checks the current executable, then $GOPATH/bin, and finally the CWD, in that
// order. If it can't be found, an error is returned.
func NomadExecutable() (string, error) {
	// Check the current executable.
	bin, err := osext.Executable()
	if err != nil {
		return "", fmt.Errorf("Failed to determine the nomad executable: %v", err)
	}

	if filepath.Base(bin) == nomadExe {
		return bin, nil
	}

	// Check the $GOPATH.
	bin = filepath.Join(os.Getenv("GOPATH"), "bin", nomadExe)
	if _, err := os.Stat(bin); err == nil {
		return bin, nil
	}

	// Check the CWD.
	bin = filepath.Join(os.Getenv("GOPATH"), "bin", nomadExe)
	if _, err := os.Stat(bin); err == nil {
		return bin, nil
	}

	pwd, err := os.Getwd()
	if err != nil {
		return "", fmt.Errorf("Could not find Nomad executable (%v): %v", err)
	}

	bin = filepath.Join(pwd, nomadExe)
	if _, err := os.Stat(bin); err == nil {
		return bin, nil
	}

	return "", fmt.Errorf("Could not find Nomad executable (%v)", nomadExe)
}
<commit_msg>Correctly scan the CWD for the nomad executable<commit_after>package discover

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/kardianos/osext"
)

const (
	nomadExe = "nomad"
)

// Checks the current executable, then $GOPATH/bin, and finally the CWD, in that
// order. If it can't be found, an error is returned.
func NomadExecutable() (string, error) {
	// Check the current executable.
	bin, err := osext.Executable()
	if err != nil {
		return "", fmt.Errorf("Failed to determine the nomad executable: %v", err)
	}

	if filepath.Base(bin) == nomadExe {
		return bin, nil
	}

	// Check the $GOPATH.
	bin = filepath.Join(os.Getenv("GOPATH"), "bin", nomadExe)
	if _, err := os.Stat(bin); err == nil {
		return bin, nil
	}

	// Check the CWD.
	pwd, err := os.Getwd()
	if err != nil {
		return "", fmt.Errorf("Could not find Nomad executable (%v): %v", err)
	}

	bin = filepath.Join(pwd, nomadExe)
	if _, err := os.Stat(bin); err == nil {
		return bin, nil
	}

	// Check CWD/bin
	bin = filepath.Join(pwd, "bin", nomadExe)
	if _, err := os.Stat(bin); err == nil {
		return bin, nil
	}

	return "", fmt.Errorf("Could not find Nomad executable (%v)", nomadExe)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package service

import (
	"fmt"

	kubeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
)

// CreateContainer creates a new container in specified PodSandbox
func (u *UnikernelRuntime) CreateContainer(podSandboxID string, config *kubeapi.ContainerConfig, sandboxConfig *kubeapi.PodSandboxConfig) (string, error) {
	return "", fmt.Errorf("not implemented")
}

// StartContainer starts the container.
func (u *UnikernelRuntime) StartContainer(rawContainerID string) error {
	return fmt.Errorf("not implemented")
}

// StopContainer stops a running container with a grace period (i.e. timeout).
func (u *UnikernelRuntime) StopContainer(rawContainerID string, timeout int64) error {
	return fmt.Errorf("not implemented")
}

// RemoveContainer removes the container. If the container is running, the container
// should be force removed.
func (u *UnikernelRuntime) RemoveContainer(rawContainerID string) error {
	return fmt.Errorf("not implemented")
}

// ListContainers lists all containers by filters.
func (u *UnikernelRuntime) ListContainers(filter *kubeapi.ContainerFilter) ([]*kubeapi.Container, error) {
	return nil, fmt.Errorf("not implemented")
}

// ContainerStatus returns the container status.
func (u *UnikernelRuntime) ContainerStatus(containerID string) (*kubeapi.ContainerStatus, error) {
	return nil, fmt.Errorf("not implemented")
}
<commit_msg>implement container lifecycle management<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package service

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/docker/docker/pkg/stringid"
	"github.com/golang/glog"
	"k8s.io/frakti/pkg/unikernel/libvirt"
	"k8s.io/frakti/pkg/unikernel/metadata"
	"k8s.io/frakti/pkg/unikernel/metadata/store"

	kubeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
)

// CreateContainer creates a new container in specified PodSandbox
func (u *UnikernelRuntime) CreateContainer(podSandboxID string, config *kubeapi.ContainerConfig, sandboxConfig *kubeapi.PodSandboxConfig) (string, error) {
	var err error
	// Check if there is any pod already created in pod
	// We now only support one-container-per-pod
	createdContainers, err := u.getAllContainersInPod(podSandboxID)
	if err != nil {
		return "", fmt.Errorf("get containers for pod(%q) failed: %v", podSandboxID, err)
	}
	for _, container := range createdContainers {
		glog.Warningf("Unikernel/CreateContainer: container(%q) already exist in pod(%q), remove it", container.ID, podSandboxID)
		if err := u.RemoveContainer(container.ID); err != nil {
			glog.Errorf("Clean up legacy container(%q) failed: %v", container.ID, err)
		}
	}

	sandbox, err := u.sandboxStore.Get(podSandboxID)
	if err != nil {
		return "", fmt.Errorf("failed to get sandbox(%q) from store: %v", podSandboxID, err)
	}

	cid := generateID()
	cName := makeContainerName(config.GetMetadata(), sandboxConfig.GetMetadata())
	if err = u.containerNameIndex.Reserve(cName, cid); err != nil {
		return "", fmt.Errorf("reserve container name %q failed: %v", cName, err)
	}
	defer func() {
		if err != nil {
			u.containerNameIndex.ReleaseByName(cName)
		}
	}()

	// Create internal container metadata
	meta := metadata.ContainerMetadata{
		ID:        cid,
		Name:      cName,
		SandboxID: podSandboxID,
		Config:    config,
	}

	// TODO(Crazykev): Prepare container image
	// use a temp image for test
	imageRef, err := u.prepareTestImage()
	if err != nil {
		return "", fmt.Errorf("prepare image failed: %v", err)
	}
	meta.ImageRef = imageRef

	// Create container in VM, for now we actually create VM
	err = u.vmTool.CreateContainer(&meta, sandbox)
	if err != nil {
		return "", fmt.Errorf("failed to create containerd container: %v", err)
	}
	defer func() {
		// Note: For now, we just remove VM directly if create container(VM) failed.
		if err != nil {
			if err1 := u.vmTool.RemoveVM(podSandboxID); err1 != nil {
				glog.Errorf("Failed to clean up failed VM container: %v", err1)
			}
		}
	}()

	meta.CreatedAt = time.Now().UnixNano()
	if err = u.containerStore.Create(meta); err != nil {
		return "", fmt.Errorf("failed to add container metadata %+v into meta store: %v", meta, err)
	}

	return cid, nil
}

// StartContainer starts the container.
func (u *UnikernelRuntime) StartContainer(rawContainerID string) error {
	container, err := u.containerStore.Get(rawContainerID)
	if err != nil {
		return fmt.Errorf("failed to find container with ID(%q): %v", rawContainerID, err)
	}

	// Start related vm
	err = u.vmTool.StartVM(container.SandboxID)
	if err != nil {
		return fmt.Errorf("failed to start VM(%q): %v", container.SandboxID, err)
	}

	// Update container state
	err = u.containerStore.Update(container.ID, func(meta metadata.ContainerMetadata) (metadata.ContainerMetadata, error) {
		meta.StartedAt = time.Now().UnixNano()
		return meta, nil
	})
	if err != nil {
		return fmt.Errorf("failed to update conatiner(%q) metadata: %v", container.ID, err)
	}
	return nil
}

// StopContainer stops a running container with a grace period (i.e. timeout).
func (u *UnikernelRuntime) StopContainer(rawContainerID string, timeout int64) error {
	container, err := u.containerStore.Get(rawContainerID)
	if err != nil {
		return fmt.Errorf("failed to find container with ID(%q): %v", rawContainerID, err)
	}
	// Stop related VM
	err = u.vmTool.StopVM(container.SandboxID, timeout)
	if err != nil {
		return fmt.Errorf("failed to stop container related vm(%q): %v", container.SandboxID, err)
	}
	// Update container state
	err = u.containerStore.Update(container.ID, func(meta metadata.ContainerMetadata) (metadata.ContainerMetadata, error) {
		meta.FinishedAt = time.Now().UnixNano()
		return meta, nil
	})
	if err != nil {
		return fmt.Errorf("failed to update conatiner(%q) metadata: %v", container.ID, err)
	}
	return nil
}

// RemoveContainer removes the container. If the container is running, the container
// should be force removed.
func (u *UnikernelRuntime) RemoveContainer(rawContainerID string) error {
	container, err := u.containerStore.Get(rawContainerID)
	if err != nil {
		if err == store.ErrNotExist {
			return nil
		}
		return fmt.Errorf("failed to find container with ID(%q): %v", rawContainerID, err)
	}
	// Remove related VM
	err = u.vmTool.RemoveVM(container.SandboxID)
	if err != nil {
		return fmt.Errorf("failed to remove container related vm(%q): %v", container.SandboxID, err)
	}
	// Remove container metadata
	if err = u.containerStore.Delete(container.ID); err != nil {
		return fmt.Errorf("failed to delete conatiner(%q) metadata: %v", container.ID, err)
	}
	return nil
}

// ListContainers lists all containers by filters.
func (u *UnikernelRuntime) ListContainers(filter *kubeapi.ContainerFilter) ([]*kubeapi.Container, error) {
	glog.V(5).Infof("Unikernel: ListContainers with filter %+v", filter)
	// Get containers in store
	ctrInStore, err := u.containerStore.List()
	if err != nil {
		return nil, fmt.Errorf("failed to list container in store: %v", err)
	}
	// Get VMs managed by libvirt
	vmMap, err := u.vmTool.ListVMs()
	if err != nil {
		return nil, err
	}
	// Update container state from libvirt
	var containers []*kubeapi.Container
	for _, container := range ctrInStore {
		vm, exist := vmMap[container.SandboxID]
		if !exist {
			glog.Warningf("Find sandbox(%q) and container(%q) does not have related VM", container.SandboxID, container.ID)
			// FIXME(Crazykev): Should we remove these container and sandbox?
			continue
		}
		curState := getCRIContainerStateFromVMState(vm.State, container.State())
		containers = append(containers, toCRIContainer(container, curState))
	}

	// Filter containers
	containers = u.filterCRIContainers(containers, filter)
	return containers, nil
}

// ContainerStatus returns the container status.
func (u *UnikernelRuntime) ContainerStatus(rawContainerID string) (*kubeapi.ContainerStatus, error) {
	container, err := u.containerStore.Get(rawContainerID)
	if err != nil {
		return nil, fmt.Errorf("failed to find container with ID(%q): %v", rawContainerID, err)
	}
	// Get related VM info
	vmInfo, err := u.vmTool.GetVMInfo(container.SandboxID)
	if err != nil {
		return nil, err
	}
	lastState := container.State()
	curState := getCRIContainerStateFromVMState(vmInfo.State, lastState)
	if err = u.transformContainerState(container, curState, lastState); err != nil {
		return nil, err
	}

	return &kubeapi.ContainerStatus{
		Id:          container.ID,
		Metadata:    container.Config.GetMetadata(),
		State:       curState,
		CreatedAt:   container.CreatedAt,
		StartedAt:   container.StartedAt,
		FinishedAt:  container.FinishedAt,
		Image:       container.Config.GetImage(),
		ImageRef:    container.ImageRef,
		Labels:      container.Config.GetLabels(),
		Annotations: container.Config.GetAnnotations(),
	}, nil
}

// transformContainerState update container state according to container state transformation.
func (u *UnikernelRuntime) transformContainerState(meta *metadata.ContainerMetadata, curState, lastState kubeapi.ContainerState) error {
	if curState == lastState {
		return nil
	}
	if curState != kubeapi.ContainerState_CONTAINER_EXITED {
		glog.Warningf("Unexpected container(%s) state transform from %d to %d", meta.ID, lastState, curState)
	}
	err := u.containerStore.Update(meta.ID, func(meta metadata.ContainerMetadata) (metadata.ContainerMetadata, error) {
		meta.FinishedAt = time.Now().UnixNano()
		return meta, nil
	})
	if err != nil {
		return fmt.Errorf("failed to update container(%s) state: %v", meta.ID, err)
	}
	return nil
}

// getCRIContainerStateFromVMState get CRI container state from last container state and current vm state
func getCRIContainerStateFromVMState(vmState libvirt.DomainState, lastState kubeapi.ContainerState) kubeapi.ContainerState {
	switch vmState {
	case libvirt.DOMAIN_SHUTDOWN:
		// shutdown means between running and shutoff, that is still running
		fallthrough
	case libvirt.DOMAIN_RUNNING:
		return kubeapi.ContainerState_CONTAINER_RUNNING
	case libvirt.DOMAIN_SHUTOFF:
		if lastState == kubeapi.ContainerState_CONTAINER_CREATED {
			return kubeapi.ContainerState_CONTAINER_CREATED
		}
		return kubeapi.ContainerState_CONTAINER_EXITED
	case libvirt.DOMAIN_CRASHED:
		return kubeapi.ContainerState_CONTAINER_EXITED
	case libvirt.DOMAIN_PMSUSPENDED:
		return kubeapi.ContainerState_CONTAINER_EXITED
	case libvirt.DOMAIN_PAUSED:
		if lastState == kubeapi.ContainerState_CONTAINER_CREATED {
			return kubeapi.ContainerState_CONTAINER_CREATED
		}
		return kubeapi.ContainerState_CONTAINER_EXITED
	default:
		return kubeapi.ContainerState_CONTAINER_UNKNOWN
	}
}

// toCRIContainer get CRI defined container from ContainerMetadata
func toCRIContainer(meta *metadata.ContainerMetadata, state kubeapi.ContainerState) *kubeapi.Container {
	return &kubeapi.Container{
		Id:           meta.ID,
		PodSandboxId: meta.SandboxID,
		Metadata:     meta.Config.GetMetadata(),
		Image:        meta.Config.GetImage(),
		ImageRef:     meta.ImageRef,
		State:        state,
		CreatedAt:    meta.CreatedAt,
		Labels:       meta.Config.GetLabels(),
		Annotations:  meta.Config.GetAnnotations(),
	}
}

func (u *UnikernelRuntime) filterCRIContainers(containers []*kubeapi.Container, filter *kubeapi.ContainerFilter) []*kubeapi.Container {
	if filter == nil {
		return containers
	}

	result := []*kubeapi.Container{}
	for _, ctr := range containers {
		if filter.GetId() != "" && filter.GetId() != ctr.Id {
			continue
		}
		if filter.GetPodSandboxId() != "" && filter.GetPodSandboxId() != ctr.PodSandboxId {
			continue
		}
		if filter.GetState() != nil && filter.GetState().GetState() != ctr.State {
			continue
		}
		if filter.GetLabelSelector() != nil {
			match := true
			for k, v := range filter.GetLabelSelector() {
				value, exist := ctr.Labels[k]
				if !exist || value != v {
					match = false
					break
				}
			}
			if !match {
				continue
			}
		}
		result = append(result, ctr)
	}

	return result
}

func (u *UnikernelRuntime) getAllContainersInPod(podID string) ([]*metadata.ContainerMetadata, error) {
	containersInStore, err := u.containerStore.List()
	if err != nil {
		return nil, err
	}
	var containers []*metadata.ContainerMetadata
	for _, container := range containersInStore {
		if container.SandboxID == podID {
			containers = append(containers, container)
		}
	}
	return containers, nil
}

// generateID generates a random unique id.
func generateID() string {
	return stringid.GenerateNonCryptoID()
}

func makeContainerName(c *kubeapi.ContainerMetadata, s *kubeapi.PodSandboxMetadata) string {
	return strings.Join([]string{
		c.Name,
		s.Name,
		s.Namespace,
		s.Uid,
		fmt.Sprintf("%d", c.Attempt),
	}, "_")
}

// TODO(Crazykev): Remove this when we have image management
func (u *UnikernelRuntime) prepareTestImage() (string, error) {
	imagePath := filepath.Join(u.rootDir, "image", "test-image.qcow2")
	if _, err := os.Stat(imagePath); err != nil {
		return "", err
	}
	return imagePath, nil
}
<|endoftext|>