<commit_before>// Copyright ©2015 The corridor Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.package main

package corridor

import (
	"math"
	"sort"

	"github.com/gonum/matrix/mat64"
)

// compute euclidean distance for a pair of subscript indices
func Distance(aSubs, bSubs []int) (dist float64) {

	// initialize variables
	var x0 float64 = float64(aSubs[0])
	var x1 float64 = float64(bSubs[0])
	var y0 float64 = float64(aSubs[1])
	var y1 float64 = float64(bSubs[1])
	var pow float64 = 2.0
	var dx float64 = x1 - x0
	var dy float64 = y1 - y0

	// compute distance
	var output float64 = math.Sqrt(math.Pow(dx, pow) + math.Pow(dy, pow))

	// return final output
	return output
}

func MinDistance(aSubs []int, lineSubs [][]int) (minDist float64) {

	// initialize variables
	maxLen := len(lineSubs)
	distVec := make([]float64, maxLen)

	// loop through and compute distances
	for i := 0; i < maxLen; i++ {
		distVec[i] = Distance(aSubs, lineSubs[i])
	}

	// sort distances
	sort.Float64s(distVec)

	// get final output
	output := distVec[0]

	// return final output
	return output
}

// bresenham generates the list of subscript indices corresponding to the
// euclidean shortest paths connecting two subscript pairs in discrete space
func Bresenham(aSubs, bSubs []int) (lineSubs [][]int) {

	// initialize variables
	var x0 int = aSubs[0]
	var x1 int = bSubs[0]
	var y0 int = aSubs[1]
	var y1 int = bSubs[1]

	// check row differential
	dx := x1 - x0
	if dx < 0 {
		dx = -dx
	}

	// check column differential
	dy := y1 - y0

	if dy < 0 {
		dy = -dy
	}

	// initialize stride variables
	var sx, sy int

	// set row stride direction
	if x0 < x1 {
		sx = 1
	} else {
		sx = -1
	}

	// set column stride direction
	if y0 < y1 {
		sy = 1
	} else {
		sy = -1
	}

	// calculate error component
	err := dx - dy

	// initialize output 2D slice vector
	dist := math.Ceil(Distance(aSubs, bSubs))
	maxLen := int(dist)
	output := make([][]int, 0, maxLen)

	// loop through and generate subscripts
	for {
		var val = []int{x0, y0}
		output = append(output, val)
		if x0 == x1 && y0 == y1 {
			break
		}
		e2 := 2 * err
		if e2 > -dy {
			err -= dy
			x0 += sx
		}
		if e2 < dx {
			err += dx
			y0 += sy
		}
	}

	// return final output
	return output
}

// fitness function to generate the total fitness and individual
// fitness values for a given input set of subscripts
// corresponding to a single individual
func Fitness(subs [][]int, obj *mat64.Dense) (fitnessValues []float64, totalFitness float64) {

	// get individual length
	indSize := len(subs)

	// initialize fitness values and total fitness
	fitVal := make([]float64, indSize)
	var totFit float64 = 0.0

	// evaluate individual fitness according to input objective
	for i := 0; i < indSize; i++ {
		curFit := obj.At(subs[i][0], subs[i][1])
		fitVal[i] = curFit
		totFit = totFit + curFit
	}

	// return outputs
	return fitVal, totFit

}
<commit_msg>Implemented a much more efficient mindistance algorithm that does not require the generation of the euclidean solution subscripts.<commit_after>// Copyright ©2015 The corridor Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.package main

package corridor

import (
	"math"

	"github.com/gonum/matrix/mat64"
)

// compute euclidean distance for a pair of subscript indices
func Distance(aSubs, bSubs []int) (dist float64) {

	// initialize variables
	var x0 float64 = float64(aSubs[0])
	var x1 float64 = float64(bSubs[0])
	var y0 float64 = float64(aSubs[1])
	var y1 float64 = float64(bSubs[1])
	var pow float64 = 2.0
	var dx float64 = x1 - x0
	var dy float64 = y1 - y0

	// compute distance
	var output float64 = math.Sqrt(math.Pow(dx, pow) + math.Pow(dy, pow))

	// return final output
	return output
}

func MinDistance(pSubs, aSubs, bSubs []int) (minDist float64) {

	var x float64 = float64(pSubs[0])
	var y float64 = float64(pSubs[0])
	var x0 float64 = float64(aSubs[0])
	var x1 float64 = float64(bSubs[0])
	var y0 float64 = float64(aSubs[1])
	var y1 float64 = float64(bSubs[1])

	a := x - x0
	b := y - y0
	c := x1 - x0
	d := y1 - y0

	dot := a*c + b*d
	lenSq := c*c + d*d

	var param float64 = -1.0
	var xx, yy float64

	if lenSq != 0 {
		param = dot / lenSq
	}

	if param < 0 {
		xx = x0
		yy = y0
	} else if param > 1 {
		xx = x1
		yy = y1
	} else {
		xx = x1 + param*c
		yy = y1 + param*d
	}

	var dx float64 = x - xx
	var dy float64 = y - yy

	output := math.Sqrt(dx*dx + dy*dy)
	// return final output
	return output
}

// bresenham generates the list of subscript indices corresponding to the
// euclidean shortest paths connecting two subscript pairs in discrete space
func Bresenham(aSubs, bSubs []int) (lineSubs [][]int) {

	// initialize variables
	var x0 int = aSubs[0]
	var x1 int = bSubs[0]
	var y0 int = aSubs[1]
	var y1 int = bSubs[1]

	// check row differential
	dx := x1 - x0
	if dx < 0 {
		dx = -dx
	}

	// check column differential
	dy := y1 - y0

	if dy < 0 {
		dy = -dy
	}

	// initialize stride variables
	var sx, sy int

	// set row stride direction
	if x0 < x1 {
		sx = 1
	} else {
		sx = -1
	}

	// set column stride direction
	if y0 < y1 {
		sy = 1
	} else {
		sy = -1
	}

	// calculate error component
	err := dx - dy

	// initialize output 2D slice vector
	dist := math.Ceil(Distance(aSubs, bSubs))
	maxLen := int(dist)
	output := make([][]int, 0, maxLen)

	// loop through and generate subscripts
	for {
		var val = []int{x0, y0}
		output = append(output, val)
		if x0 == x1 && y0 == y1 {
			break
		}
		e2 := 2 * err
		if e2 > -dy {
			err -= dy
			x0 += sx
		}
		if e2 < dx {
			err += dx
			y0 += sy
		}
	}

	// return final output
	return output
}

// fitness function to generate the total fitness and individual
// fitness values for a given input set of subscripts
// corresponding to a single individual
func Fitness(subs [][]int, obj *mat64.Dense) (fitnessValues []float64, totalFitness float64) {

	// get individual length
	indSize := len(subs)

	// initialize fitness values and total fitness
	fitVal := make([]float64, indSize)
	var totFit float64 = 0.0

	// evaluate individual fitness according to input objective
	for i := 0; i < indSize; i++ {
		curFit := obj.At(subs[i][0], subs[i][1])
		fitVal[i] = curFit
		totFit = totFit + curFit
	}

	// return outputs
	return fitVal, totFit

}
<|endoftext|>


<commit_before>package bench

import (
	"time"
)

type Stats struct {
	Min   time.Duration
	Max   time.Duration
	Total time.Duration
	Num   int64
}

func NewStats() *Stats {
	return &Stats{
		Min: 1<<63 - 1,
	}
}

func (s *Stats) Add(td time.Duration) {
	s.Num += 1
	s.Total += td
	if td < s.Min {
		s.Min = td
	}
	if td > s.Max {
		s.Max = td
	}
}

func (s *Stats) Avg() time.Duration {
	return s.Total / time.Duration(s.Num)
}

func AddToResults(s *Stats, results map[string]interface{}) {
	results["min"] = s.Min
	results["max"] = s.Max
	results["avg"] = s.Avg()
}
<commit_msg>Add standard deviation to stats<commit_after>package bench

import (
	"math"
	"time"
)

type Stats struct {
	Min            time.Duration
	Max            time.Duration
	Mean           time.Duration
	SumSquareDelta float64
	Variance       float64
	StdDev         time.Duration
	Total          time.Duration
	Num            int64
}

func NewStats() *Stats {
	return &Stats{
		Min: 1<<63 - 1,
	}
}

func (s *Stats) Add(td time.Duration) {
	s.Num += 1
	s.Total += td
	if td < s.Min {
		s.Min = td
	}
	if td > s.Max {
		s.Max = td
	}

	// these three comprise an online variance calculation
	// https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm
	delta := td - s.Mean
	s.Mean += delta / time.Duration(s.Num)
	s.SumSquareDelta += float64(delta * (td - s.Mean))

	// these are the useful results, but don't need to be updated every iteration
	s.Variance = s.SumSquareDelta / float64(s.Num)
	s.StdDev = time.Duration(math.Sqrt(s.Variance))
}

func (s *Stats) Avg() time.Duration {
	return s.Total / time.Duration(s.Num)
}

func AddToResults(s *Stats, results map[string]interface{}) {
	results["min"] = s.Min
	results["max"] = s.Max
	results["avg"] = s.Avg()
}
<|endoftext|>


<commit_before>// Package auth contains types and functions to manage authentication
// credentials for service hosts.
package auth

import (
	"net/http"

	"github.com/hashicorp/terraform/svchost"
)

// Credentials is a list of CredentialsSource objects that can be tried in
// turn until one returns credentials for a host, or one returns an error.
//
// A Credentials is itself a CredentialsSource, wrapping its members.
// In principle one CredentialsSource can be nested inside another, though
// there is no good reason to do so.
type Credentials []CredentialsSource

// A CredentialsSource is an object that may be able to provide credentials
// for a given host.
//
// Credentials lookups are not guaranteed to be concurrency-safe. Callers
// using these facilities in concurrent code must use external concurrency
// primitives to prevent race conditions.
type CredentialsSource interface {
	// ForHost returns a non-nil HostCredentials if the source has credentials
	// available for the host, and a nil HostCredentials if it does not.
	//
	// If an error is returned, progress through a list of CredentialsSources
	// is halted and the error is returned to the user.
	ForHost(host svchost.Hostname) (HostCredentials, error)
}

// HostCredentials represents a single set of credentials for a particular
// host.
type HostCredentials interface {
	// PrepareRequest modifies the given request in-place to apply the
	// receiving credentials. The usual behavior of this method is to
	// add some sort of Authorization header to the request.
	PrepareRequest(req *http.Request)
}

// ForHost iterates over the contained CredentialsSource objects and
// tries to obtain credentials for the given host from each one in turn.
//
// If any source returns either a non-nil HostCredentials or a non-nil error
// then this result is returned. Otherwise, the result is nil, nil.
func (c Credentials) ForHost(host svchost.Hostname) (HostCredentials, error) {
	for _, source := range c {
		creds, err := source.ForHost(host)
		if creds != nil || err != nil {
			return creds, err
		}
	}
	return nil, nil
}
<commit_msg>svchost/auth: expose a "NoCredentials" credentials source<commit_after>// Package auth contains types and functions to manage authentication
// credentials for service hosts.
package auth

import (
	"net/http"

	"github.com/hashicorp/terraform/svchost"
)

// Credentials is a list of CredentialsSource objects that can be tried in
// turn until one returns credentials for a host, or one returns an error.
//
// A Credentials is itself a CredentialsSource, wrapping its members.
// In principle one CredentialsSource can be nested inside another, though
// there is no good reason to do so.
type Credentials []CredentialsSource

// NoCredentials is an empty CredentialsSource that always returns nil
// when asked for credentials.
var NoCredentials CredentialsSource = Credentials{}

// A CredentialsSource is an object that may be able to provide credentials
// for a given host.
//
// Credentials lookups are not guaranteed to be concurrency-safe. Callers
// using these facilities in concurrent code must use external concurrency
// primitives to prevent race conditions.
type CredentialsSource interface {
	// ForHost returns a non-nil HostCredentials if the source has credentials
	// available for the host, and a nil HostCredentials if it does not.
	//
	// If an error is returned, progress through a list of CredentialsSources
	// is halted and the error is returned to the user.
	ForHost(host svchost.Hostname) (HostCredentials, error)
}

// HostCredentials represents a single set of credentials for a particular
// host.
type HostCredentials interface {
	// PrepareRequest modifies the given request in-place to apply the
	// receiving credentials. The usual behavior of this method is to
	// add some sort of Authorization header to the request.
	PrepareRequest(req *http.Request)
}

// ForHost iterates over the contained CredentialsSource objects and
// tries to obtain credentials for the given host from each one in turn.
//
// If any source returns either a non-nil HostCredentials or a non-nil error
// then this result is returned. Otherwise, the result is nil, nil.
func (c Credentials) ForHost(host svchost.Hostname) (HostCredentials, error) {
	for _, source := range c {
		creds, err := source.ForHost(host)
		if creds != nil || err != nil {
			return creds, err
		}
	}
	return nil, nil
}
<|endoftext|>


<commit_before>package getter

import (
	"net/http"
	"strings"
	"testing"
)

const testBBUrl = "https://bitbucket.org/hashicorp/tf-test-git"

func TestBitBucketDetector(t *testing.T) {
	t.Parallel()

	if _, err := http.Get(testBBUrl); err != nil {
		t.Log("internet may not be working, skipping BB tests")
		t.Skip()
	}

	cases := []struct {
		Input  string
		Output string
	}{
		// HTTP
		{
			"bitbucket.org/hashicorp/tf-test-git",
			"git::https://bitbucket.org/hashicorp/tf-test-git.git",
		},
		{
			"bitbucket.org/hashicorp/tf-test-git.git",
			"git::https://bitbucket.org/hashicorp/tf-test-git.git",
		},
		{
			"bitbucket.org/hashicorp/tf-test-hg",
			"hg::https://bitbucket.org/hashicorp/tf-test-hg",
		},
	}

	pwd := "/pwd"
	f := new(BitBucketDetector)
	for i, tc := range cases {
		var err error
		for i := 0; i < 3; i++ {
			var output string
			var ok bool
			output, ok, err = f.Detect(tc.Input, pwd)
			if err != nil {
				if strings.Contains(err.Error(), "invalid character") {
					continue
				}

				t.Fatalf("err: %s", err)
			}
			if !ok {
				t.Fatal("not ok")
			}

			if output != tc.Output {
				t.Fatalf("%d: bad: %#v", i, output)
			}

			break
		}
		if i >= 3 {
			t.Fatalf("failure from bitbucket: %s", err)
		}
	}
}
<commit_msg>Remove broken bitbucket link (#276)<commit_after>package getter

import (
	"net/http"
	"strings"
	"testing"
)

const testBBUrl = "https://bitbucket.org/hashicorp/tf-test-git"

func TestBitBucketDetector(t *testing.T) {
	t.Parallel()

	if _, err := http.Get(testBBUrl); err != nil {
		t.Log("internet may not be working, skipping BB tests")
		t.Skip()
	}

	cases := []struct {
		Input  string
		Output string
	}{
		// HTTP
		{
			"bitbucket.org/hashicorp/tf-test-git",
			"git::https://bitbucket.org/hashicorp/tf-test-git.git",
		},
		{
			"bitbucket.org/hashicorp/tf-test-git.git",
			"git::https://bitbucket.org/hashicorp/tf-test-git.git",
		},
	}

	pwd := "/pwd"
	f := new(BitBucketDetector)
	for i, tc := range cases {
		var err error
		for i := 0; i < 3; i++ {
			var output string
			var ok bool
			output, ok, err = f.Detect(tc.Input, pwd)
			if err != nil {
				if strings.Contains(err.Error(), "invalid character") {
					continue
				}

				t.Fatalf("err: %s", err)
			}
			if !ok {
				t.Fatal("not ok")
			}

			if output != tc.Output {
				t.Fatalf("%d: bad: %#v", i, output)
			}

			break
		}
		if i >= 3 {
			t.Fatalf("failure from bitbucket: %s", err)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (C) 2020 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mali

import (
	"context"
	"fmt"

	"github.com/google/gapid/core/log"
	"github.com/google/gapid/core/os/device"
	"github.com/google/gapid/gapis/api"
	"github.com/google/gapid/gapis/api/sync"
	"github.com/google/gapid/gapis/perfetto"
	perfetto_service "github.com/google/gapid/gapis/perfetto/service"
	"github.com/google/gapid/gapis/service"
	"github.com/google/gapid/gapis/service/path"
	"github.com/google/gapid/gapis/trace/android/profile"
	"github.com/google/gapid/gapis/trace/android/utils"
)

var (
	slicesQuery = "" +
		"SELECT s.context_id, s.render_target, s.frame_id, s.submission_id, s.hw_queue_id, s.command_buffer, s.render_pass, s.ts, s.dur, s.id, s.name, depth, arg_set_id, track_id, t.name " +
		"FROM gpu_track t LEFT JOIN gpu_slice s " +
		"ON s.track_id = t.id WHERE t.scope = 'gpu_render_stage' ORDER BY s.ts"
	argsQueryFmt = "" +
		"SELECT key, string_value FROM args WHERE args.arg_set_id = %d"
	queueSubmitQuery = "" +
		"SELECT submission_id, command_buffer FROM gpu_slice s JOIN track t ON s.track_id = t.id WHERE s.name = 'vkQueueSubmit' AND t.name = 'Vulkan Events' ORDER BY submission_id"
	counterTracksQuery = "" +
		"SELECT id, name, unit, description FROM gpu_counter_track ORDER BY id"
	countersQueryFmt = "" +
		"SELECT ts, value FROM counter c WHERE c.track_id = %d ORDER BY ts"
)

func ProcessProfilingData(ctx context.Context, processor *perfetto.Processor, capture *path.Capture, desc *device.GpuCounterDescriptor, handleMapping *map[uint64][]service.VulkanHandleMappingItem, syncData *sync.Data) (*service.ProfilingData, error) {
	slices, err := processGpuSlices(ctx, processor, capture, handleMapping, syncData)
	if err != nil {
		log.Err(ctx, err, "Failed to get GPU slices")
	}
	counters, err := processCounters(ctx, processor, desc)
	if err != nil {
		log.Err(ctx, err, "Failed to get GPU counters")
	}
	gpuCounters, err := profile.ComputeCounters(ctx, slices, counters)
	if err != nil {
		log.Err(ctx, err, "Failed to calculate performance data based on GPU slices and counters")
	}

	return &service.ProfilingData{
		Slices:      slices,
		Counters:    counters,
		GpuCounters: gpuCounters,
	}, nil
}

func extractTraceHandles(ctx context.Context, replayHandles *[]int64, replayHandleType string, handleMapping *map[uint64][]service.VulkanHandleMappingItem) {
	for i, v := range *replayHandles {
		handles, ok := (*handleMapping)[uint64(v)]
		if !ok {
			log.E(ctx, "%v not found in replay: %v", replayHandleType, v)
			continue
		}

		found := false
		for _, handle := range handles {
			if handle.HandleType == replayHandleType {
				(*replayHandles)[i] = int64(handle.TraceValue)
				found = true
				break
			}
		}

		if !found {
			log.E(ctx, "Incorrect Handle type for %v: %v", replayHandleType, v)
		}
	}
}

func processGpuSlices(ctx context.Context, processor *perfetto.Processor, capture *path.Capture, handleMapping *map[uint64][]service.VulkanHandleMappingItem, syncData *sync.Data) (*service.ProfilingData_GpuSlices, error) {
	slicesQueryResult, err := processor.Query(slicesQuery)
	if err != nil {
		return nil, log.Errf(ctx, err, "SQL query failed: %v", slicesQuery)
	}

	queueSubmitQueryResult, err := processor.Query(queueSubmitQuery)
	if err != nil {
		return nil, log.Errf(ctx, err, "SQL query failed: %v", queueSubmitQuery)
	}
	queueSubmitColumns := queueSubmitQueryResult.GetColumns()
	queueSubmitIds := queueSubmitColumns[0].GetLongValues()
	queueSubmitCommandBuffers := queueSubmitColumns[1].GetLongValues()
	submissionOrdering := make(map[int64]uint64)

	order := 0
	for i, v := range queueSubmitIds {
		if queueSubmitCommandBuffers[i] == 0 {
			// This is a spurious submission. See b/150854367
			log.W(ctx, "Spurious vkQueueSubmit slice with submission id %v", v)
			continue
		}
		submissionOrdering[v] = uint64(order)
		order++
	}

	trackIdCache := make(map[int64]bool)
	argsQueryCache := make(map[int64]*perfetto_service.QueryResult)
	slicesColumns := slicesQueryResult.GetColumns()
	numSliceRows := slicesQueryResult.GetNumRecords()
	slices := make([]*service.ProfilingData_GpuSlices_Slice, numSliceRows)
	groupsMap := map[api.CmdSubmissionKey]*service.ProfilingData_GpuSlices_Group{}
	groupIds := make([]int32, numSliceRows)
	var tracks []*service.ProfilingData_GpuSlices_Track
	// Grab all the column values. Depends on the order of columns selected in slicesQuery

	contextIds := slicesColumns[0].GetLongValues()
	extractTraceHandles(ctx, &contextIds, "VkDevice", handleMapping)

	renderTargets := slicesColumns[1].GetLongValues()
	extractTraceHandles(ctx, &renderTargets, "VkFramebuffer", handleMapping)

	commandBuffers := slicesColumns[5].GetLongValues()
	extractTraceHandles(ctx, &commandBuffers, "VkCommandBuffer", handleMapping)

	renderPasses := slicesColumns[6].GetLongValues()
	extractTraceHandles(ctx, &renderPasses, "VkRenderPass", handleMapping)

	frameIds := slicesColumns[2].GetLongValues()
	submissionIds := slicesColumns[3].GetLongValues()
	hwQueueIds := slicesColumns[4].GetLongValues()
	timestamps := slicesColumns[7].GetLongValues()
	durations := slicesColumns[8].GetLongValues()
	ids := slicesColumns[9].GetLongValues()
	names := slicesColumns[10].GetStringValues()
	depths := slicesColumns[11].GetLongValues()
	argSetIds := slicesColumns[12].GetLongValues()
	trackIds := slicesColumns[13].GetLongValues()
	trackNames := slicesColumns[14].GetStringValues()

	for i, v := range submissionIds {
		subOrder, ok := submissionOrdering[v]
		groupId := int32(-1)
		if ok {
			cb := uint64(commandBuffers[i])
			key := api.CmdSubmissionKey{subOrder, cb, uint64(renderPasses[i]), uint64(renderTargets[i])}
			if names[i] == "vertex" || names[i] == "fragment" {
				if group, ok := groupsMap[key]; ok {
					groupId = group.Id
				} else if indices, ok := syncData.SubmissionIndices[key]; ok {
					parent := utils.FindParentGroup(ctx, subOrder, cb, groupsMap, syncData.SubmissionIndices, capture)
					groupId = int32(len(groupsMap))
					group := &service.ProfilingData_GpuSlices_Group{
						Id:     groupId,
						Name:   fmt.Sprintf("RenderPass %v, RenderTarget %v", uint64(renderPasses[i]), uint64(renderTargets[i])),
						Parent: parent,
						Link:   &path.Command{Capture: capture, Indices: indices[0]},
					}
					groupsMap[key] = group
				}
				names[i] = fmt.Sprintf("%v %v", groupsMap[key].Link.Indices, names[i])
			}
		} else {
			log.W(ctx, "Encountered submission ID mismatch %v", v)
		}

		groupIds[i] = groupId
	}
	groups := []*service.ProfilingData_GpuSlices_Group{}
	for _, group := range groupsMap {
		groups = append(groups, group)
	}

	for i := uint64(0); i < numSliceRows; i++ {
		var argsQueryResult *perfetto_service.QueryResult
		var ok bool
		if argsQueryResult, ok = argsQueryCache[argSetIds[i]]; !ok {
			argsQuery := fmt.Sprintf(argsQueryFmt, argSetIds[i])
			argsQueryResult, err = processor.Query(argsQuery)
			if err != nil {
				log.W(ctx, "SQL query failed: %v", argsQuery)
			}
			argsQueryCache[argSetIds[i]] = argsQueryResult
		}
		argsColumns := argsQueryResult.GetColumns()
		numArgsRows := argsQueryResult.GetNumRecords()
		var extras []*service.ProfilingData_GpuSlices_Slice_Extra
		for j := uint64(0); j < numArgsRows; j++ {
			keys := argsColumns[0].GetStringValues()
			values := argsColumns[1].GetStringValues()
			extras = append(extras, &service.ProfilingData_GpuSlices_Slice_Extra{
				Name:  keys[j],
				Value: &service.ProfilingData_GpuSlices_Slice_Extra_StringValue{StringValue: values[j]},
			})
		}
		extras = append(extras, &service.ProfilingData_GpuSlices_Slice_Extra{
			Name:  "contextId",
			Value: &service.ProfilingData_GpuSlices_Slice_Extra_IntValue{IntValue: uint64(contextIds[i])},
		})
		extras = append(extras, &service.ProfilingData_GpuSlices_Slice_Extra{
			Name:  "renderTarget",
			Value: &service.ProfilingData_GpuSlices_Slice_Extra_IntValue{IntValue: uint64(renderTargets[i])},
		})
		extras = append(extras, &service.ProfilingData_GpuSlices_Slice_Extra{
			Name:  "commandBuffer",
			Value: &service.ProfilingData_GpuSlices_Slice_Extra_IntValue{IntValue: uint64(commandBuffers[i])},
		})
		extras = append(extras, &service.ProfilingData_GpuSlices_Slice_Extra{
			Name:  "renderPass",
			Value: &service.ProfilingData_GpuSlices_Slice_Extra_IntValue{IntValue: uint64(renderPasses[i])},
		})
		extras = append(extras, &service.ProfilingData_GpuSlices_Slice_Extra{
			Name:  "frameId",
			Value: &service.ProfilingData_GpuSlices_Slice_Extra_IntValue{IntValue: uint64(frameIds[i])},
		})
		extras = append(extras, &service.ProfilingData_GpuSlices_Slice_Extra{
			Name:  "submissionId",
			Value: &service.ProfilingData_GpuSlices_Slice_Extra_IntValue{IntValue: uint64(submissionIds[i])},
		})
		extras = append(extras, &service.ProfilingData_GpuSlices_Slice_Extra{
			Name:  "hwQueueId",
			Value: &service.ProfilingData_GpuSlices_Slice_Extra_IntValue{IntValue: uint64(hwQueueIds[i])},
		})

		slices[i] = &service.ProfilingData_GpuSlices_Slice{
			Ts:      uint64(timestamps[i]),
			Dur:     uint64(durations[i]),
			Id:      uint64(ids[i]),
			Label:   names[i],
			Depth:   int32(depths[i]),
			Extras:  extras,
			TrackId: int32(trackIds[i]),
			GroupId: groupIds[i],
		}

		if _, ok := trackIdCache[trackIds[i]]; !ok {
			trackIdCache[trackIds[i]] = true
			tracks = append(tracks, &service.ProfilingData_GpuSlices_Track{
				Id:   int32(trackIds[i]),
				Name: trackNames[i],
			})
		}
	}

	return &service.ProfilingData_GpuSlices{
		Slices: slices,
		Tracks: tracks,
		Groups: groups,
	}, nil
}

func processCounters(ctx context.Context, processor *perfetto.Processor, desc *device.GpuCounterDescriptor) ([]*service.ProfilingData_Counter, error) {
	counterTracksQueryResult, err := processor.Query(counterTracksQuery)
	if err != nil {
		return nil, log.Errf(ctx, err, "SQL query failed: %v", counterTracksQuery)
	}
	// t.id, name, unit, description, ts, value
	tracksColumns := counterTracksQueryResult.GetColumns()
	numTracksRows := counterTracksQueryResult.GetNumRecords()
	counters := make([]*service.ProfilingData_Counter, numTracksRows)
	// Grab all the column values. Depends on the order of columns selected in countersQuery
	trackIds := tracksColumns[0].GetLongValues()
	names := tracksColumns[1].GetStringValues()
	units := tracksColumns[2].GetStringValues()
	descriptions := tracksColumns[3].GetStringValues()

	for i := uint64(0); i < numTracksRows; i++ {
		countersQuery := fmt.Sprintf(countersQueryFmt, trackIds[i])
		countersQueryResult, err := processor.Query(countersQuery)
		countersColumns := countersQueryResult.GetColumns()
		if err != nil {
			return nil, log.Errf(ctx, err, "SQL query failed: %v", counterTracksQuery)
		}
		timestampsLong := countersColumns[0].GetLongValues()
		timestamps := make([]uint64, len(timestampsLong))
		for i, t := range timestampsLong {
			timestamps[i] = uint64(t)
		}
		values := countersColumns[1].GetDoubleValues()
		// TODO(apbodnar) Populate the `default` field once the trace processor supports it (b/147432390)
		counters[i] = &service.ProfilingData_Counter{
			Id:          uint32(trackIds[i]),
			Name:        names[i],
			Unit:        units[i],
			Description: descriptions[i],
			Timestamps:  timestamps,
			Values:      values,
		}
	}
	return counters, nil
}
<commit_msg>Fix possible segfault in mali.processGpuSlices (wrong map access) (#625)<commit_after>// Copyright (C) 2020 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mali

import (
	"context"
	"fmt"

	"github.com/google/gapid/core/log"
	"github.com/google/gapid/core/os/device"
	"github.com/google/gapid/gapis/api"
	"github.com/google/gapid/gapis/api/sync"
	"github.com/google/gapid/gapis/perfetto"
	perfetto_service "github.com/google/gapid/gapis/perfetto/service"
	"github.com/google/gapid/gapis/service"
	"github.com/google/gapid/gapis/service/path"
	"github.com/google/gapid/gapis/trace/android/profile"
	"github.com/google/gapid/gapis/trace/android/utils"
)

var (
	slicesQuery = "" +
		"SELECT s.context_id, s.render_target, s.frame_id, s.submission_id, s.hw_queue_id, s.command_buffer, s.render_pass, s.ts, s.dur, s.id, s.name, depth, arg_set_id, track_id, t.name " +
		"FROM gpu_track t LEFT JOIN gpu_slice s " +
		"ON s.track_id = t.id WHERE t.scope = 'gpu_render_stage' ORDER BY s.ts"
	argsQueryFmt = "" +
		"SELECT key, string_value FROM args WHERE args.arg_set_id = %d"
	queueSubmitQuery = "" +
		"SELECT submission_id, command_buffer FROM gpu_slice s JOIN track t ON s.track_id = t.id WHERE s.name = 'vkQueueSubmit' AND t.name = 'Vulkan Events' ORDER BY submission_id"
	counterTracksQuery = "" +
		"SELECT id, name, unit, description FROM gpu_counter_track ORDER BY id"
	countersQueryFmt = "" +
		"SELECT ts, value FROM counter c WHERE c.track_id = %d ORDER BY ts"
)

func ProcessProfilingData(ctx context.Context, processor *perfetto.Processor, capture *path.Capture, desc *device.GpuCounterDescriptor, handleMapping *map[uint64][]service.VulkanHandleMappingItem, syncData *sync.Data) (*service.ProfilingData, error) {
	slices, err := processGpuSlices(ctx, processor, capture, handleMapping, syncData)
	if err != nil {
		log.Err(ctx, err, "Failed to get GPU slices")
	}
	counters, err := processCounters(ctx, processor, desc)
	if err != nil {
		log.Err(ctx, err, "Failed to get GPU counters")
	}
	gpuCounters, err := profile.ComputeCounters(ctx, slices, counters)
	if err != nil {
		log.Err(ctx, err, "Failed to calculate performance data based on GPU slices and counters")
	}

	return &service.ProfilingData{
		Slices:      slices,
		Counters:    counters,
		GpuCounters: gpuCounters,
	}, nil
}

func extractTraceHandles(ctx context.Context, replayHandles *[]int64, replayHandleType string, handleMapping *map[uint64][]service.VulkanHandleMappingItem) {
	for i, v := range *replayHandles {
		handles, ok := (*handleMapping)[uint64(v)]
		if !ok {
			log.E(ctx, "%v not found in replay: %v", replayHandleType, v)
			continue
		}

		found := false
		for _, handle := range handles {
			if handle.HandleType == replayHandleType {
				(*replayHandles)[i] = int64(handle.TraceValue)
				found = true
				break
			}
		}

		if !found {
			log.E(ctx, "Incorrect Handle type for %v: %v", replayHandleType, v)
		}
	}
}

func processGpuSlices(ctx context.Context, processor *perfetto.Processor, capture *path.Capture, handleMapping *map[uint64][]service.VulkanHandleMappingItem, syncData *sync.Data) (*service.ProfilingData_GpuSlices, error) {
	slicesQueryResult, err := processor.Query(slicesQuery)
	if err != nil {
		return nil, log.Errf(ctx, err, "SQL query failed: %v", slicesQuery)
	}

	queueSubmitQueryResult, err := processor.Query(queueSubmitQuery)
	if err != nil {
		return nil, log.Errf(ctx, err, "SQL query failed: %v", queueSubmitQuery)
	}
	queueSubmitColumns := queueSubmitQueryResult.GetColumns()
	queueSubmitIds := queueSubmitColumns[0].GetLongValues()
	queueSubmitCommandBuffers := queueSubmitColumns[1].GetLongValues()
	submissionOrdering := make(map[int64]uint64)

	order := 0
	for i, v := range queueSubmitIds {
		if queueSubmitCommandBuffers[i] == 0 {
			// This is a spurious submission. See b/150854367
			log.W(ctx, "Spurious vkQueueSubmit slice with submission id %v", v)
			continue
		}
		submissionOrdering[v] = uint64(order)
		order++
	}

	trackIdCache := make(map[int64]bool)
	argsQueryCache := make(map[int64]*perfetto_service.QueryResult)
	slicesColumns := slicesQueryResult.GetColumns()
	numSliceRows := slicesQueryResult.GetNumRecords()
	slices := make([]*service.ProfilingData_GpuSlices_Slice, numSliceRows)
	groupsMap := map[api.CmdSubmissionKey]*service.ProfilingData_GpuSlices_Group{}
	groupIds := make([]int32, numSliceRows)
	var tracks []*service.ProfilingData_GpuSlices_Track
	// Grab all the column values. Depends on the order of columns selected in slicesQuery

	contextIds := slicesColumns[0].GetLongValues()
	extractTraceHandles(ctx, &contextIds, "VkDevice", handleMapping)

	renderTargets := slicesColumns[1].GetLongValues()
	extractTraceHandles(ctx, &renderTargets, "VkFramebuffer", handleMapping)

	commandBuffers := slicesColumns[5].GetLongValues()
	extractTraceHandles(ctx, &commandBuffers, "VkCommandBuffer", handleMapping)

	renderPasses := slicesColumns[6].GetLongValues()
	extractTraceHandles(ctx, &renderPasses, "VkRenderPass", handleMapping)

	frameIds := slicesColumns[2].GetLongValues()
	submissionIds := slicesColumns[3].GetLongValues()
	hwQueueIds := slicesColumns[4].GetLongValues()
	timestamps := slicesColumns[7].GetLongValues()
	durations := slicesColumns[8].GetLongValues()
	ids := slicesColumns[9].GetLongValues()
	names := slicesColumns[10].GetStringValues()
	depths := slicesColumns[11].GetLongValues()
	argSetIds := slicesColumns[12].GetLongValues()
	trackIds := slicesColumns[13].GetLongValues()
	trackNames := slicesColumns[14].GetStringValues()

	for i, v := range submissionIds {
		subOrder, ok := submissionOrdering[v]
		groupId := int32(-1)
		if ok {
			cb := uint64(commandBuffers[i])
			key := api.CmdSubmissionKey{subOrder, cb, uint64(renderPasses[i]), uint64(renderTargets[i])}
			if names[i] == "vertex" || names[i] == "fragment" {
				if group, ok := groupsMap[key]; ok {
					groupId = group.Id
				} else if indices, ok := syncData.SubmissionIndices[key]; ok {
					parent := utils.FindParentGroup(ctx, subOrder, cb, groupsMap, syncData.SubmissionIndices, capture)
					groupId = int32(len(groupsMap))
					group := &service.ProfilingData_GpuSlices_Group{
						Id:     groupId,
						Name:   fmt.Sprintf("RenderPass %v, RenderTarget %v", uint64(renderPasses[i]), uint64(renderTargets[i])),
						Parent: parent,
						Link:   &path.Command{Capture: capture, Indices: indices[0]},
					}
					groupsMap[key] = group
					names[i] = fmt.Sprintf("%v %v", groupsMap[key].Link.Indices, names[i])
				}
			}
		} else {
			log.W(ctx, "Encountered submission ID mismatch %v", v)
		}

		groupIds[i] = groupId
	}
	groups := []*service.ProfilingData_GpuSlices_Group{}
	for _, group := range groupsMap {
		groups = append(groups, group)
	}

	for i := uint64(0); i < numSliceRows; i++ {
		var argsQueryResult *perfetto_service.QueryResult
		var ok bool
		if argsQueryResult, ok = argsQueryCache[argSetIds[i]]; !ok {
			argsQuery := fmt.Sprintf(argsQueryFmt, argSetIds[i])
			argsQueryResult, err = processor.Query(argsQuery)
			if err != nil {
				log.W(ctx, "SQL query failed: %v", argsQuery)
			}
			argsQueryCache[argSetIds[i]] = argsQueryResult
		}
		argsColumns := argsQueryResult.GetColumns()
		numArgsRows := argsQueryResult.GetNumRecords()
		var extras []*service.ProfilingData_GpuSlices_Slice_Extra
		for j := uint64(0); j < numArgsRows; j++ {
			keys := argsColumns[0].GetStringValues()
			values := argsColumns[1].GetStringValues()
			extras = append(extras, &service.ProfilingData_GpuSlices_Slice_Extra{
				Name:  keys[j],
				Value: &service.ProfilingData_GpuSlices_Slice_Extra_StringValue{StringValue: values[j]},
			})
		}
		extras = append(extras, &service.ProfilingData_GpuSlices_Slice_Extra{
			Name:  "contextId",
			Value: &service.ProfilingData_GpuSlices_Slice_Extra_IntValue{IntValue: uint64(contextIds[i])},
		})
		extras = append(extras, &service.ProfilingData_GpuSlices_Slice_Extra{
			Name:  "renderTarget",
			Value: &service.ProfilingData_GpuSlices_Slice_Extra_IntValue{IntValue: uint64(renderTargets[i])},
		})
		extras = append(extras, &service.ProfilingData_GpuSlices_Slice_Extra{
			Name:  "commandBuffer",
			Value: &service.ProfilingData_GpuSlices_Slice_Extra_IntValue{IntValue: uint64(commandBuffers[i])},
		})
		extras = append(extras, &service.ProfilingData_GpuSlices_Slice_Extra{
			Name:  "renderPass",
			Value: &service.ProfilingData_GpuSlices_Slice_Extra_IntValue{IntValue: uint64(renderPasses[i])},
		})
		extras = append(extras, &service.ProfilingData_GpuSlices_Slice_Extra{
			Name:  "frameId",
			Value: &service.ProfilingData_GpuSlices_Slice_Extra_IntValue{IntValue: uint64(frameIds[i])},
		})
		extras = append(extras, &service.ProfilingData_GpuSlices_Slice_Extra{
			Name:  "submissionId",
			Value: &service.ProfilingData_GpuSlices_Slice_Extra_IntValue{IntValue: uint64(submissionIds[i])},
		})
		extras = append(extras, &service.ProfilingData_GpuSlices_Slice_Extra{
			Name:  "hwQueueId",
			Value: &service.ProfilingData_GpuSlices_Slice_Extra_IntValue{IntValue: uint64(hwQueueIds[i])},
		})

		slices[i] = &service.ProfilingData_GpuSlices_Slice{
			Ts:      uint64(timestamps[i]),
			Dur:     uint64(durations[i]),
			Id:      uint64(ids[i]),
			Label:   names[i],
			Depth:   int32(depths[i]),
			Extras:  extras,
			TrackId: int32(trackIds[i]),
			GroupId: groupIds[i],
		}

		if _, ok := trackIdCache[trackIds[i]]; !ok {
			trackIdCache[trackIds[i]] = true
			tracks = append(tracks, &service.ProfilingData_GpuSlices_Track{
				Id:   int32(trackIds[i]),
				Name: trackNames[i],
			})
		}
	}

	return &service.ProfilingData_GpuSlices{
		Slices: slices,
		Tracks: tracks,
		Groups: groups,
	}, nil
}

func processCounters(ctx context.Context, processor *perfetto.Processor, desc *device.GpuCounterDescriptor) ([]*service.ProfilingData_Counter, error) {
	counterTracksQueryResult, err := processor.Query(counterTracksQuery)
	if err != nil {
		return nil, log.Errf(ctx, err, "SQL query failed: %v", counterTracksQuery)
	}
	// t.id, name, unit, description, ts, value
	tracksColumns := counterTracksQueryResult.GetColumns()
	numTracksRows := counterTracksQueryResult.GetNumRecords()
	counters := make([]*service.ProfilingData_Counter, numTracksRows)
	// Grab all the column values. Depends on the order of columns selected in countersQuery
	trackIds := tracksColumns[0].GetLongValues()
	names := tracksColumns[1].GetStringValues()
	units := tracksColumns[2].GetStringValues()
	descriptions := tracksColumns[3].GetStringValues()

	for i := uint64(0); i < numTracksRows; i++ {
		countersQuery := fmt.Sprintf(countersQueryFmt, trackIds[i])
		countersQueryResult, err := processor.Query(countersQuery)
		countersColumns := countersQueryResult.GetColumns()
		if err != nil {
			return nil, log.Errf(ctx, err, "SQL query failed: %v", counterTracksQuery)
		}
		timestampsLong := countersColumns[0].GetLongValues()
		timestamps := make([]uint64, len(timestampsLong))
		for i, t := range timestampsLong {
			timestamps[i] = uint64(t)
		}
		values := countersColumns[1].GetDoubleValues()
		// TODO(apbodnar) Populate the `default` field once the trace processor supports it (b/147432390)
		counters[i] = &service.ProfilingData_Counter{
			Id:          uint32(trackIds[i]),
			Name:        names[i],
			Unit:        units[i],
			Description: descriptions[i],
			Timestamps:  timestamps,
			Values:      values,
		}
	}
	return counters, nil
}
<|endoftext|>


<commit_before>// +build darwin

package darwin

import (
	"time"

	"github.com/mackerelio/go-osstat/network"
	"github.com/mackerelio/golib/logging"
	"github.com/mackerelio/mackerel-agent/metrics"
)

/*
collect network interface I/O

`interface.{interface}.{metric}.delta`: The increased amount of network I/O per minute retrieved from the result of netstat -bni

interface = "en0", "en1" and so on...
*/

// InterfaceGenerator XXX
type InterfaceGenerator struct {
	Interval time.Duration
}

// metrics for posting to Mackerel

var interfaceLogger = logging.GetLogger("metrics.interface")

// Generate XXX
func (g *InterfaceGenerator) Generate() (metrics.Values, error) {
	prevValues, err := g.collectInterfacesValues()
	if err != nil {
		return nil, err
	}

	time.Sleep(g.Interval)

	currValues, err := g.collectInterfacesValues()
	if err != nil {
		return nil, err
	}

	ret := make(map[string]float64)
	for name, value := range prevValues {
		currValue, ok := currValues[name]
		if ok {
			ret[name+".delta"] = float64(currValue-value) / g.Interval.Seconds()
		}
	}

	return metrics.Values(ret), nil
}

func (g *InterfaceGenerator) collectInterfacesValues() (map[string]uint64, error) {
	networks, err := network.Get()
	if err != nil {
		interfaceLogger.Errorf("failed to get network statistics: %s", err)
		return nil, err
	}
	if len(networks) == 0 {
		return nil, nil
	}
	results := make(map[string]uint64, len(networks)*2)
	for _, network := range networks {
		results["interface."+network.Name+".rxBytes"] = network.RxBytes
		results["interface."+network.Name+".txBytes"] = network.TxBytes
	}
	return results, nil
}
<commit_msg>sanitize interface name on darwin<commit_after>// +build darwin

package darwin

import (
	"time"

	"github.com/mackerelio/go-osstat/network"
	"github.com/mackerelio/golib/logging"
	"github.com/mackerelio/mackerel-agent/metrics"
	"github.com/mackerelio/mackerel-agent/util"
)

/*
collect network interface I/O

`interface.{interface}.{metric}.delta`: The increased amount of network I/O per minute retrieved from the result of netstat -bni

interface = "en0", "en1" and so on...
*/

// InterfaceGenerator XXX
type InterfaceGenerator struct {
	Interval time.Duration
}

// metrics for posting to Mackerel

var interfaceLogger = logging.GetLogger("metrics.interface")

// Generate XXX
func (g *InterfaceGenerator) Generate() (metrics.Values, error) {
	prevValues, err := g.collectInterfacesValues()
	if err != nil {
		return nil, err
	}

	time.Sleep(g.Interval)

	currValues, err := g.collectInterfacesValues()
	if err != nil {
		return nil, err
	}

	ret := make(map[string]float64)
	for name, value := range prevValues {
		currValue, ok := currValues[name]
		if ok {
			ret[name+".delta"] = float64(currValue-value) / g.Interval.Seconds()
		}
	}

	return metrics.Values(ret), nil
}

func (g *InterfaceGenerator) collectInterfacesValues() (map[string]uint64, error) {
	networks, err := network.Get()
	if err != nil {
		interfaceLogger.Errorf("failed to get network statistics: %s", err)
		return nil, err
	}
	if len(networks) == 0 {
		return nil, nil
	}
	results := make(map[string]uint64, len(networks)*2)
	for _, network := range networks {
		name := util.SanitizeMetricKey(network.Name)
		results["interface."+name+".rxBytes"] = network.RxBytes
		results["interface."+name+".txBytes"] = network.TxBytes
	}
	return results, nil
}
<|endoftext|>


<commit_before>// Copyright 2012 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fauxfile

import (
	"errors"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"
)

func GetPathError(path string, message string) error {
	return &os.PathError{
		Path: path,
		Err:  errors.New(message),
	}
}

type MockFilesystem struct {
	cwd  *MockFileInfo
	root *MockFileInfo
}

func NewMockFilesystem() *MockFilesystem {
	root := &MockFileInfo{
		file: &MockFile{
			name:       "/",
			path:       "/",
			filesystem: nil,
			mode:       os.ModeDir | 0755,
			modified:   time.Now(),
			data:       nil,
			parent:     nil,
			children:   map[string]*MockFileInfo{},
		},
	}
	mf := &MockFilesystem{
		cwd:  root,
		root: root,
	}
	root.file.filesystem = mf
	return mf
}

func (mf *MockFilesystem) getpath(path string) string {
	if filepath.IsAbs(path) {
		return filepath.Clean(path)
	}
	return filepath.Join(mf.cwd.file.path, path)
}

func (mf *MockFilesystem) resolve(path string) (*MockFileInfo, error) {
	path = mf.getpath(path)
	parts := strings.Split(path, string(filepath.Separator))
	ptr := mf.root
	for _, part := range parts {
		if part == "" {
			continue
		}
		if child := ptr.Child(part); child != nil {
			ptr = ptr.Child(part)
		} else {
			return nil, GetPathError(path, "Path does not exist")
		}
	}
	return ptr, nil
}

func (mf *MockFilesystem) exists(path string) bool {
	_, err := mf.resolve(path)
	return err == nil
}

func (mf *MockFilesystem) Chdir(dir string) error {
	fi, err := mf.resolve(dir)
	if err == nil {
		if fi.IsDir() {
			mf.cwd = fi
			return nil
		} else {
			return GetPathError(dir, "Path is not a directory")
		}
	}
	return err
}

func (mf *MockFilesystem) Mkdir(name string, perm os.FileMode) error {
	path := mf.getpath(name)
	parentpath, dirname := filepath.Split(path)
	fi, err := mf.resolve(parentpath)
	if err != nil {
		return err
	}
	if child := fi.Child(dirname); child != nil {
		return GetPathError(path, "Path already exists")
	}
	fi.file.children[dirname] = &MockFileInfo{
		file: &MockFile{
			name:       dirname,
			path:       path,
			filesystem: mf,
			mode:       perm | os.ModeDir,
			modified:   time.Now(),
			data:       nil,
			parent:     fi,
			children:   map[string]*MockFileInfo{},
		},
	}
	fi.file.modified = time.Now()
	return nil
}

func (mf *MockFilesystem) MkdirAll(path string, perm os.FileMode) error {
	path = mf.getpath(path)
	parts := strings.Split(path, string(filepath.Separator))
	base := "/"
	for _, part := range parts {
		if part == "" {
			continue
		}
		base = filepath.Join(base, part)
		if err := mf.Mkdir(base, perm); err != nil {
			return err
		}
	}
	return nil
}

func (mf *MockFilesystem) Remove(name string) error {
	fi, err := mf.resolve(name)
	if err != nil {
		return err
	}
	if len(fi.Children()) > 0 {
		return GetPathError(name, "Directory contains children")
	}
	delete(fi.Parent().Children(), fi.file.name)
	fi.Parent().file.modified = time.Now()
	return nil
}

func (mf *MockFilesystem) RemoveAll(path string) error {
	fi, err := mf.resolve(path)
	if err != nil {
		return err
	}
	delete(fi.Parent().Children(), fi.file.name)
	fi.Parent().file.modified = time.Now()
	return nil
}

func (mf *MockFilesystem) Rename(oldname string, newname string) error {
	return errors.New("Not implemented")
}

func (mf *MockFilesystem) Create(name string) (file File, err error) {
	path := mf.getpath(name)
	dir, filename := filepath.Split(path)
	fi, err := mf.resolve(dir)
	if err != nil {
		return nil, err
	}
	fi.file.children[filename] = &MockFileInfo{
		file: &MockFile{
			name:       filename,
			path:       path,
			filesystem: mf,
			mode:       0666,
			modified:   time.Now(),
			data:       nil,
			parent:     fi,
			children:   nil,
		},
	}
	fi.file.modified = time.Now()
	return fi.Child(filename).file, nil
}

func (mf *MockFilesystem) Open(name string) (file File, err error) {
	fi, err := mf.resolve(name)
	if err != nil {
		return nil, err
	}
	return fi.file, nil
}

func (mf *MockFilesystem) OpenFile(name string, flag int, perm os.FileMode) (file File, err error) {
	return nil, errors.New("Not implemented")
}

type MockFile struct {
	name       string
	path       string
	filesystem *MockFilesystem
	mode       os.FileMode
	modified   time.Time
	data       *[]byte
	parent     *MockFileInfo
	children   map[string]*MockFileInfo
}

func (mf *MockFile) Chdir() error {
	return mf.filesystem.Chdir(filepath.Dir(mf.path))
}

func (mf *MockFile) Chmod(mode os.FileMode) error {
	mf.mode = mode
	return nil
}

func (mf *MockFile) Close() error {
	return errors.New("Not implemented")
}

func (mf *MockFile) Name() string {
	return mf.name
}

func (mf *MockFile) Read(b []byte) (n int, err error) {
	return 0, errors.New("Not implemented")
}

func (mf *MockFile) ReadAt(b []byte, off int64) (n int, err error) {
	return 0, errors.New("Not implemented")
}

func (mf *MockFile) Readdir(n int) (fi []os.FileInfo, err error) {
	// TODO: Enable returning additional elements in subsequent calls.
	fi = make([]os.FileInfo, 0)
	limit := len(mf.children)
	if n > 0 {
		limit = n
	}
	if len(mf.children) < limit {
		err = io.EOF
	}
	if len(mf.children) == 0 {
		return
	}
	i := 0
	for _, child := range mf.children {
		if i == limit {
			break
		}
		fi = append(fi, child)
		i++
	}
	return
}

func (mf *MockFile) Readdirnames(n int) (names []string, err error) {
	fi, err := mf.Readdir(n)
	names = make([]string, len(fi))
	for i, f := range fi {
		names[i] = f.Name()
	}
	return names, err
}

func (mf *MockFile) Seek(offset int64, whence int) (ret int64, err error) {
	return 0, errors.New("Not implemented")
}

func (mf *MockFile) Stat() (fi os.FileInfo, err error) {
	return nil, errors.New("Not implemented")
}

func (mf *MockFile) Sync() (err error) {
	return errors.New("Not implemented")
}

func (mf *MockFile) Truncate(size int64) error {
	return errors.New("Not implemented")
}

func (mf *MockFile) Write(b []byte) (n int, err error) {
	return 0, errors.New("Not implemented")
}

func (mf *MockFile) WriteAt(b []byte, off int64) (n int, err error) {
	return 0, errors.New("Not implemented")
}

func (mf *MockFile) WriteString(s string) (ret int, err error) {
	return 0, errors.New("Not implemented")
}

type MockFileInfo struct {
	file *MockFile
}

func (mfi *MockFileInfo) Parent() *MockFileInfo {
	return mfi.file.parent
}

func (mfi *MockFileInfo) Children() map[string]*MockFileInfo {
	return mfi.file.children
}

func (mfi *MockFileInfo) Child(name string) *MockFileInfo {
	return mfi.file.children[name]
}

func (mfi *MockFileInfo) Name() string {
	return mfi.file.name
}

func (mfi *MockFileInfo) Size() int64 {
	return int64(len(*mfi.file.data))
}

func (mfi *MockFileInfo) Mode() os.FileMode {
	return mfi.file.mode
}

func (mfi *MockFileInfo) ModTime() time.Time {
	return mfi.file.modified
}

func (mfi *MockFileInfo) IsDir() bool {
	return mfi.file.mode.IsDir()
}

func (mfi *MockFileInfo) Sys() interface{} {
	return nil
}
<commit_msg>Implements Close<commit_after>// Copyright 2012 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fauxfile

import (
	"errors"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"
)

func GetPathError(path string, message string) error {
	return &os.PathError{
		Path: path,
		Err:  errors.New(message),
	}
}

type MockFilesystem struct {
	cwd  *MockFileInfo
	root *MockFileInfo
}

func NewMockFilesystem() *MockFilesystem {
	root := &MockFileInfo{
		file: &MockFile{
			name:       "/",
			path:       "/",
			filesystem: nil,
			mode:       os.ModeDir | 0755,
			modified:   time.Now(),
			data:       nil,
			parent:     nil,
			children:   map[string]*MockFileInfo{},
		},
	}
	mf := &MockFilesystem{
		cwd:  root,
		root: root,
	}
	root.file.filesystem = mf
	return mf
}

func (mf *MockFilesystem) getpath(path string) string {
	if filepath.IsAbs(path) {
		return filepath.Clean(path)
	}
	return filepath.Join(mf.cwd.file.path, path)
}

func (mf *MockFilesystem) resolve(path string) (*MockFileInfo, error) {
	path = mf.getpath(path)
	parts := strings.Split(path, string(filepath.Separator))
	ptr := mf.root
	for _, part := range parts {
		if part == "" {
			continue
		}
		if child := ptr.Child(part); child != nil {
			ptr = ptr.Child(part)
		} else {
			return nil, GetPathError(path, "Path does not exist")
		}
	}
	return ptr, nil
}

func (mf *MockFilesystem) exists(path string) bool {
	_, err := mf.resolve(path)
	return err == nil
}

func (mf *MockFilesystem) Chdir(dir string) error {
	fi, err := mf.resolve(dir)
	if err == nil {
		if fi.IsDir() {
			mf.cwd = fi
			return nil
		} else {
			return GetPathError(dir, "Path is not a directory")
		}
	}
	return err
}

func (mf *MockFilesystem) Mkdir(name string, perm os.FileMode) error {
	path := mf.getpath(name)
	parentpath, dirname := filepath.Split(path)
	fi, err := mf.resolve(parentpath)
	if err != nil {
		return err
	}
	if child := fi.Child(dirname); child != nil {
		return GetPathError(path, "Path already exists")
	}
	fi.file.children[dirname] = &MockFileInfo{
		file: &MockFile{
			name:       dirname,
			path:       path,
			filesystem: mf,
			mode:       perm | os.ModeDir,
			modified:   time.Now(),
			data:       nil,
			parent:     fi,
			children:   map[string]*MockFileInfo{},
			closed:     true,
		},
	}
	fi.file.modified = time.Now()
	return nil
}

func (mf *MockFilesystem) MkdirAll(path string, perm os.FileMode) error {
	path = mf.getpath(path)
	parts := strings.Split(path, string(filepath.Separator))
	base := "/"
	for _, part := range parts {
		if part == "" {
			continue
		}
		base = filepath.Join(base, part)
		if err := mf.Mkdir(base, perm); err != nil {
			return err
		}
	}
	return nil
}

func (mf *MockFilesystem) Remove(name string) error {
	fi, err := mf.resolve(name)
	if err != nil {
		return err
	}
	if len(fi.Children()) > 0 {
		return GetPathError(name, "Directory contains children")
	}
	delete(fi.Parent().Children(), fi.file.name)
	fi.Parent().file.modified = time.Now()
	return nil
}

func (mf *MockFilesystem) RemoveAll(path string) error {
	fi, err := mf.resolve(path)
	if err != nil {
		return err
	}
	delete(fi.Parent().Children(), fi.file.name)
	fi.Parent().file.modified = time.Now()
	return nil
}

func (mf *MockFilesystem) Rename(oldname string, newname string) error {
	return errors.New("Not implemented")
}

func (mf *MockFilesystem) Create(name string) (file File, err error) {
	path := mf.getpath(name)
	dir, filename := filepath.Split(path)
	fi, err := mf.resolve(dir)
	if err != nil {
		return nil, err
	}
	fi.file.children[filename] = &MockFileInfo{
		file: &MockFile{
			name:       filename,
			path:       path,
			filesystem: mf,
			mode:       0666,
			modified:   time.Now(),
			data:       nil,
			parent:     fi,
			children:   nil,
			closed:     false,
		},
	}
	fi.file.modified = time.Now()
	return fi.Child(filename).file, nil
}

func (mf *MockFilesystem) Open(name string) (file File, err error) {
	fi, err := mf.resolve(name)
	if err != nil {
		return nil, err
	}
	fi.file.closed = false
	return fi.file, nil
}

func (mf *MockFilesystem) OpenFile(name string, flag int, perm os.FileMode) (file File, err error) {
	return nil, errors.New("Not implemented")
}

type MockFile struct {
	name       string
	path       string
	filesystem *MockFilesystem
	mode       os.FileMode
	modified   time.Time
	data       *[]byte
	parent     *MockFileInfo
	children   map[string]*MockFileInfo
	closed     bool
}

func (mf *MockFile) Chdir() error {
	return mf.filesystem.Chdir(filepath.Dir(mf.path))
}

func (mf *MockFile) Chmod(mode os.FileMode) error {
	mf.mode = mode
	return nil
}

func (mf *MockFile) Close() error {
	mf.closed = true
	return nil
}

func (mf *MockFile) Name() string {
	return mf.name
}

func (mf *MockFile) Read(b []byte) (n int, err error) {
	return 0, errors.New("Not implemented")
}

func (mf *MockFile) ReadAt(b []byte, off int64) (n int, err error) {
	return 0, errors.New("Not implemented")
}

func (mf *MockFile) Readdir(n int) (fi []os.FileInfo, err error) {
	// TODO: Enable returning additional elements in subsequent calls.
	fi = make([]os.FileInfo, 0)
	limit := len(mf.children)
	if n > 0 {
		limit = n
	}
	if len(mf.children) < limit {
		err = io.EOF
	}
	if len(mf.children) == 0 {
		return
	}
	i := 0
	for _, child := range mf.children {
		if i == limit {
			break
		}
		fi = append(fi, child)
		i++
	}
	return
}

func (mf *MockFile) Readdirnames(n int) (names []string, err error) {
	fi, err := mf.Readdir(n)
	names = make([]string, len(fi))
	for i, f := range fi {
		names[i] = f.Name()
	}
	return names, err
}

func (mf *MockFile) Seek(offset int64, whence int) (ret int64, err error) {
	return 0, errors.New("Not implemented")
}

func (mf *MockFile) Stat() (fi os.FileInfo, err error) {
	return nil, errors.New("Not implemented")
}

func (mf *MockFile) Sync() (err error) {
	return errors.New("Not implemented")
}

func (mf *MockFile) Truncate(size int64) error {
	return errors.New("Not implemented")
}

func (mf *MockFile) Write(b []byte) (n int, err error) {
	return 0, errors.New("Not implemented")
}

func (mf *MockFile) WriteAt(b []byte, off int64) (n int, err error) {
	return 0, errors.New("Not implemented")
}

func (mf *MockFile) WriteString(s string) (ret int, err error) {
	return 0, errors.New("Not implemented")
}

type MockFileInfo struct {
	file *MockFile
}

func (mfi *MockFileInfo) Parent() *MockFileInfo {
	return mfi.file.parent
}

func (mfi *MockFileInfo) Children() map[string]*MockFileInfo {
	return mfi.file.children
}

func (mfi *MockFileInfo) Child(name string) *MockFileInfo {
	return mfi.file.children[name]
}

func (mfi *MockFileInfo) Name() string {
	return mfi.file.name
}

func (mfi *MockFileInfo) Size() int64 {
	return int64(len(*mfi.file.data))
}

func (mfi *MockFileInfo) Mode() os.FileMode {
	return mfi.file.mode
}

func (mfi *MockFileInfo) ModTime() time.Time {
	return mfi.file.modified
}

func (mfi *MockFileInfo) IsDir() bool {
	return mfi.file.mode.IsDir()
}

func (mfi *MockFileInfo) Sys() interface{} {
	return nil
}
<|endoftext|>


<commit_before>package gorma

const modelTmpl = `// {{if .Description}}{{.Description}}{{else}}app.{{gotypename . 0}} storage type{{end}}
// Identifier: {{ $typeName :=  gotypename . 0}}{{$typeName := demodel $typeName}}
{{$td := gotypedef . 0 true false}}type {{$typeName}} {{modeldef $td .}}
{{ $belongsto := index .Metadata "github.com/bketelsen/gorma#belongsto" }}
{{ $m2m := index .Metadata "github.com/bketelsen/gorma#many2many" }}
func {{$typeName}}FromCreatePayload(ctx *app.Create{{demodel $typeName}}Context) {{$typeName}} {
	payload := ctx.Payload
	m := {{$typeName}}{}
	copier.Copy(&m, payload)
	{{ if ne $belongsto "" }} m.{{ $belongsto }}ID=int(ctx.{{ demodel $belongsto }}ID){{end}}
	return m
}

func {{$typeName}}FromUpdatePayload(ctx *app.Update{{demodel $typeName}}Context) {{$typeName}} {
	payload := ctx.Payload
	m := {{$typeName}}{}
	copier.Copy(&m, payload)
	return m
}
func (m {{$typeName}}) ToApp() *app.{{demodel $typeName}} {
	target := app.{{demodel $typeName}}{}
	copier.Copy(&target, &m)
	return &target
}
{{ $roler := index .Metadata "github.com/bketelsen/gorma#roler" }}
{{ if ne $roler "" }}
func (m {{$typeName}}) GetRole() string {
	return m.Role
}
{{end}}

type {{$typeName}}Storage interface {
	List(ctx *app.List{{demodel $typeName}}Context) []{{$typeName}}
	Get(ctx *app.Show{{demodel $typeName }}Context) ({{$typeName}}, error)
	Add(ctx *app.Create{{demodel $typeName}}Context) ({{$typeName}}, error)
	Update(ctx *app.Update{{demodel $typeName}}Context) (error)
	Delete(ctx *app.Delete{{demodel $typeName}}Context) (error)
	{{ storagedef . }}
}

type {{$typeName}}DB struct {
	DB gorm.DB
}
{{ if ne $belongsto "" }}{{$barray := split $belongsto ","}}{{ range $idx, $bt := $barray}}
// would prefer to just pass a context in here, but they're all different, so can't
func {{$typeName}}Filter(parentid int, originaldb *gorm.DB) func(db *gorm.DB) *gorm.DB {
	if parentid > 0 {
		return func(db *gorm.DB) *gorm.DB {
			return db.Where("{{ snake $bt }}_id = ?", parentid)
		}
	} else {
		return func(db *gorm.DB) *gorm.DB {
			return db
		}
	}
}{{end}}{{end}}
func New{{$typeName}}DB(db gorm.DB) *{{$typeName}}DB {
	return &{{$typeName}}DB{DB: db}
}

func (m *{{$typeName}}DB) List(ctx *app.List{{demodel $typeName}}Context) []{{$typeName}} {

	var objs []{{$typeName}}
    {{ if ne $belongsto "" }}m.DB.Scopes({{$typeName}}Filter(ctx.{{demodel $belongsto}}ID, &m.DB)).Find(&objs){{ else }} m.DB.Find(&objs) {{end}}
	return objs
}

func (m *{{$typeName}}DB) Get(ctx *app.Show{{demodel $typeName}}Context) ({{$typeName}}, error) {

	var obj {{$typeName}}

	err := m.DB.Find(&obj, ctx.{{demodel $typeName}}ID).Error
	if err != nil {
		ctx.Error(err.Error())
	}
	return obj, err
}

func (m *{{$typeName}}DB) Add(ctx *app.Create{{demodel $typeName}}Context) ({{$typeName}}, error) {
	model := {{$typeName}}FromCreatePayload(ctx)
	err := m.DB.Create(&model).Error
	return model, err
}
func (m *{{$typeName}}DB) Update(ctx *app.Update{{demodel $typeName}}Context) error {
	getCtx, err := app.NewShow{{demodel $typeName}}Context(ctx.Context)
	if err != nil {
		return  err
	}
	obj, err := m.Get(getCtx)
	if err != nil {
		return  err
	}
	err = m.DB.Model(&obj).Updates({{$typeName}}FromUpdatePayload(ctx)).Error
	if err != nil {
		ctx.Error(err.Error())
	}
	return err
}
func (m *{{$typeName}}DB) Delete(ctx *app.Delete{{demodel $typeName}}Context)  error {
	var obj {{$typeName}}
	err := m.DB.Delete(&obj, ctx.{{demodel $typeName}}ID).Error
	if err != nil {
		ctx.Logger.Error(err.Error())
		return  err
	}
	return  nil
}

{{ if ne $m2m "" }}{{$barray := split $m2m ","}}{{ range $idx, $bt := $barray}}
{{ $pieces := split $bt ":" }} {{ $lowertype := index $pieces 0  }} {{ $lower := lower $lowertype }}  {{ $lowerplural := index $pieces 1  }} {{ $lowerplural := lower $lowerplural}}
func (m *{{$typeName}}DB) Delete{{index $pieces 1}}(ctx *app.Delete{{$lower}}{{$typeName}}Context)  error {
	var obj {{$typeName}}
	err := m.DB.Delete(&obj, ctx.{{demodel $typeName}}ID).Error
	if err != nil {
		ctx.Logger.Error(err.Error())
		return  err
	}
	return  nil
}
func (m *{{$typeName}}DB) Add{{index $pieces 1}}(ctx *app.Add{{$lower}}{{$typeName}}Context) ( {{$typeName}}, error) {
	var obj {{$typeName}}
	err := m.DB.Delete(&obj, ctx.{{demodel $typeName}}ID).Error
	if err != nil {
		ctx.Logger.Error(err.Error())
		return  err
	}
	return  nil
}
func (m *{{$typeName}}DB) List{{index $pieces 0}}(ctx *app.List{{plural $lowerplural}}{{$typeName}}Context)  {{$typeName}} {
	var obj {{$typeName}}
	err := m.DB.Delete(&obj, ctx.{{demodel $typeName}}ID).Error
	if err != nil {
		ctx.Logger.Error(err.Error())
		return  err
	}
	return  nil
}
{{end}}{{end}}

type Mock{{$typeName}}Storage struct {
	{{$typeName}}List  map[int]{{$typeName}}
	nextID int
	mut sync.Mutex
}
{{if ne $belongsto ""}}{{$barray := split $belongsto ","}}{{ range $idx, $bt := $barray}}
func filter{{$typeName}}By{{$bt}}(parent int, list []{{$typeName}}) []{{$typeName}} {
	filtered := make([]{{$typeName}},0)
	for _,o := range list {
		if o.{{$bt}}ID == int(parent) {
			filtered = append(filtered,o)
		}
	}
	return filtered
}
{{end}}{{end}}

func NewMock{{$typeName}}Storage() *Mock{{$typeName}}Storage {
	ml := make(map[int]{{$typeName}}, 0)
	return &Mock{{$typeName}}Storage{ {{$typeName}}List: ml}
}

func (db *Mock{{$typeName}}Storage) List(ctx *app.List{{demodel $typeName}}Context) []{{$typeName}} {
	var list []{{$typeName}} = make([]{{$typeName}}, 0)
	for _, v := range db.{{$typeName}}List {
		list = append(list, v)
	}
{{if ne $belongsto ""}}
return filter{{$typeName}}By{{$belongsto}}(ctx.{{$belongsto}}ID, list) {{else}}return list{{end}}
}

func (db *Mock{{$typeName}}Storage) Get(ctx *app.Show{{demodel $typeName}}Context) ({{$typeName}}, error) {

	var obj {{$typeName}}

	obj, ok := db.{{$typeName}}List[int(ctx.{{demodel $typeName}}ID)]
	if ok {
		return obj, nil
	} else {
		return obj, errors.New("{{$typeName}} does not exist")
	}
}

func (db *Mock{{$typeName}}Storage) Add(ctx *app.Create{{demodel $typeName}}Context)  ({{$typeName}}, error) {
	u := {{$typeName}}FromCreatePayload(ctx)
	db.mut.Lock()
	db.nextID = db.nextID + 1
	u.ID = db.nextID
	db.mut.Unlock()

	db.{{$typeName}}List[u.ID] = u
	return u, nil
}

func (db *Mock{{$typeName}}Storage) Update(ctx *app.Update{{demodel $typeName}}Context) error {
	id := int(ctx.{{demodel $typeName}}ID)
	_, ok := db.{{$typeName}}List[id]
	if ok {
		db.{{$typeName}}List[id] = {{$typeName}}FromUpdatePayload(ctx)
		return  nil
	} else {
		return errors.New("{{$typeName}} does not exist")
	}
}

func (db *Mock{{$typeName}}Storage) Delete(ctx *app.Delete{{demodel $typeName}}Context)  error {
	_, ok := db.{{$typeName}}List[int(ctx.{{demodel $typeName}}ID)]
	if ok {
		delete(db.{{$typeName}}List, int(ctx.{{demodel $typeName}}ID))
		return  nil
	} else {
		return  errors.New("Could not delete this user")
	}
}
`
<commit_msg>update<commit_after>package gorma

const modelTmpl = `// {{if .Description}}{{.Description}}{{else}}app.{{gotypename . 0}} storage type{{end}}
// Identifier: {{ $typeName :=  gotypename . 0}}{{$typeName := demodel $typeName}}
{{$td := gotypedef . 0 true false}}type {{$typeName}} {{modeldef $td .}}
{{ $belongsto := index .Metadata "github.com/bketelsen/gorma#belongsto" }}
{{ $m2m := index .Metadata "github.com/bketelsen/gorma#many2many" }}
func {{$typeName}}FromCreatePayload(ctx *app.Create{{demodel $typeName}}Context) {{$typeName}} {
	payload := ctx.Payload
	m := {{$typeName}}{}
	copier.Copy(&m, payload)
	{{ if ne $belongsto "" }} m.{{ $belongsto }}ID=int(ctx.{{ demodel $belongsto }}ID){{end}}
	return m
}

func {{$typeName}}FromUpdatePayload(ctx *app.Update{{demodel $typeName}}Context) {{$typeName}} {
	payload := ctx.Payload
	m := {{$typeName}}{}
	copier.Copy(&m, payload)
	return m
}
func (m {{$typeName}}) ToApp() *app.{{demodel $typeName}} {
	target := app.{{demodel $typeName}}{}
	copier.Copy(&target, &m)
	return &target
}
{{ $roler := index .Metadata "github.com/bketelsen/gorma#roler" }}
{{ if ne $roler "" }}
func (m {{$typeName}}) GetRole() string {
	return m.Role
}
{{end}}

type {{$typeName}}Storage interface {
	List(ctx *app.List{{demodel $typeName}}Context) []{{$typeName}}
	Get(ctx *app.Show{{demodel $typeName }}Context) ({{$typeName}}, error)
	Add(ctx *app.Create{{demodel $typeName}}Context) ({{$typeName}}, error)
	Update(ctx *app.Update{{demodel $typeName}}Context) (error)
	Delete(ctx *app.Delete{{demodel $typeName}}Context) (error)
	{{ storagedef . }}
}

type {{$typeName}}DB struct {
	DB gorm.DB
}
{{ if ne $belongsto "" }}{{$barray := split $belongsto ","}}{{ range $idx, $bt := $barray}}
// would prefer to just pass a context in here, but they're all different, so can't
func {{$typeName}}Filter(parentid int, originaldb *gorm.DB) func(db *gorm.DB) *gorm.DB {
	if parentid > 0 {
		return func(db *gorm.DB) *gorm.DB {
			return db.Where("{{ snake $bt }}_id = ?", parentid)
		}
	} else {
		return func(db *gorm.DB) *gorm.DB {
			return db
		}
	}
}{{end}}{{end}}
func New{{$typeName}}DB(db gorm.DB) *{{$typeName}}DB {
	return &{{$typeName}}DB{DB: db}
}

func (m *{{$typeName}}DB) List(ctx *app.List{{demodel $typeName}}Context) []{{$typeName}} {

	var objs []{{$typeName}}
    {{ if ne $belongsto "" }}m.DB.Scopes({{$typeName}}Filter(ctx.{{demodel $belongsto}}ID, &m.DB)).Find(&objs){{ else }} m.DB.Find(&objs) {{end}}
	return objs
}

func (m *{{$typeName}}DB) Get(ctx *app.Show{{demodel $typeName}}Context) ({{$typeName}}, error) {

	var obj {{$typeName}}

	err := m.DB.Find(&obj, ctx.{{demodel $typeName}}ID).Error
	if err != nil {
		ctx.Error(err.Error())
	}
	return obj, err
}

func (m *{{$typeName}}DB) Add(ctx *app.Create{{demodel $typeName}}Context) ({{$typeName}}, error) {
	model := {{$typeName}}FromCreatePayload(ctx)
	err := m.DB.Create(&model).Error
	return model, err
}
func (m *{{$typeName}}DB) Update(ctx *app.Update{{demodel $typeName}}Context) error {
	getCtx, err := app.NewShow{{demodel $typeName}}Context(ctx.Context)
	if err != nil {
		return  err
	}
	obj, err := m.Get(getCtx)
	if err != nil {
		return  err
	}
	err = m.DB.Model(&obj).Updates({{$typeName}}FromUpdatePayload(ctx)).Error
	if err != nil {
		ctx.Error(err.Error())
	}
	return err
}
func (m *{{$typeName}}DB) Delete(ctx *app.Delete{{demodel $typeName}}Context)  error {
	var obj {{$typeName}}
	err := m.DB.Delete(&obj, ctx.{{demodel $typeName}}ID).Error
	if err != nil {
		ctx.Logger.Error(err.Error())
		return  err
	}
	return  nil
}

{{ if ne $m2m "" }}{{$barray := split $m2m ","}}{{ range $idx, $bt := $barray}}
{{ $pieces := split $bt ":" }} {{ $lowertype := index $pieces 1  }} {{ $lower := lower $lowertype }}  {{ $lowerplural := index $pieces 0  }} {{ $lowerplural := lower $lowerplural}}
func (m *{{$typeName}}DB) Delete{{index $pieces 1}}(ctx *app.Delete{{$lower}}{{$typeName}}Context)  error {
	var obj {{$typeName}}
	err := m.DB.Delete(&obj, ctx.{{demodel $typeName}}ID).Error
	if err != nil {
		ctx.Logger.Error(err.Error())
		return  err
	}
	return  nil
}
func (m *{{$typeName}}DB) Add{{index $pieces 1}}(ctx *app.Add{{$lower}}{{$typeName}}Context) ( {{$typeName}}, error) {
	var obj {{$typeName}}
	err := m.DB.Delete(&obj, ctx.{{demodel $typeName}}ID).Error
	if err != nil {
		ctx.Logger.Error(err.Error())
		return  err
	}
	return  nil
}
func (m *{{$typeName}}DB) List{{index $pieces 0}}(ctx *app.List{{plural $lowerplural}}{{$typeName}}Context)  {{$typeName}} {
	var obj {{$typeName}}
	err := m.DB.Delete(&obj, ctx.{{demodel $typeName}}ID).Error
	if err != nil {
		ctx.Logger.Error(err.Error())
		return  err
	}
	return  nil
}
{{end}}{{end}}

type Mock{{$typeName}}Storage struct {
	{{$typeName}}List  map[int]{{$typeName}}
	nextID int
	mut sync.Mutex
}
{{if ne $belongsto ""}}{{$barray := split $belongsto ","}}{{ range $idx, $bt := $barray}}
func filter{{$typeName}}By{{$bt}}(parent int, list []{{$typeName}}) []{{$typeName}} {
	filtered := make([]{{$typeName}},0)
	for _,o := range list {
		if o.{{$bt}}ID == int(parent) {
			filtered = append(filtered,o)
		}
	}
	return filtered
}
{{end}}{{end}}

func NewMock{{$typeName}}Storage() *Mock{{$typeName}}Storage {
	ml := make(map[int]{{$typeName}}, 0)
	return &Mock{{$typeName}}Storage{ {{$typeName}}List: ml}
}

func (db *Mock{{$typeName}}Storage) List(ctx *app.List{{demodel $typeName}}Context) []{{$typeName}} {
	var list []{{$typeName}} = make([]{{$typeName}}, 0)
	for _, v := range db.{{$typeName}}List {
		list = append(list, v)
	}
{{if ne $belongsto ""}}
return filter{{$typeName}}By{{$belongsto}}(ctx.{{$belongsto}}ID, list) {{else}}return list{{end}}
}

func (db *Mock{{$typeName}}Storage) Get(ctx *app.Show{{demodel $typeName}}Context) ({{$typeName}}, error) {

	var obj {{$typeName}}

	obj, ok := db.{{$typeName}}List[int(ctx.{{demodel $typeName}}ID)]
	if ok {
		return obj, nil
	} else {
		return obj, errors.New("{{$typeName}} does not exist")
	}
}

func (db *Mock{{$typeName}}Storage) Add(ctx *app.Create{{demodel $typeName}}Context)  ({{$typeName}}, error) {
	u := {{$typeName}}FromCreatePayload(ctx)
	db.mut.Lock()
	db.nextID = db.nextID + 1
	u.ID = db.nextID
	db.mut.Unlock()

	db.{{$typeName}}List[u.ID] = u
	return u, nil
}

func (db *Mock{{$typeName}}Storage) Update(ctx *app.Update{{demodel $typeName}}Context) error {
	id := int(ctx.{{demodel $typeName}}ID)
	_, ok := db.{{$typeName}}List[id]
	if ok {
		db.{{$typeName}}List[id] = {{$typeName}}FromUpdatePayload(ctx)
		return  nil
	} else {
		return errors.New("{{$typeName}} does not exist")
	}
}

func (db *Mock{{$typeName}}Storage) Delete(ctx *app.Delete{{demodel $typeName}}Context)  error {
	_, ok := db.{{$typeName}}List[int(ctx.{{demodel $typeName}}ID)]
	if ok {
		delete(db.{{$typeName}}List, int(ctx.{{demodel $typeName}}ID))
		return  nil
	} else {
		return  errors.New("Could not delete this user")
	}
}
`
<|endoftext|>


<commit_before>package cah

import (
	// "errors"
	"encoding/json"
	//"fmt"
	"github.com/gamelost/bot3server/server"
	"io/ioutil"
	// "log"
	"math/rand"
	"net/http"
	"strings"
	"time"
	"unicode"
)

// source URL for CAH card templates
const CAH_SOURCE_URL = "https://raw.githubusercontent.com/nodanaonlyzuul/against-humanity/master/source/cards.json"

type CahService struct {
	RandomNG          *rand.Rand
	CahCardCollection *CahCardCollection
}

type CahCard struct {
	Id         int64
	CardType   string
	NumAnswers int64
	Text       string
}

type CahCardCollection []struct {
	CardType   string `json:"cardType"`
	Expansion  string `json:"expansion"`
	Id         int64  `json:"id"`
	NumAnswers int64  `json:"numAnswers"`
	Text       string `json:"text"`
}

func (svc *CahService) NewService() server.BotHandler {

	var newSvc = &CahService{}

	newSvc.CahCardCollection = &CahCardCollection{}

	// set up the rng
	newSvc.RandomNG = rand.New(rand.NewSource(time.Now().UnixNano()))

	// download the CAH json file
	resp, err := http.Get(CAH_SOURCE_URL)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	json.Unmarshal(body, newSvc.CahCardCollection)

	return newSvc
}

func (svc *CahService) Handle(botRequest *server.BotRequest, botResponse *server.BotResponse) {

	strInput := parseInput(botRequest.RawLine.Text())
	if len(strInput) > 0 {
		botResponse.SetSingleLineResponse(svc.RandomCahMessageWithArgument(strInput))
	} else {
		botResponse.SetSingleLineResponse(svc.RandomCahMessage())
	}
}

func (svc *CahService) RandomCard() *CahCard {

	randVal := svc.RandomNG.Intn(svc.CahCardCollection.CardCount())
	return svc.CahCardCollection.GetCardAt(randVal)
}

func (svc *CahService) RandomQuestionCard() *CahCard {

	cCard := svc.RandomCard()
	for {
		if cCard.CardType == "Q" {
			break
		} else {
			cCard = svc.RandomCard()
		}
	}
	return cCard
}

func (svc *CahService) RandomOneAnswerQuestionCard() *CahCard {

	cCard := svc.RandomQuestionCard()
	for {
		if cCard.NumAnswers == 1 {
			break
		} else {
			cCard = svc.RandomQuestionCard()
		}
	}
	return cCard
}

func (svc *CahService) RandomAnswerCard() *CahCard {

	cCard := svc.RandomCard()
	for {
		if cCard.CardType == "A" {
			break
		} else {
			cCard = svc.RandomCard()
		}
	}
	return cCard
}

func (svc *CahService) MessageFromQuestionAndAnswers(questionStr string, answers []string) string {

	var finalStr string
	substrings := strings.Split(questionStr, "_")
	if len(substrings) < 2 {
		finalStr = questionStr + " " + convertToStandaloneAnswer(answers[0])
	} else {

		ansCounter := 0
		for _, value := range substrings {
			finalStr += value
			if ansCounter < len(answers) {
				finalStr += convertToInlineAnswer(answers[ansCounter])
				ansCounter++
			}
		}
	}

	return finalStr
}

func (svc *CahService) RandomCahMessage() string {

	qCard := svc.RandomQuestionCard()

	// find out how many answers we need
	numAnswers := qCard.NumAnswers

	// queue up all needed answer cards
	var answers = make([]string, numAnswers)
	for i := 0; i < int(numAnswers); i++ {
		answers[i] = svc.RandomAnswerCard().Text
	}

	return svc.MessageFromQuestionAndAnswers(qCard.Text, answers)
}

func (svc *CahService) RandomCahMessageWithArgument(argStr string) string {

	qCard := svc.RandomOneAnswerQuestionCard()

	// queue up all needed answer cards
	var answers = make([]string, 1)
	answers[0] = argStr

	return svc.MessageFromQuestionAndAnswers(qCard.Text, answers)
}

func (ccc CahCardCollection) CardCount() int {
	return len(ccc)
}

func (ccc CahCardCollection) GetCardAt(cardLoc int) *CahCard {
	return &CahCard{CardType: ccc[cardLoc].CardType, Id: ccc[cardLoc].Id, NumAnswers: ccc[cardLoc].NumAnswers, Text: ccc[cardLoc].Text}
}

func parseInput(input string) string {

	input = strings.TrimPrefix(input, "!cah")
	input = strings.Trim(input, " ")

	return input
}

func convertToInlineAnswer(orig string) string {

	runes := []rune(orig)
	runes[0] = unicode.ToLower(runes[0])
	orig = string(runes)
	return strings.TrimRight(orig, ".")
}

func convertToStandaloneAnswer(orig string) string {

	runes := []rune(orig)
	runes[0] = unicode.ToUpper(runes[0])

	// if no punctuation, add a period
	lastChar := runes[len(runes)-1]
	if lastChar == '.' || lastChar == '!' || lastChar == '?' {
		return string(runes)
	} else {
		return string(runes) + "."
	}
}
<commit_msg>re-pointing to new json location with removed html entities<commit_after>package cah

import (
	// "errors"
	"encoding/json"
	//"fmt"
	"github.com/gamelost/bot3server/server"
	"io/ioutil"
	// "log"
	"math/rand"
	"net/http"
	"strings"
	"time"
	"unicode"
)

// source URL for CAH card templates
const CAH_SOURCE_URL = "https://raw.githubusercontent.com/gamelost/bot3server/master/module/cah/cah-cards-standard.json"

type CahService struct {
	RandomNG          *rand.Rand
	CahCardCollection *CahCardCollection
}

type CahCard struct {
	Id         int64
	CardType   string
	NumAnswers int64
	Text       string
}

type CahCardCollection []struct {
	CardType   string `json:"cardType"`
	Expansion  string `json:"expansion"`
	Id         int64  `json:"id"`
	NumAnswers int64  `json:"numAnswers"`
	Text       string `json:"text"`
}

func (svc *CahService) NewService() server.BotHandler {

	var newSvc = &CahService{}

	newSvc.CahCardCollection = &CahCardCollection{}

	// set up the rng
	newSvc.RandomNG = rand.New(rand.NewSource(time.Now().UnixNano()))

	// download the CAH json file
	resp, err := http.Get(CAH_SOURCE_URL)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	json.Unmarshal(body, newSvc.CahCardCollection)

	return newSvc
}

func (svc *CahService) Handle(botRequest *server.BotRequest, botResponse *server.BotResponse) {

	strInput := parseInput(botRequest.RawLine.Text())
	if len(strInput) > 0 {
		botResponse.SetSingleLineResponse(svc.RandomCahMessageWithArgument(strInput))
	} else {
		botResponse.SetSingleLineResponse(svc.RandomCahMessage())
	}
}

func (svc *CahService) RandomCard() *CahCard {

	randVal := svc.RandomNG.Intn(svc.CahCardCollection.CardCount())
	return svc.CahCardCollection.GetCardAt(randVal)
}

func (svc *CahService) RandomQuestionCard() *CahCard {

	cCard := svc.RandomCard()
	for {
		if cCard.CardType == "Q" {
			break
		} else {
			cCard = svc.RandomCard()
		}
	}
	return cCard
}

func (svc *CahService) RandomOneAnswerQuestionCard() *CahCard {

	cCard := svc.RandomQuestionCard()
	for {
		if cCard.NumAnswers == 1 {
			break
		} else {
			cCard = svc.RandomQuestionCard()
		}
	}
	return cCard
}

func (svc *CahService) RandomAnswerCard() *CahCard {

	cCard := svc.RandomCard()
	for {
		if cCard.CardType == "A" {
			break
		} else {
			cCard = svc.RandomCard()
		}
	}
	return cCard
}

func (svc *CahService) MessageFromQuestionAndAnswers(questionStr string, answers []string) string {

	var finalStr string
	substrings := strings.Split(questionStr, "_")
	if len(substrings) < 2 {
		finalStr = questionStr + " " + convertToStandaloneAnswer(answers[0])
	} else {

		ansCounter := 0
		for _, value := range substrings {
			finalStr += value
			if ansCounter < len(answers) {
				finalStr += convertToInlineAnswer(answers[ansCounter])
				ansCounter++
			}
		}
	}

	return finalStr
}

func (svc *CahService) RandomCahMessage() string {

	qCard := svc.RandomQuestionCard()

	// find out how many answers we need
	numAnswers := qCard.NumAnswers

	// queue up all needed answer cards
	var answers = make([]string, numAnswers)
	for i := 0; i < int(numAnswers); i++ {
		answers[i] = svc.RandomAnswerCard().Text
	}

	return svc.MessageFromQuestionAndAnswers(qCard.Text, answers)
}

func (svc *CahService) RandomCahMessageWithArgument(argStr string) string {

	qCard := svc.RandomOneAnswerQuestionCard()

	// queue up all needed answer cards
	var answers = make([]string, 1)
	answers[0] = argStr

	return svc.MessageFromQuestionAndAnswers(qCard.Text, answers)
}

func (ccc CahCardCollection) CardCount() int {
	return len(ccc)
}

func (ccc CahCardCollection) GetCardAt(cardLoc int) *CahCard {
	return &CahCard{CardType: ccc[cardLoc].CardType, Id: ccc[cardLoc].Id, NumAnswers: ccc[cardLoc].NumAnswers, Text: ccc[cardLoc].Text}
}

func parseInput(input string) string {

	input = strings.TrimPrefix(input, "!cah")
	input = strings.Trim(input, " ")

	return input
}

func convertToInlineAnswer(orig string) string {

	runes := []rune(orig)
	runes[0] = unicode.ToLower(runes[0])
	orig = string(runes)
	return strings.TrimRight(orig, ".")
}

func convertToStandaloneAnswer(orig string) string {

	runes := []rune(orig)
	runes[0] = unicode.ToUpper(runes[0])

	// if no punctuation, add a period
	lastChar := runes[len(runes)-1]
	if lastChar == '.' || lastChar == '!' || lastChar == '?' {
		return string(runes)
	} else {
		return string(runes) + "."
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io"
	"koding/kontrol/kontrolproxy/proxyconfig"
	"koding/tools/db"
	"koding/virt"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
)

type UserInfo struct {
	Domain       *proxyconfig.Domain
	IP           string
	Country      string
	Target       *url.URL
	Redirect     bool
	LoadBalancer *proxyconfig.LoadBalancer
}

func NewUserInfo(domain *proxyconfig.Domain) *UserInfo {
	return &UserInfo{
		Domain: domain,
	}
}

func populateUser(outreq *http.Request) (*UserInfo, io.Reader, error) {
	user, err := parseDomain(outreq.Host)
	if err != nil {
		return nil, nil, err
	}

	user.IP, _, err = net.SplitHostPort(outreq.RemoteAddr)
	if err == nil {
		if geoIP != nil {
			loc := geoIP.GetLocationByIP(user.IP)
			if loc != nil {
				user.Country = loc.CountryName
			}
		}
	}

	buf, err := user.populateTarget()
	if err != nil {
		return nil, buf, err
	}

	fmt.Printf("--\nmode '%s'\t: %s %s\n", user.Domain.Proxy.Mode, user.IP, user.Country)
	return user, buf, nil
}

func parseDomain(host string) (*UserInfo, error) {
	// remove www from the hostname (i.e. www.foo.com -> foo.com)
	if strings.HasPrefix(host, "www.") {
		host = strings.TrimPrefix(host, "www.")
	}

	// Then make a lookup for domains
	domain, err := proxyDB.GetDomain(host)
	if err != nil {
		if err != mgo.ErrNotFound {
			return &UserInfo{}, fmt.Errorf("domain lookup error '%s'", err)
		}

		// lookup didn't found anything, move on to .x.koding.com domains
		if strings.HasSuffix(host, "x.koding.com") {
			// hostsin form {name}-{key}.kd.io or {name}-{key}.x.koding.com is used by koding
			subdomain := strings.TrimSuffix(host, ".x.koding.com")
			servicename := strings.Split(subdomain, "-")[0]
			key := strings.Split(subdomain, "-")[1]

			domain := proxyconfig.NewDomain(host, "internal", "koding", servicename, key, "", []string{})
			return NewUserInfo(domain), nil
		}

		return &UserInfo{}, fmt.Errorf("domain %s is unknown.", host)
	}

	return NewUserInfo(&domain), nil
}

func (u *UserInfo) populateTarget() (io.Reader, error) {
	var err error
	var hostname string

	switch u.Domain.Proxy.Mode {
	case "maintenance":
		buf, err := executeTemplate("maintenance.html", nil)
		if err != nil {
			return nil, err
		}
		return buf, nil
	case "redirect":
		fullurl := u.Domain.Proxy.FullUrl
		if !strings.HasPrefix(fullurl, "http://") && !strings.HasPrefix(fullurl, "https://") {
			fullurl = "https://" + fullurl
		}

		u.Target, err = url.Parse(fullurl)
		if err != nil {
			return nil, err
		}
		return nil, nil
	case "vm":
		switch u.Domain.LoadBalancer.Mode {
		case "roundrobin": // equal weights
			N := float64(len(u.Domain.HostnameAlias))
			n := int(math.Mod(float64(u.Domain.LoadBalancer.Index+1), N))
			hostname = u.Domain.HostnameAlias[n]

			u.Domain.LoadBalancer.Index = n
			go proxyDB.UpdateDomain(u.Domain)
		case "sticky":
			hostname = u.Domain.HostnameAlias[u.Domain.LoadBalancer.Index]
		case "random":
			randomIndex := rand.Intn(len(u.Domain.HostnameAlias) - 1)
			hostname = u.Domain.HostnameAlias[randomIndex]
		default:
			hostname = u.Domain.HostnameAlias[0]
		}

		var vm virt.VM
		if err := db.VMs.Find(bson.M{"hostnameAlias": hostname}).One(&vm); err != nil {
			buf, err := executeTemplate("notfound.html", hostname)
			if err != nil {
				return nil, err
			}

			return buf, errors.New("vm not found")
		}
		if vm.IP == nil {
			buf, err := executeTemplate("notactiveVM.html", hostname)
			if err != nil {
				return nil, err
			}

			return buf, errors.New("vm not active")
		}

		vmAddr := vm.IP.String()
		if !hasPort(vmAddr) {
			vmAddr = addPort(vmAddr, "80")
		}

		err := checkServer(vmAddr)
		if err != nil {
			buf, errTemp := executeTemplate("notactiveVM.html", hostname)
			if errTemp != nil {
				return nil, errTemp
			}
			return buf, fmt.Errorf("vm is down: '%s'", err)
		}

		u.Target, err = url.Parse("http://" + vmAddr)
		if err != nil {
			return nil, err
		}
		u.LoadBalancer = &u.Domain.LoadBalancer

		return nil, nil
	case "internal":
		username := u.Domain.Proxy.Username
		servicename := u.Domain.Proxy.Servicename
		key := u.Domain.Proxy.Key

		keyData, err := proxyDB.GetKey(username, servicename, key)
		if err != nil {
			return nil, fmt.Errorf("no keyData for username '%s', servicename '%s' and key '%s'", username, servicename, key)
		}

		switch keyData.LoadBalancer.Mode {
		case "roundrobin":
			N := float64(len(keyData.Host))
			n := int(math.Mod(float64(keyData.LoadBalancer.Index+1), N))
			hostname = keyData.Host[n]

			keyData.LoadBalancer.Index = n
			go proxyDB.UpdateKeyData(username, servicename, keyData)
		case "sticky":
			hostname = keyData.Host[keyData.LoadBalancer.Index]
		case "random":
			randomIndex := rand.Intn(len(keyData.Host) - 1)
			hostname = keyData.Host[randomIndex]
		default:
			hostname = keyData.Host[0]
		}

		if servicename == "broker" {
			u.Redirect = true
			hostname = "https://" + hostname
		} else {
			hostname = "http://" + hostname
		}

		u.Target, err = url.Parse(hostname)
		if err != nil {
			return nil, err
		}
		u.LoadBalancer = &keyData.LoadBalancer
		return nil, nil
	default:
		return nil, fmt.Errorf("ERROR: proxy mode is not supported: %s", u.Domain.Proxy.Mode)
	}

	return nil, nil
}

func validate(u *UserInfo) (bool, error) {
	// restrictionId, err := proxyDB.GetDomainRestrictionId(u.Domain.Id)
	// if err != nil {
	// 	return true, nil //don't block if we don't get a rule (pre-caution))
	// }

	restriction, err := proxyDB.GetRestrictionByDomain(u.Domain.Domain)
	if err != nil {
		return true, nil //don't block if we don't get a rule (pre-caution))
	}

	// restriction, err := proxyDB.GetRestrictionByID(restrictionId)
	// if err != nil {
	// 	return true, nil //don't block if we don't get a rule (pre-caution))
	// }

	return validator(restriction, u).AddRules().Check()
}

func isWebsocket(req *http.Request) bool {
	conn_hdr := ""
	conn_hdrs := req.Header["Connection"]
	if len(conn_hdrs) > 0 {
		conn_hdr = conn_hdrs[0]
	}

	upgrade_websocket := false
	if strings.ToLower(conn_hdr) == "upgrade" {
		upgrade_hdrs := req.Header["Upgrade"]
		if len(upgrade_hdrs) > 0 {
			upgrade_websocket = (strings.ToLower(upgrade_hdrs[0]) == "websocket")
		}
	}

	return upgrade_websocket
}

func logDomainRequests(domain string) {
	if domain == "" {
		return
	}

	err := proxyDB.AddDomainRequests(domain)
	if err != nil {
		fmt.Printf("could not add domain statistisitcs for %s\n", err.Error())
	}
}

func logProxyStat(name, country string) {
	err := proxyDB.AddProxyStat(name, country)
	if err != nil {
		fmt.Printf("could not add proxy statistisitcs for %s\n", err.Error())
	}
}

func logDomainDenied(domain, ip, country, reason string) {
	if domain == "" {
		return
	}

	err := proxyDB.AddDomainDenied(domain, ip, country, reason)
	if err != nil {
		fmt.Printf("could not add domain statistisitcs for %s\n", err.Error())
	}
}
<commit_msg>kontrolproxy: remove redirect for broker<commit_after>package main

import (
	"errors"
	"fmt"
	"io"
	"koding/kontrol/kontrolproxy/proxyconfig"
	"koding/tools/db"
	"koding/virt"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
)

type UserInfo struct {
	Domain       *proxyconfig.Domain
	IP           string
	Country      string
	Target       *url.URL
	Redirect     bool
	LoadBalancer *proxyconfig.LoadBalancer
}

func NewUserInfo(domain *proxyconfig.Domain) *UserInfo {
	return &UserInfo{
		Domain: domain,
	}
}

func populateUser(outreq *http.Request) (*UserInfo, io.Reader, error) {
	user, err := parseDomain(outreq.Host)
	if err != nil {
		return nil, nil, err
	}

	user.IP, _, err = net.SplitHostPort(outreq.RemoteAddr)
	if err == nil {
		if geoIP != nil {
			loc := geoIP.GetLocationByIP(user.IP)
			if loc != nil {
				user.Country = loc.CountryName
			}
		}
	}

	buf, err := user.populateTarget()
	if err != nil {
		return nil, buf, err
	}

	fmt.Printf("--\nmode '%s'\t: %s %s\n", user.Domain.Proxy.Mode, user.IP, user.Country)
	return user, buf, nil
}

func parseDomain(host string) (*UserInfo, error) {
	// remove www from the hostname (i.e. www.foo.com -> foo.com)
	if strings.HasPrefix(host, "www.") {
		host = strings.TrimPrefix(host, "www.")
	}

	// Then make a lookup for domains
	domain, err := proxyDB.GetDomain(host)
	if err != nil {
		if err != mgo.ErrNotFound {
			return &UserInfo{}, fmt.Errorf("domain lookup error '%s'", err)
		}

		// lookup didn't found anything, move on to .x.koding.com domains
		if strings.HasSuffix(host, "x.koding.com") {
			// hostsin form {name}-{key}.kd.io or {name}-{key}.x.koding.com is used by koding
			subdomain := strings.TrimSuffix(host, ".x.koding.com")
			servicename := strings.Split(subdomain, "-")[0]
			key := strings.Split(subdomain, "-")[1]

			domain := proxyconfig.NewDomain(host, "internal", "koding", servicename, key, "", []string{})
			return NewUserInfo(domain), nil
		}

		return &UserInfo{}, fmt.Errorf("domain %s is unknown.", host)
	}

	return NewUserInfo(&domain), nil
}

func (u *UserInfo) populateTarget() (io.Reader, error) {
	var err error
	var hostname string

	switch u.Domain.Proxy.Mode {
	case "maintenance":
		buf, err := executeTemplate("maintenance.html", nil)
		if err != nil {
			return nil, err
		}
		return buf, nil
	case "redirect":
		fullurl := u.Domain.Proxy.FullUrl
		if !strings.HasPrefix(fullurl, "http://") && !strings.HasPrefix(fullurl, "https://") {
			fullurl = "https://" + fullurl
		}

		u.Target, err = url.Parse(fullurl)
		if err != nil {
			return nil, err
		}
		return nil, nil
	case "vm":
		switch u.Domain.LoadBalancer.Mode {
		case "roundrobin": // equal weights
			N := float64(len(u.Domain.HostnameAlias))
			n := int(math.Mod(float64(u.Domain.LoadBalancer.Index+1), N))
			hostname = u.Domain.HostnameAlias[n]

			u.Domain.LoadBalancer.Index = n
			go proxyDB.UpdateDomain(u.Domain)
		case "sticky":
			hostname = u.Domain.HostnameAlias[u.Domain.LoadBalancer.Index]
		case "random":
			randomIndex := rand.Intn(len(u.Domain.HostnameAlias) - 1)
			hostname = u.Domain.HostnameAlias[randomIndex]
		default:
			hostname = u.Domain.HostnameAlias[0]
		}

		var vm virt.VM
		if err := db.VMs.Find(bson.M{"hostnameAlias": hostname}).One(&vm); err != nil {
			buf, err := executeTemplate("notfound.html", hostname)
			if err != nil {
				return nil, err
			}

			return buf, errors.New("vm not found")
		}
		if vm.IP == nil {
			buf, err := executeTemplate("notactiveVM.html", hostname)
			if err != nil {
				return nil, err
			}

			return buf, errors.New("vm not active")
		}

		vmAddr := vm.IP.String()
		if !hasPort(vmAddr) {
			vmAddr = addPort(vmAddr, "80")
		}

		err := checkServer(vmAddr)
		if err != nil {
			buf, errTemp := executeTemplate("notactiveVM.html", hostname)
			if errTemp != nil {
				return nil, errTemp
			}
			return buf, fmt.Errorf("vm is down: '%s'", err)
		}

		u.Target, err = url.Parse("http://" + vmAddr)
		if err != nil {
			return nil, err
		}
		u.LoadBalancer = &u.Domain.LoadBalancer

		return nil, nil
	case "internal":
		username := u.Domain.Proxy.Username
		servicename := u.Domain.Proxy.Servicename
		key := u.Domain.Proxy.Key

		keyData, err := proxyDB.GetKey(username, servicename, key)
		if err != nil {
			return nil, fmt.Errorf("no keyData for username '%s', servicename '%s' and key '%s'", username, servicename, key)
		}

		switch keyData.LoadBalancer.Mode {
		case "roundrobin":
			N := float64(len(keyData.Host))
			n := int(math.Mod(float64(keyData.LoadBalancer.Index+1), N))
			hostname = keyData.Host[n]

			keyData.LoadBalancer.Index = n
			go proxyDB.UpdateKeyData(username, servicename, keyData)
		case "sticky":
			hostname = keyData.Host[keyData.LoadBalancer.Index]
		case "random":
			randomIndex := rand.Intn(len(keyData.Host) - 1)
			hostname = keyData.Host[randomIndex]
		default:
			hostname = keyData.Host[0]
		}

		if !strings.HasPrefix(hostname, "http://") {
			hostname = "http://" + hostname
		}

		u.Target, err = url.Parse(hostname)
		if err != nil {
			return nil, err
		}
		u.LoadBalancer = &keyData.LoadBalancer
		return nil, nil
	default:
		return nil, fmt.Errorf("ERROR: proxy mode is not supported: %s", u.Domain.Proxy.Mode)
	}

	return nil, nil
}

func validate(u *UserInfo) (bool, error) {
	// restrictionId, err := proxyDB.GetDomainRestrictionId(u.Domain.Id)
	// if err != nil {
	// 	return true, nil //don't block if we don't get a rule (pre-caution))
	// }

	restriction, err := proxyDB.GetRestrictionByDomain(u.Domain.Domain)
	if err != nil {
		return true, nil //don't block if we don't get a rule (pre-caution))
	}

	// restriction, err := proxyDB.GetRestrictionByID(restrictionId)
	// if err != nil {
	// 	return true, nil //don't block if we don't get a rule (pre-caution))
	// }

	return validator(restriction, u).AddRules().Check()
}

func isWebsocket(req *http.Request) bool {
	conn_hdr := ""
	conn_hdrs := req.Header["Connection"]
	if len(conn_hdrs) > 0 {
		conn_hdr = conn_hdrs[0]
	}

	upgrade_websocket := false
	if strings.ToLower(conn_hdr) == "upgrade" {
		upgrade_hdrs := req.Header["Upgrade"]
		if len(upgrade_hdrs) > 0 {
			upgrade_websocket = (strings.ToLower(upgrade_hdrs[0]) == "websocket")
		}
	}

	return upgrade_websocket
}

func logDomainRequests(domain string) {
	if domain == "" {
		return
	}

	err := proxyDB.AddDomainRequests(domain)
	if err != nil {
		fmt.Printf("could not add domain statistisitcs for %s\n", err.Error())
	}
}

func logProxyStat(name, country string) {
	err := proxyDB.AddProxyStat(name, country)
	if err != nil {
		fmt.Printf("could not add proxy statistisitcs for %s\n", err.Error())
	}
}

func logDomainDenied(domain, ip, country, reason string) {
	if domain == "" {
		return
	}

	err := proxyDB.AddDomainDenied(domain, ip, country, reason)
	if err != nil {
		fmt.Printf("could not add domain statistisitcs for %s\n", err.Error())
	}
}
<|endoftext|>