<commit_before>package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"github.com/google/go-github/v28/github"
	"golang.org/x/oauth2"
)

type (
	Repo struct {
		Owner string
		Name  string
	}

	Build struct {
		Event string
	}

	Commit struct {
		Ref string
	}

	Config struct {
		APIKey          string
		Files           []string
		FileExists      string
		Checksum        []string
		ChecksumFile    string
		ChecksumFlatten bool
		Draft           bool
		Prerelease      bool
		BaseURL         string
		UploadURL       string
		Title           string
		Note            string
		Overwrite       bool
	}

	Plugin struct {
		Repo   Repo
		Build  Build
		Commit Commit
		Config Config
	}
)

func (p Plugin) Exec() error {
	var (
		files []string
	)

	if p.Build.Event != "tag" {
		return fmt.Errorf("The GitHub Release plugin is only available for tags")
	}

	if p.Config.APIKey == "" {
		return fmt.Errorf("You must provide an API key")
	}

	if !fileExistsValues[p.Config.FileExists] {
		return fmt.Errorf("Invalid value for file_exists")
	}

	if !strings.HasSuffix(p.Config.BaseURL, "/") {
		p.Config.BaseURL = p.Config.BaseURL + "/"
	}

	if !strings.HasSuffix(p.Config.UploadURL, "/") {
		p.Config.UploadURL = p.Config.UploadURL + "/"
	}

	var err error
	if p.Config.Note != "" {
		if p.Config.Note, err = readStringOrFile(p.Config.Note); err != nil {
			return fmt.Errorf("error while reading %s: %v", p.Config.Note, err)
		}
	}

	if p.Config.Title != "" {
		if p.Config.Title, err = readStringOrFile(p.Config.Title); err != nil {
			return fmt.Errorf("error while reading %s: %v", p.Config.Note, err)
		}
	}

	for _, glob := range p.Config.Files {
		globed, err := filepath.Glob(glob)

		if err != nil {
			return fmt.Errorf("Failed to glob %s. %s", glob, err)
		}

		if globed != nil {
			files = append(files, globed...)
		}
	}

	if len(p.Config.Checksum) > 0 {
		var (
			err error
		)

		files, err = writeChecksums(files, p.Config.Checksum, p.Config.ChecksumFile, p.Config.ChecksumFlatten)

		if err != nil {
			return fmt.Errorf("Failed to write checksums. %s", err)
		}
	}

	baseURL, err := url.Parse(p.Config.BaseURL)

	if err != nil {
		return fmt.Errorf("Failed to parse base URL. %s", err)
	}

	uploadURL, err := url.Parse(p.Config.UploadURL)

	if err != nil {
		return fmt.Errorf("Failed to parse upload URL. %s", err)
	}

	ctx := context.Background()
	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: p.Config.APIKey})
	tc := oauth2.NewClient(ctx, ts)

	client := github.NewClient(tc)

	client.BaseURL = baseURL
	client.UploadURL = uploadURL

	rc := releaseClient{
		Client:     client,
		Context:    ctx,
		Owner:      p.Repo.Owner,
		Repo:       p.Repo.Name,
		Tag:        strings.TrimPrefix(p.Commit.Ref, "refs/tags/"),
		Draft:      p.Config.Draft,
		Prerelease: p.Config.Prerelease,
		FileExists: p.Config.FileExists,
		Title:      p.Config.Title,
		Note:       p.Config.Note,
		Overwrite:  p.Config.Overwrite,
	}

	release, err := rc.buildRelease()

	if err != nil {
		return fmt.Errorf("Failed to create the release. %s", err)
	}

	if err := rc.uploadFiles(*release.ID, files); err != nil {
		return fmt.Errorf("Failed to upload the files. %s", err)
	}

	return nil
}

func readStringOrFile(input string) (string, error) {
	if len(input) > 255 {
		return input, nil
	}
	// Check if input is a file path
	if _, err := os.Stat(input); err != nil && os.IsNotExist(err) {
		// No file found => use input as result
		return input, nil
	} else if err != nil {
		return "", err
	}
	result, err := ioutil.ReadFile(input)
	if err != nil {
		return "", err
	}
	return string(result), nil
}
<commit_msg>Fail if no file detected<commit_after>package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"github.com/google/go-github/v28/github"
	"golang.org/x/oauth2"
)

type (
	Repo struct {
		Owner string
		Name  string
	}

	Build struct {
		Event string
	}

	Commit struct {
		Ref string
	}

	Config struct {
		APIKey          string
		Files           []string
		FileExists      string
		Checksum        []string
		ChecksumFile    string
		ChecksumFlatten bool
		Draft           bool
		Prerelease      bool
		BaseURL         string
		UploadURL       string
		Title           string
		Note            string
		Overwrite       bool
	}

	Plugin struct {
		Repo   Repo
		Build  Build
		Commit Commit
		Config Config
	}
)

func (p Plugin) Exec() error {
	var (
		files []string
	)

	if p.Build.Event != "tag" {
		return fmt.Errorf("The GitHub Release plugin is only available for tags")
	}

	if p.Config.APIKey == "" {
		return fmt.Errorf("You must provide an API key")
	}

	if !fileExistsValues[p.Config.FileExists] {
		return fmt.Errorf("Invalid value for file_exists")
	}

	if !strings.HasSuffix(p.Config.BaseURL, "/") {
		p.Config.BaseURL = p.Config.BaseURL + "/"
	}

	if !strings.HasSuffix(p.Config.UploadURL, "/") {
		p.Config.UploadURL = p.Config.UploadURL + "/"
	}

	var err error
	if p.Config.Note != "" {
		if p.Config.Note, err = readStringOrFile(p.Config.Note); err != nil {
			return fmt.Errorf("error while reading %s: %v", p.Config.Note, err)
		}
	}

	if p.Config.Title != "" {
		if p.Config.Title, err = readStringOrFile(p.Config.Title); err != nil {
			return fmt.Errorf("error while reading %s: %v", p.Config.Note, err)
		}
	}

	for _, glob := range p.Config.Files {
		globed, err := filepath.Glob(glob)

		if err != nil {
			return fmt.Errorf("Failed to glob %s. %s", glob, err)
		}

		if globed != nil {
			files = append(files, globed...)
		}
	}

	if len(p.Config.Files) > 0 && len(files) < 1 {
		return fmt.Errorf("Failed to find any file to release")
	}

	if len(p.Config.Checksum) > 0 {
		var (
			err error
		)

		files, err = writeChecksums(files, p.Config.Checksum, p.Config.ChecksumFile, p.Config.ChecksumFlatten)

		if err != nil {
			return fmt.Errorf("Failed to write checksums. %s", err)
		}
	}

	baseURL, err := url.Parse(p.Config.BaseURL)

	if err != nil {
		return fmt.Errorf("Failed to parse base URL. %s", err)
	}

	uploadURL, err := url.Parse(p.Config.UploadURL)

	if err != nil {
		return fmt.Errorf("Failed to parse upload URL. %s", err)
	}

	ctx := context.Background()
	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: p.Config.APIKey})
	tc := oauth2.NewClient(ctx, ts)

	client := github.NewClient(tc)

	client.BaseURL = baseURL
	client.UploadURL = uploadURL

	rc := releaseClient{
		Client:     client,
		Context:    ctx,
		Owner:      p.Repo.Owner,
		Repo:       p.Repo.Name,
		Tag:        strings.TrimPrefix(p.Commit.Ref, "refs/tags/"),
		Draft:      p.Config.Draft,
		Prerelease: p.Config.Prerelease,
		FileExists: p.Config.FileExists,
		Title:      p.Config.Title,
		Note:       p.Config.Note,
		Overwrite:  p.Config.Overwrite,
	}

	release, err := rc.buildRelease()

	if err != nil {
		return fmt.Errorf("Failed to create the release. %s", err)
	}

	if err := rc.uploadFiles(*release.ID, files); err != nil {
		return fmt.Errorf("Failed to upload the files. %s", err)
	}

	return nil
}

func readStringOrFile(input string) (string, error) {
	if len(input) > 255 {
		return input, nil
	}
	// Check if input is a file path
	if _, err := os.Stat(input); err != nil && os.IsNotExist(err) {
		// No file found => use input as result
		return input, nil
	} else if err != nil {
		return "", err
	}
	result, err := ioutil.ReadFile(input)
	if err != nil {
		return "", err
	}
	return string(result), nil
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package s3util_test

import (
	"github.com/jacobsa/aws/s3/mock"
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestKeys(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type ListAllKeysTest struct {
	bucket   mock_s3.MockBucket

	keys []string
	err error
}

func init() { RegisterTestSuite(&ListAllKeysTest{}) }

func (t *ListAllKeysTest) SetUp(i *TestInfo) {
	t.bucket = mock_s3.NewMockBucket(i.MockController, "bucket")
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ListAllKeysTest) DoesFoo() {
	ExpectEq("TODO", "")
}
<commit_msg>Added tests.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package s3util_test

import (
	"errors"
	"github.com/jacobsa/aws/s3/mock"
	"github.com/jacobsa/aws/s3/s3util"
	. "github.com/jacobsa/oglematchers"
	"github.com/jacobsa/oglemock"
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestKeys(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type ListAllKeysTest struct {
	bucket   mock_s3.MockBucket

	keys []string
	err error
}

func init() { RegisterTestSuite(&ListAllKeysTest{}) }

func (t *ListAllKeysTest) SetUp(i *TestInfo) {
	t.bucket = mock_s3.NewMockBucket(i.MockController, "bucket")
}

func (t *ListAllKeysTest) call() {
	t.keys, t.err = s3util.ListAllKeys(t.bucket)
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ListAllKeysTest) CallsListKeyRepeatedly() {
	// ListKeys (call 0)
	keys0 := []string{"burrito", "enchilada"}

	ExpectCall(t.bucket, "ListKeys")("").
		WillOnce(oglemock.Return(keys0, nil))

	// ListKeys (call 1)
	keys1 := []string{"queso", "taco"}

	ExpectCall(t.bucket, "ListKeys")("enchilada").
		WillOnce(oglemock.Return(keys1, nil))

	// ListKeys (call 2)
	ExpectCall(t.bucket, "ListKeys")("taco").
		WillOnce(oglemock.Return(nil, errors.New("")))

	// Call
	t.call()
}

func (t *ListAllKeysTest) ListKeyReturnsError() {
	// ListKeys
	ExpectCall(t.bucket, "ListKeys")(Any()).
		WillOnce(oglemock.Return([]string{"a"}, nil)).
		WillOnce(oglemock.Return(nil, errors.New("taco")))

	// Call
	t.call()

	ExpectThat(t.err, Error(HasSubstr("ListKeys")))
	ExpectThat(t.err, Error(HasSubstr("taco")))
}

func (t *ListAllKeysTest) ListKeyReturnsNoKeys() {
	// ListKeys
	ExpectCall(t.bucket, "ListKeys")(Any()).
		WillOnce(oglemock.Return([]string{}, nil))

	// Call
	t.call()
	AssertEq(nil, t.err)

	ExpectThat(t.keys, ElementsAre())
}

func (t *ListAllKeysTest) ListKeyReturnsSomeKeys() {
	// ListKeys
	ExpectCall(t.bucket, "ListKeys")(Any()).
		WillOnce(oglemock.Return([]string{"burrito", "enchilada"}, nil)).
		WillOnce(oglemock.Return([]string{"taco"}, nil)).
		WillOnce(oglemock.Return([]string{}, nil))

	// Call
	t.call()
	AssertEq(nil, t.err)

	ExpectThat(
		t.keys,
		ElementsAre(
			"burrito",
			"enchilada",
			"taco",
		),
	)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	crand "crypto/rand"
	"errors"
	"flag"
	"fmt"
	"io"
	"math/rand"
	"os"
	"strings"
)

func NewRand() *rand.Rand {
	var b [8]byte
	_, err := io.ReadFull(crand.Reader, b[:])
	if err != nil {
		panic(err)
	}
	seed := int64(b[0]) | int64(b[1])<<8 | int64(b[2])<<16 | int64(b[3])<<24 |
		int64(b[4])<<32 | int64(b[5])<<40 | int64(b[6])<<48 | int64(b[7])<<56
	return rand.New(rand.NewSource(seed))
}

var (
	ErrPassLength      = errors.New("Length of password must be greater than 1")
	ErrPassAlphaLength = errors.New("Length of password must be greater than or " +
		"equal to number of required characters")
	ErrAlphaLength = errors.New("Alphabet must not be empty")
)

func NewPassword(length int, alphabets ...string) (string, error) {
	alphabet := strings.Join(alphabets, "")

	if length < 1 {

		return "", ErrPassLength
	}

	if length < len(alphabets) {
		return "", ErrPassAlphaLength
	}

	if alphabet == "" {
		return "", ErrAlphaLength
	}

	pass := make([]byte, 0, length)
	r := NewRand()

	// Loop until the generated password has required characteristics
loop:
	for i := 0; i < cap(pass); i++ {
		char := alphabet[r.Intn(len(alphabet))]
		pass = append(pass, char)
	}

	for _, subalpha := range alphabets {
		if !bytes.ContainsAny(pass, subalpha) {
			pass = pass[:0]
			goto loop
		}
	}

	return string(pass), nil
}

func main() {
	const (
		upper           = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
		lower           = "abcdefghijklmnopqrstuvwxyz"
		digit           = "0123456789"
		defaultAlphabet = "upper lower digit"
	)

	length := flag.Int("length", 8, "length of password to generate")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage of %s [opts] [alphabet]:

	Creates a password by randomly selecting characters from its alphabet.

	Alphabet is a space separated list of character classes to use.
	At least one character in each class will be output.
	Character classes are either literal sets (like "abc" and "123") or the
	special names "upper", "lower", "digit", and "default".

	Default alphabet is %q.

`, os.Args[0], defaultAlphabet)
		flag.PrintDefaults()
	}
	flag.Parse()

	alpha := flag.Arg(0)
	if alpha == "" {
		alpha = defaultAlphabet
	}
	alpha = strings.Replace(alpha, "default", defaultAlphabet, -1)
	alpha = strings.Replace(alpha, "upper", upper, -1)
	alpha = strings.Replace(alpha, "lower", lower, -1)
	alpha = strings.Replace(alpha, "digits", digit, -1)
	alpha = strings.Replace(alpha, "digit", digit, -1)

	alphas := strings.Split(alpha, " ")

	if pass, err := NewPassword(*length, alphas...); err == nil {
		fmt.Println(pass)
	} else {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(2)
	}
}
<commit_msg>NewPassword: Use a timeout, not an infinite loop<commit_after>package main

import (
	"bytes"
	crand "crypto/rand"
	"errors"
	"flag"
	"fmt"
	"io"
	"math/rand"
	"os"
	"strings"
	"time"
)

func NewRand() *rand.Rand {
	var b [8]byte
	_, err := io.ReadFull(crand.Reader, b[:])
	if err != nil {
		panic(err)
	}
	seed := int64(b[0]) | int64(b[1])<<8 | int64(b[2])<<16 | int64(b[3])<<24 |
		int64(b[4])<<32 | int64(b[5])<<40 | int64(b[6])<<48 | int64(b[7])<<56
	return rand.New(rand.NewSource(seed))
}

const deadline = 500 * time.Millisecond

var (
	ErrPassLength      = errors.New("Length of password must be greater than 1")
	ErrPassAlphaLength = errors.New("Length of password must be greater than or " +
		"equal to number of required characters")
	ErrAlphaLength = errors.New("Alphabet must not be empty")
	ErrTimeOut     = errors.New("Could not find a matching password in " + deadline.String())
)

func NewPassword(length int, alphabets ...string) (string, error) {
	alphabet := strings.Join(alphabets, "")

	if length < 1 {

		return "", ErrPassLength
	}

	if length < len(alphabets) {
		return "", ErrPassAlphaLength
	}

	if alphabet == "" {
		return "", ErrAlphaLength
	}

	pass := make([]byte, 0, length)
	r := NewRand()

	// Loop until the generated password has required characteristics
	// or time runs out
	start := time.Now()
	for time.Since(start) < deadline {

		for i := 0; i < cap(pass); i++ {
			char := alphabet[r.Intn(len(alphabet))]
			pass = append(pass, char)
		}

		missingAlphabet := false
		for _, subalpha := range alphabets {
			if !bytes.ContainsAny(pass, subalpha) {
				pass = pass[:0]
				missingAlphabet = true
				break
			}
		}

		if !missingAlphabet {
			return string(pass), nil
		}
	}
	return "", ErrTimeOut
}

func main() {
	const (
		upper           = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
		lower           = "abcdefghijklmnopqrstuvwxyz"
		digit           = "0123456789"
		defaultAlphabet = "upper lower digit"
	)

	length := flag.Int("length", 8, "length of password to generate")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage of %s [opts] [alphabet]:

	Creates a password by randomly selecting characters from its alphabet.

	Alphabet is a space separated list of character classes to use.
	At least one character in each class will be output.
	Character classes are either literal sets (like "abc" and "123") or the
	special names "upper", "lower", "digit", and "default".

	Default alphabet is %q.

`, os.Args[0], defaultAlphabet)
		flag.PrintDefaults()
	}
	flag.Parse()

	alpha := flag.Arg(0)
	if alpha == "" {
		alpha = defaultAlphabet
	}
	alpha = strings.Replace(alpha, "default", defaultAlphabet, -1)
	alpha = strings.Replace(alpha, "upper", upper, -1)
	alpha = strings.Replace(alpha, "lower", lower, -1)
	alpha = strings.Replace(alpha, "digits", digit, -1)
	alpha = strings.Replace(alpha, "digit", digit, -1)

	alphas := strings.Split(alpha, " ")

	if pass, err := NewPassword(*length, alphas...); err == nil {
		fmt.Println(pass)
	} else {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(2)
	}
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gles

import (
	"context"
	"fmt"

	"github.com/google/gapid/core/log"
	"github.com/google/gapid/core/os/device"
	"github.com/google/gapid/core/os/device/bind"
	"github.com/google/gapid/gapis/api"
	"github.com/google/gapid/gapis/capture"
	"github.com/google/gapid/gapis/database"
	"github.com/google/gapid/gapis/memory"
	rb "github.com/google/gapid/gapis/replay/builder"
	"github.com/google/gapid/gapis/resolve"
	"github.com/google/gapid/gapis/service/path"
	"github.com/google/gapid/gapis/stringtable"
	"github.com/pkg/errors"
)

type externs struct {
	ctx   context.Context // Allowed because the externs struct is only a parameter proxy for a single call
	cmd   api.Cmd
	cmdID api.CmdID
	s     *api.GlobalState
	b     *rb.Builder
}

func (e externs) mapMemory(slice memory.Slice) {
	ctx := e.ctx
	if b := e.b; b != nil {
		switch e.cmd.(type) {
		case *GlMapBufferRange, *GlMapBufferRangeEXT, *GlMapBufferOES, *GlMapBuffer:
			// Base address is on the stack.
			b.MapMemory(memory.Range{Base: slice.Base(), Size: slice.Size()})

		default:
			log.E(ctx, "mapBuffer extern called for unsupported command: %v", e.cmd)
		}
	}
}

func (e externs) unmapMemory(slice memory.Slice) {
	if b := e.b; b != nil {
		b.UnmapMemory(memory.Range{Base: slice.Base(), Size: slice.Size()})
	}
}

func (e externs) GetEGLStaticContextState(EGLDisplay, EGLContext) *StaticContextState {
	return FindStaticContextState(e.cmd.Extras())
}

func (e externs) GetEGLDynamicContextState(EGLDisplay, EGLSurface, EGLContext) *DynamicContextState {
	return FindDynamicContextState(e.cmd.Extras())
}

func (e externs) GetAndroidNativeBufferExtra(Voidᵖ) *AndroidNativeBufferExtra {
	return FindAndroidNativeBufferExtra(e.cmd.Extras())
}

func (e externs) GetEGLImageData(id EGLImageKHR, _ GLsizei, _ GLsizei) {
	if d := FindEGLImageData(e.cmd.Extras()); d != nil {
		if GetState(e.s).EGLImages.Contains(id) {
			ei := GetState(e.s).EGLImages.Get(id)
			for _, img := range ei.Images.Range() {
				poolID, pool := e.s.Memory.New()
				pool.Write(0, memory.Resource(d.ID, d.Size))
				data := U8ˢ{pool: poolID, count: d.Size}
				img.Width = GLsizei(d.Width)
				img.Height = GLsizei(d.Height)
				img.Data = data
				img.DataFormat = d.Format
				img.DataType = d.Type
				break // TODO: Support image arrays.
			}
		}
	}
}

func (e externs) calcIndexLimits(data U8ˢ, indexSize int) resolve.IndexRange {
	id := data.ResourceID(e.ctx, e.s)
	count := int(data.count) / int(indexSize)
	littleEndian := e.s.MemoryLayout.GetEndian() == device.LittleEndian
	limits, err := resolve.IndexLimits(e.ctx, id, count, indexSize, littleEndian)
	if err != nil {
		if errors.Cause(err) == context.Canceled {
			// TODO: Propagate error
			return resolve.IndexRange{}
		} else {
			panic(fmt.Errorf("Could not calculate index limits: %v", err))
		}
	}
	return *limits
}

func (e externs) IndexLimits(data U8ˢ, indexSize int32) U32Limits {
	limits := e.calcIndexLimits(data, int(indexSize))
	return U32Limits{First: limits.First, Count: limits.Count}
}

func (e externs) substr(str string, start, end int32) string {
	return str[start:end]
}

func (e externs) GetCompileShaderExtra(ctx *Context, obj *Shader, bin *BinaryExtra) *CompileShaderExtra {
	return FindCompileShaderExtra(e.cmd.Extras(), obj)
}

func (e externs) GetLinkProgramExtra(ctx *Context, obj *Program, bin *BinaryExtra) *LinkProgramExtra {
	return FindLinkProgramExtra(e.cmd.Extras())
}

func (e externs) GetValidateProgramExtra(ctx *Context, obj *Program) *ValidateProgramExtra {
	return FindValidateProgramExtra(e.cmd.Extras())
}

func (e externs) GetValidateProgramPipelineExtra(ctx *Context, obj *Pipeline) *ValidateProgramPipelineExtra {
	return FindValidateProgramPipelineExtra(e.cmd.Extras())
}

func (e externs) onGlError(err GLenum) {
	// Call the state's callback function for API error.
	if f := e.s.OnError; f != nil {
		f(err)
	}
}

func (e externs) newMsg(severity Severity, message *stringtable.Msg) uint32 {
	// Call the state's callback function for message.
	if f := e.s.NewMessage; f != nil {
		return f(severityFromEnum(severity), message)
	}
	return 0
}

// Maps generated Severity to one of the const values defined in log.
func severityFromEnum(enumValue Severity) log.Severity {
	switch enumValue {
	case Severity_SEVERITY_DEBUG:
		return log.Debug
	case Severity_SEVERITY_INFO:
		return log.Info
	case Severity_SEVERITY_WARNING:
		return log.Warning
	case Severity_SEVERITY_ERROR:
		return log.Error
	case Severity_SEVERITY_FATAL:
		return log.Fatal
	default:
		panic(fmt.Errorf("Bad Severity value %v", enumValue))
	}
}

func (e externs) addTag(msgID uint32, message *stringtable.Msg) {
	// Call the state's callback function for message.
	if f := e.s.AddTag; f != nil {
		f(msgID, message)
	}
}

func (e externs) ReadGPUTextureData(texture *Texture, level, layer GLint) U8ˢ {
	poolID, dst := e.s.Memory.New()
	registry := bind.GetRegistry(e.ctx)
	img := texture.Levels.Get(level).Layers.Get(layer)
	dataFormat, dataType := img.getUnsizedFormatAndType()
	format, err := getImageFormat(dataFormat, dataType)
	if err != nil {
		panic(err)
	}
	size := format.Size(int(img.Width), int(img.Height), 1)
	device := registry.DefaultDevice() // TODO: Device selection.
	if device == nil {
		log.W(e.ctx, "No device found for GPU texture read")
		return U8ˢ{count: uint64(size), pool: poolID}
	}
	dataID, err := database.Store(e.ctx, &ReadGPUTextureDataResolveable{
		Capture:    path.NewCapture(capture.Get(e.ctx).Id.ID()),
		Device:     path.NewDevice(device.Instance().Id.ID()),
		After:      uint64(e.cmdID),
		Thread:     e.cmd.Thread(),
		Texture:    uint32(texture.ID),
		Level:      uint32(level),
		Layer:      uint32(layer),
		DataFormat: uint32(dataFormat),
		DataType:   uint32(dataType),
	})
	if err != nil {
		panic(err)
	}
	data := memory.Resource(dataID, uint64(size))
	dst.Write(0, data)
	return U8ˢ{count: uint64(size), pool: poolID}
}
<commit_msg>gles: Fix for GPU texture readbacks<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gles

import (
	"context"
	"fmt"

	"github.com/google/gapid/core/log"
	"github.com/google/gapid/core/os/device"
	"github.com/google/gapid/core/os/device/bind"
	"github.com/google/gapid/gapis/api"
	"github.com/google/gapid/gapis/capture"
	"github.com/google/gapid/gapis/database"
	"github.com/google/gapid/gapis/memory"
	rb "github.com/google/gapid/gapis/replay/builder"
	"github.com/google/gapid/gapis/resolve"
	"github.com/google/gapid/gapis/service/path"
	"github.com/google/gapid/gapis/stringtable"
	"github.com/pkg/errors"
)

type externs struct {
	ctx   context.Context // Allowed because the externs struct is only a parameter proxy for a single call
	cmd   api.Cmd
	cmdID api.CmdID
	s     *api.GlobalState
	b     *rb.Builder
}

func (e externs) mapMemory(slice memory.Slice) {
	ctx := e.ctx
	if b := e.b; b != nil {
		switch e.cmd.(type) {
		case *GlMapBufferRange, *GlMapBufferRangeEXT, *GlMapBufferOES, *GlMapBuffer:
			// Base address is on the stack.
			b.MapMemory(memory.Range{Base: slice.Base(), Size: slice.Size()})

		default:
			log.E(ctx, "mapBuffer extern called for unsupported command: %v", e.cmd)
		}
	}
}

func (e externs) unmapMemory(slice memory.Slice) {
	if b := e.b; b != nil {
		b.UnmapMemory(memory.Range{Base: slice.Base(), Size: slice.Size()})
	}
}

func (e externs) GetEGLStaticContextState(EGLDisplay, EGLContext) *StaticContextState {
	return FindStaticContextState(e.cmd.Extras())
}

func (e externs) GetEGLDynamicContextState(EGLDisplay, EGLSurface, EGLContext) *DynamicContextState {
	return FindDynamicContextState(e.cmd.Extras())
}

func (e externs) GetAndroidNativeBufferExtra(Voidᵖ) *AndroidNativeBufferExtra {
	return FindAndroidNativeBufferExtra(e.cmd.Extras())
}

func (e externs) GetEGLImageData(id EGLImageKHR, _ GLsizei, _ GLsizei) {
	if d := FindEGLImageData(e.cmd.Extras()); d != nil {
		if GetState(e.s).EGLImages.Contains(id) {
			ei := GetState(e.s).EGLImages.Get(id)
			for _, img := range ei.Images.Range() {
				poolID, pool := e.s.Memory.New()
				pool.Write(0, memory.Resource(d.ID, d.Size))
				data := U8ˢ{pool: poolID, count: d.Size, size: d.Size}
				img.Width = GLsizei(d.Width)
				img.Height = GLsizei(d.Height)
				img.Data = data
				img.DataFormat = d.Format
				img.DataType = d.Type
				break // TODO: Support image arrays.
			}
		}
	}
}

func (e externs) calcIndexLimits(data U8ˢ, indexSize int) resolve.IndexRange {
	id := data.ResourceID(e.ctx, e.s)
	count := int(data.count) / int(indexSize)
	littleEndian := e.s.MemoryLayout.GetEndian() == device.LittleEndian
	limits, err := resolve.IndexLimits(e.ctx, id, count, indexSize, littleEndian)
	if err != nil {
		if errors.Cause(err) == context.Canceled {
			// TODO: Propagate error
			return resolve.IndexRange{}
		} else {
			panic(fmt.Errorf("Could not calculate index limits: %v", err))
		}
	}
	return *limits
}

func (e externs) IndexLimits(data U8ˢ, indexSize int32) U32Limits {
	limits := e.calcIndexLimits(data, int(indexSize))
	return U32Limits{First: limits.First, Count: limits.Count}
}

func (e externs) substr(str string, start, end int32) string {
	return str[start:end]
}

func (e externs) GetCompileShaderExtra(ctx *Context, obj *Shader, bin *BinaryExtra) *CompileShaderExtra {
	return FindCompileShaderExtra(e.cmd.Extras(), obj)
}

func (e externs) GetLinkProgramExtra(ctx *Context, obj *Program, bin *BinaryExtra) *LinkProgramExtra {
	return FindLinkProgramExtra(e.cmd.Extras())
}

func (e externs) GetValidateProgramExtra(ctx *Context, obj *Program) *ValidateProgramExtra {
	return FindValidateProgramExtra(e.cmd.Extras())
}

func (e externs) GetValidateProgramPipelineExtra(ctx *Context, obj *Pipeline) *ValidateProgramPipelineExtra {
	return FindValidateProgramPipelineExtra(e.cmd.Extras())
}

func (e externs) onGlError(err GLenum) {
	// Call the state's callback function for API error.
	if f := e.s.OnError; f != nil {
		f(err)
	}
}

func (e externs) newMsg(severity Severity, message *stringtable.Msg) uint32 {
	// Call the state's callback function for message.
	if f := e.s.NewMessage; f != nil {
		return f(severityFromEnum(severity), message)
	}
	return 0
}

// Maps generated Severity to one of the const values defined in log.
func severityFromEnum(enumValue Severity) log.Severity {
	switch enumValue {
	case Severity_SEVERITY_DEBUG:
		return log.Debug
	case Severity_SEVERITY_INFO:
		return log.Info
	case Severity_SEVERITY_WARNING:
		return log.Warning
	case Severity_SEVERITY_ERROR:
		return log.Error
	case Severity_SEVERITY_FATAL:
		return log.Fatal
	default:
		panic(fmt.Errorf("Bad Severity value %v", enumValue))
	}
}

func (e externs) addTag(msgID uint32, message *stringtable.Msg) {
	// Call the state's callback function for message.
	if f := e.s.AddTag; f != nil {
		f(msgID, message)
	}
}

func (e externs) ReadGPUTextureData(texture *Texture, level, layer GLint) U8ˢ {
	poolID, dst := e.s.Memory.New()
	registry := bind.GetRegistry(e.ctx)
	img := texture.Levels.Get(level).Layers.Get(layer)
	dataFormat, dataType := img.getUnsizedFormatAndType()
	format, err := getImageFormat(dataFormat, dataType)
	if err != nil {
		panic(err)
	}
	size := format.Size(int(img.Width), int(img.Height), 1)
	device := registry.DefaultDevice() // TODO: Device selection.
	if device == nil {
		log.W(e.ctx, "No device found for GPU texture read")
		return U8ˢ{pool: poolID, count: uint64(size), size: uint64(size)}
	}
	dataID, err := database.Store(e.ctx, &ReadGPUTextureDataResolveable{
		Capture:    path.NewCapture(capture.Get(e.ctx).Id.ID()),
		Device:     path.NewDevice(device.Instance().Id.ID()),
		After:      uint64(e.cmdID),
		Thread:     e.cmd.Thread(),
		Texture:    uint32(texture.ID),
		Level:      uint32(level),
		Layer:      uint32(layer),
		DataFormat: uint32(dataFormat),
		DataType:   uint32(dataType),
	})
	if err != nil {
		panic(err)
	}
	data := memory.Resource(dataID, uint64(size))
	dst.Write(0, data)
	return U8ˢ{pool: poolID, count: uint64(size), size: uint64(size)}
}
<|endoftext|>


<commit_before>package metrics_test

import (
	"testing"

	"github.com/m-lab/annotation-service/api/v2"
	"github.com/m-lab/annotation-service/metrics"
	"github.com/m-lab/go/prometheusx"
)

// TestPrometheusMetrics ensures that all the metrics pass the linter. We apply
// labels to all metrics which require them in an effort to run all metrics
// through the linter.
func TestPrometheusMetrics(t *testing.T) {
	api.RequestTimeHistogram.WithLabelValues("x")
	metrics.RequestTimeHistogramUsec.WithLabelValues("x", "x")
	metrics.ErrorTotal.WithLabelValues("x")
	metrics.RejectionCount.WithLabelValues("x")
	prometheusx.LintMetrics(t)
}
<commit_msg>fix prom code<commit_after>package metrics_test

import (
	"testing"

	"github.com/m-lab/annotation-service/api/v2"
	"github.com/m-lab/annotation-service/metrics"
	"github.com/m-lab/go/prometheusx/promtest"
)

// TestPrometheusMetrics ensures that all the metrics pass the linter. We apply
// labels to all metrics which require them in an effort to run all metrics
// through the linter.
func TestPrometheusMetrics(t *testing.T) {
	api.RequestTimeHistogram.WithLabelValues("x")
	metrics.RequestTimeHistogramUsec.WithLabelValues("x", "x")
	metrics.ErrorTotal.WithLabelValues("x")
	metrics.RejectionCount.WithLabelValues("x")
	promtest.LintMetrics(t)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/pkg/errors"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/sipb/homeworld/platform/keysystem/worldconfig/paths"
	"github.com/sipb/homeworld/platform/kubernetes/wrapper"
)

func getVerbosityArgument() string {
	verbosityBytes, err := ioutil.ReadFile("/etc/homeworld/config/verbosity")
	if err != nil {
		return "--v=0"
	}
	verbosity := strings.TrimSpace(string(verbosityBytes))
	if _, err := strconv.ParseUint(verbosity, 10, 32); err != nil {
		log.Printf("cannot parse verbosity field value: '%s'\n", verbosity)
		return "--v=0"
	}
	log.Printf("using launch verbosity %s\n", verbosity)
	return "--v=" + verbosity
}

func kubeConfigPath(suffix string) string {
	return "/etc/homeworld/config/kubeconfig-" + suffix
}

func LaunchAPIServer() error {
	clusterConf, err := wrapper.GetClusterConf()
	if err != nil {
		return errors.Wrap(err, "while loading cluster.conf")
	}
	localConf, err := wrapper.GetLocalConf()
	if err != nil {
		return errors.Wrap(err, "while loading local.conf")
	}
	// parse the keys before using them for the sake of validation
	apiserverCount := clusterConf["APISERVER_COUNT"]
	if _, err := strconv.ParseUint(apiserverCount, 10, 32); err != nil {
		return errors.Wrap(err, "while validating APISERVER_COUNT")
	}
	serviceCidr := clusterConf["SERVICE_CIDR"]
	if _, _, err := net.ParseCIDR(serviceCidr); err != nil {
		return errors.Wrap(err, "while parsing SERVICE_CIDR")
	}
	hostIP := localConf["HOST_IP"]
	if net.ParseIP(hostIP) == nil {
		return fmt.Errorf("invalid IP '%s'", hostIP)
	}
	etcdEndpoints := clusterConf["ETCD_ENDPOINTS"]
	if strings.Count(etcdEndpoints, "://") < 1 { // just a sniff test, because the format is complex
		return fmt.Errorf("missing or invalid ETCD_ENDPOINTS: '%s'", etcdEndpoints)
	}
	cmd := exec.Command(
		"/usr/bin/hyperkube", "kube-apiserver",
		// role-based access control
		"--authorization-mode", "AlwaysAllow",
		// number of api servers
		"--apiserver-count", apiserverCount,
		// public addresses
		"--bind-address", "0.0.0.0", "--advertise-address", hostIP,
		// IP range
		"--service-cluster-ip-range", serviceCidr,
		// use standard HTTPS port for secure port
		"--secure-port", "443",
		// etcd cluster to use
		"--etcd-servers", etcdEndpoints,
		// allow privileged containers to run
		"--allow-privileged", "true",
		// disallow anonymous users
		"--anonymous-auth", "false",
		// various plugins for limitations and protection
		"--admission-control", "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DenyEscalatingExec,SecurityContextDeny",
		// authenticate clients properly
		"--client-ca-file", paths.KubernetesCAPath,
		// do HTTPS properly
		"--tls-cert-file", paths.KubernetesMasterCert, "--tls-private-key-file", paths.KubernetesMasterKey,
		// make sure account deletion works
		"--service-account-lookup",
		// no cloud provider
		"--cloud-provider=",
		// authenticate the etcd cluster to us
		"--etcd-cafile", "/etc/homeworld/authorities/etcd-server.pem",
		// authenticate us to the etcd cluster
		"--etcd-certfile", "/etc/homeworld/keys/etcd-client.pem", "--etcd-keyfile", "/etc/homeworld/keys/etcd-client.key",
		// disallow insecure port
		"--insecure-port", "0",
		// authenticate kubelet to us
		"--kubelet-certificate-authority", paths.KubernetesCAPath,
		// authenticate us to kubelet
		"--kubelet-client-certificate", paths.KubernetesMasterCert, "--kubelet-client-key", paths.KubernetesMasterKey,
		// let controller manager's service tokens work for us
		"--service-account-key-file", "/etc/homeworld/keys/serviceaccount.key",

		getVerbosityArgument(),
	)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err != nil {
		return errors.Wrap(err, "while running apiserver")
	}
	return nil
}

func LaunchControllerManager() error {
	clusterConf, err := wrapper.GetClusterConf()
	if err != nil {
		return errors.Wrap(err, "while loading cluster.conf")
	}
	serviceCidr := clusterConf["SERVICE_CIDR"]
	if _, _, err := net.ParseCIDR(serviceCidr); err != nil {
		return errors.Wrap(err, "while parsing SERVICE_CIDR")
	}
	clusterCidr := clusterConf["CLUSTER_CIDR"]
	if _, _, err := net.ParseCIDR(clusterCidr); err != nil {
		return errors.Wrap(err, "while parsing CLUSTER_CIDR")
	}

	kubeconfig := kubeConfigPath("controller-manager")
	err = wrapper.GenerateKubeConfigToFile(paths.KubernetesWorkerKey, paths.KubernetesWorkerCert, kubeconfig)
	if err != nil {
		return err
	}

	cmd := exec.Command(
		"/usr/bin/hyperkube", "kube-controller-manager",

		"--kubeconfig", kubeconfig,

		"--cluster-cidr", clusterCidr,
		"--node-cidr-mask-size", "24",
		"--service-cluster-ip-range", serviceCidr,
		"--cluster-name", "hyades",

		"--leader-elect",
		"--allocate-node-cidrs",

		// granting service tokens
		"--service-account-private-key-file", "/etc/homeworld/keys/serviceaccount.key",
		"--root-ca-file", "/etc/homeworld/authorities/kubernetes.pem",

		getVerbosityArgument(),
	)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err != nil {
		return errors.Wrap(err, "while running controller-manager")
	}
	return nil
}

func LaunchKubelet() error {
	clusterConf, err := wrapper.GetClusterConf()
	if err != nil {
		return errors.Wrap(err, "while loading cluster.conf")
	}
	localConf, err := wrapper.GetLocalConf()
	if err != nil {
		return errors.Wrap(err, "while loading cluster.conf")
	}
	serviceDNS := clusterConf["SERVICE_DNS"]
	if net.ParseIP(serviceDNS) == nil {
		return errors.Wrap(err, "while parsing SERVICE_DNS")
	}
	clusterDomain := clusterConf["CLUSTER_DOMAIN"]
	if strings.Count(clusterDomain, ".") < 1 { // sniff test because it's not immediately clear what's okay
		return errors.Wrap(err, "while validating CLUSTER_DOMAIN")
	}
	scheduleWork, err := strconv.ParseBool(localConf["SCHEDULE_WORK"])
	if err != nil {
		return errors.Wrap(err, "while parsing SCHEDULE_WORK")
	}

	kubeconfig := kubeConfigPath("kubelet")
	err = wrapper.GenerateKubeConfigToFile(paths.KubernetesWorkerKey, paths.KubernetesWorkerCert, kubeconfig)
	if err != nil {
		return err
	}

	cmd := exec.Command(
		"/usr/bin/hyperkube", "kubelet",

		"--kubeconfig", kubeconfig,

		"--register-schedulable="+strconv.FormatBool(scheduleWork),
		// turn off anonymous authentication
		"--anonymous-auth=false",
		// add kubelet auth certs
		"--tls-cert-file", paths.KubernetesWorkerCert, "--tls-private-key-file", paths.KubernetesWorkerKey,
		// add client certificate authority
		"--client-ca-file", paths.KubernetesCAPath,
		// turn off cloud provider detection
		"--cloud-provider=",
		// use CRI-O
		"--container-runtime", "remote", "--container-runtime-endpoint", "unix:///var/run/crio/crio.sock",
		// DNS
		"--cluster-dns", serviceDNS, "--cluster-domain", clusterDomain,

		getVerbosityArgument(),
	)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err != nil {
		return errors.Wrap(err, "while running kubelet")
	}
	return nil
}

func LaunchProxy() error {
	kubeconfig := kubeConfigPath("proxy")
	err := wrapper.GenerateKubeConfigToFile(paths.KubernetesWorkerKey, paths.KubernetesWorkerCert, kubeconfig)
	if err != nil {
		return err
	}

	cmd := exec.Command(
		"/usr/bin/hyperkube", "kube-proxy",

		"--kubeconfig", kubeconfig,
		// synchronize every minute (TODO: IS THIS A GOOD AMOUNT OF TIME?)
		"--config-sync-period", "1m",

		getVerbosityArgument(),
	)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err != nil {
		return errors.Wrap(err, "while running proxy")
	}
	return nil
}

func LaunchScheduler() error {
	kubeconfig := kubeConfigPath("scheduler")
	err := wrapper.GenerateKubeConfigToFile(paths.KubernetesWorkerKey, paths.KubernetesWorkerCert, kubeconfig)
	if err != nil {
		return err
	}

	cmd := exec.Command(
		"/usr/bin/hyperkube", "kube-scheduler",

		"--kubeconfig", kubeconfig,
		"--leader-elect",

		getVerbosityArgument(),
	)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err != nil {
		return errors.Wrap(err, "while running scheduler")
	}
	return nil
}

func LaunchService(name string) error {
	switch name {
	case "apiserver":
		return LaunchAPIServer()
	case "controller-manager":
		return LaunchControllerManager()
	case "kubelet":
		return LaunchKubelet()
	case "proxy":
		return LaunchProxy()
	case "scheduler":
		return LaunchScheduler()
	default:
		return fmt.Errorf("no such service %s", name)
	}
}

func main() {
	if len(os.Args) != 2 {
		log.Fatalf("usage: kube-launch <service>\n")
	}
	err := LaunchService(os.Args[1])
	if err != nil {
		log.Fatalf("kube-launch error: %s\n", err.Error())
	}
}
<commit_msg>kubernetes: add default admission controllers<commit_after>package main

import (
	"fmt"
	"github.com/pkg/errors"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/sipb/homeworld/platform/keysystem/worldconfig/paths"
	"github.com/sipb/homeworld/platform/kubernetes/wrapper"
)

func getVerbosityArgument() string {
	verbosityBytes, err := ioutil.ReadFile("/etc/homeworld/config/verbosity")
	if err != nil {
		return "--v=0"
	}
	verbosity := strings.TrimSpace(string(verbosityBytes))
	if _, err := strconv.ParseUint(verbosity, 10, 32); err != nil {
		log.Printf("cannot parse verbosity field value: '%s'\n", verbosity)
		return "--v=0"
	}
	log.Printf("using launch verbosity %s\n", verbosity)
	return "--v=" + verbosity
}

func kubeConfigPath(suffix string) string {
	return "/etc/homeworld/config/kubeconfig-" + suffix
}

func LaunchAPIServer() error {
	clusterConf, err := wrapper.GetClusterConf()
	if err != nil {
		return errors.Wrap(err, "while loading cluster.conf")
	}
	localConf, err := wrapper.GetLocalConf()
	if err != nil {
		return errors.Wrap(err, "while loading local.conf")
	}
	// parse the keys before using them for the sake of validation
	apiserverCount := clusterConf["APISERVER_COUNT"]
	if _, err := strconv.ParseUint(apiserverCount, 10, 32); err != nil {
		return errors.Wrap(err, "while validating APISERVER_COUNT")
	}
	serviceCidr := clusterConf["SERVICE_CIDR"]
	if _, _, err := net.ParseCIDR(serviceCidr); err != nil {
		return errors.Wrap(err, "while parsing SERVICE_CIDR")
	}
	hostIP := localConf["HOST_IP"]
	if net.ParseIP(hostIP) == nil {
		return fmt.Errorf("invalid IP '%s'", hostIP)
	}
	etcdEndpoints := clusterConf["ETCD_ENDPOINTS"]
	if strings.Count(etcdEndpoints, "://") < 1 { // just a sniff test, because the format is complex
		return fmt.Errorf("missing or invalid ETCD_ENDPOINTS: '%s'", etcdEndpoints)
	}
	cmd := exec.Command(
		"/usr/bin/hyperkube", "kube-apiserver",
		// role-based access control
		"--authorization-mode", "AlwaysAllow",
		// number of api servers
		"--apiserver-count", apiserverCount,
		// public addresses
		"--bind-address", "0.0.0.0", "--advertise-address", hostIP,
		// IP range
		"--service-cluster-ip-range", serviceCidr,
		// use standard HTTPS port for secure port
		"--secure-port", "443",
		// etcd cluster to use
		"--etcd-servers", etcdEndpoints,
		// allow privileged containers to run
		"--allow-privileged", "true",
		// disallow anonymous users
		"--anonymous-auth", "false",
		// various plugins for limitations and protection
		"--admission-control", "NamespaceLifecycle,LimitRanger,ServiceAccount,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,PersistentVolumeClaimResize,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,RuntimeClass,ResourceQuota,DenyEscalatingExec,SecurityContextDeny",
		// authenticate clients properly
		"--client-ca-file", paths.KubernetesCAPath,
		// do HTTPS properly
		"--tls-cert-file", paths.KubernetesMasterCert, "--tls-private-key-file", paths.KubernetesMasterKey,
		// make sure account deletion works
		"--service-account-lookup",
		// no cloud provider
		"--cloud-provider=",
		// authenticate the etcd cluster to us
		"--etcd-cafile", "/etc/homeworld/authorities/etcd-server.pem",
		// authenticate us to the etcd cluster
		"--etcd-certfile", "/etc/homeworld/keys/etcd-client.pem", "--etcd-keyfile", "/etc/homeworld/keys/etcd-client.key",
		// disallow insecure port
		"--insecure-port", "0",
		// authenticate kubelet to us
		"--kubelet-certificate-authority", paths.KubernetesCAPath,
		// authenticate us to kubelet
		"--kubelet-client-certificate", paths.KubernetesMasterCert, "--kubelet-client-key", paths.KubernetesMasterKey,
		// let controller manager's service tokens work for us
		"--service-account-key-file", "/etc/homeworld/keys/serviceaccount.key",

		getVerbosityArgument(),
	)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err != nil {
		return errors.Wrap(err, "while running apiserver")
	}
	return nil
}

func LaunchControllerManager() error {
	clusterConf, err := wrapper.GetClusterConf()
	if err != nil {
		return errors.Wrap(err, "while loading cluster.conf")
	}
	serviceCidr := clusterConf["SERVICE_CIDR"]
	if _, _, err := net.ParseCIDR(serviceCidr); err != nil {
		return errors.Wrap(err, "while parsing SERVICE_CIDR")
	}
	clusterCidr := clusterConf["CLUSTER_CIDR"]
	if _, _, err := net.ParseCIDR(clusterCidr); err != nil {
		return errors.Wrap(err, "while parsing CLUSTER_CIDR")
	}

	kubeconfig := kubeConfigPath("controller-manager")
	err = wrapper.GenerateKubeConfigToFile(paths.KubernetesWorkerKey, paths.KubernetesWorkerCert, kubeconfig)
	if err != nil {
		return err
	}

	cmd := exec.Command(
		"/usr/bin/hyperkube", "kube-controller-manager",

		"--kubeconfig", kubeconfig,

		"--cluster-cidr", clusterCidr,
		"--node-cidr-mask-size", "24",
		"--service-cluster-ip-range", serviceCidr,
		"--cluster-name", "hyades",

		"--leader-elect",
		"--allocate-node-cidrs",

		// granting service tokens
		"--service-account-private-key-file", "/etc/homeworld/keys/serviceaccount.key",
		"--root-ca-file", "/etc/homeworld/authorities/kubernetes.pem",

		getVerbosityArgument(),
	)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err != nil {
		return errors.Wrap(err, "while running controller-manager")
	}
	return nil
}

func LaunchKubelet() error {
	clusterConf, err := wrapper.GetClusterConf()
	if err != nil {
		return errors.Wrap(err, "while loading cluster.conf")
	}
	localConf, err := wrapper.GetLocalConf()
	if err != nil {
		return errors.Wrap(err, "while loading cluster.conf")
	}
	serviceDNS := clusterConf["SERVICE_DNS"]
	if net.ParseIP(serviceDNS) == nil {
		return errors.Wrap(err, "while parsing SERVICE_DNS")
	}
	clusterDomain := clusterConf["CLUSTER_DOMAIN"]
	if strings.Count(clusterDomain, ".") < 1 { // sniff test because it's not immediately clear what's okay
		return errors.Wrap(err, "while validating CLUSTER_DOMAIN")
	}
	scheduleWork, err := strconv.ParseBool(localConf["SCHEDULE_WORK"])
	if err != nil {
		return errors.Wrap(err, "while parsing SCHEDULE_WORK")
	}

	kubeconfig := kubeConfigPath("kubelet")
	err = wrapper.GenerateKubeConfigToFile(paths.KubernetesWorkerKey, paths.KubernetesWorkerCert, kubeconfig)
	if err != nil {
		return err
	}

	cmd := exec.Command(
		"/usr/bin/hyperkube", "kubelet",

		"--kubeconfig", kubeconfig,

		"--register-schedulable="+strconv.FormatBool(scheduleWork),
		// turn off anonymous authentication
		"--anonymous-auth=false",
		// add kubelet auth certs
		"--tls-cert-file", paths.KubernetesWorkerCert, "--tls-private-key-file", paths.KubernetesWorkerKey,
		// add client certificate authority
		"--client-ca-file", paths.KubernetesCAPath,
		// turn off cloud provider detection
		"--cloud-provider=",
		// use CRI-O
		"--container-runtime", "remote", "--container-runtime-endpoint", "unix:///var/run/crio/crio.sock",
		// DNS
		"--cluster-dns", serviceDNS, "--cluster-domain", clusterDomain,

		getVerbosityArgument(),
	)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err != nil {
		return errors.Wrap(err, "while running kubelet")
	}
	return nil
}

func LaunchProxy() error {
	kubeconfig := kubeConfigPath("proxy")
	err := wrapper.GenerateKubeConfigToFile(paths.KubernetesWorkerKey, paths.KubernetesWorkerCert, kubeconfig)
	if err != nil {
		return err
	}

	cmd := exec.Command(
		"/usr/bin/hyperkube", "kube-proxy",

		"--kubeconfig", kubeconfig,
		// synchronize every minute (TODO: IS THIS A GOOD AMOUNT OF TIME?)
		"--config-sync-period", "1m",

		getVerbosityArgument(),
	)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err != nil {
		return errors.Wrap(err, "while running proxy")
	}
	return nil
}

func LaunchScheduler() error {
	kubeconfig := kubeConfigPath("scheduler")
	err := wrapper.GenerateKubeConfigToFile(paths.KubernetesWorkerKey, paths.KubernetesWorkerCert, kubeconfig)
	if err != nil {
		return err
	}

	cmd := exec.Command(
		"/usr/bin/hyperkube", "kube-scheduler",

		"--kubeconfig", kubeconfig,
		"--leader-elect",

		getVerbosityArgument(),
	)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err != nil {
		return errors.Wrap(err, "while running scheduler")
	}
	return nil
}

func LaunchService(name string) error {
	switch name {
	case "apiserver":
		return LaunchAPIServer()
	case "controller-manager":
		return LaunchControllerManager()
	case "kubelet":
		return LaunchKubelet()
	case "proxy":
		return LaunchProxy()
	case "scheduler":
		return LaunchScheduler()
	default:
		return fmt.Errorf("no such service %s", name)
	}
}

func main() {
	if len(os.Args) != 2 {
		log.Fatalf("usage: kube-launch <service>\n")
	}
	err := LaunchService(os.Args[1])
	if err != nil {
		log.Fatalf("kube-launch error: %s\n", err.Error())
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"github.com/spf13/cobra"
	"fmt"
	"github.com/nais/naisd/api"
	"io/ioutil"
	"gopkg.in/yaml.v2"
)

var validateCmd = &cobra.Command{
	Use:   "validate",
	Short: "Validates nais.yaml",
	Long:  `Validates nais.yaml`,
	Run: func(cmd *cobra.Command, args []string) {

		file, err := cmd.Flags().GetString("file")
		if err != nil {
			fmt.Printf("Error when getting flag: file. %v", err)
			return
		}
		output, err := cmd.Flags().GetBool("output")
		if err != nil {
			fmt.Printf("Error when getting flag: output. %v", err)
			return
		}

		naisYaml, err := ioutil.ReadFile(file)
		if err != nil {
			fmt.Printf("Could not read file: "+file+". %v", err)
		}

		fmt.Println("Validating the file: " + file)

		var appConfig api.NaisAppConfig

		if err := yaml.Unmarshal(naisYaml, &appConfig); err != nil {
			fmt.Printf("Error while unmarshalling yaml. %v", err)
		}

		if err := api.AddDefaultAppconfigValues(&appConfig, "appName"); err != nil {
			fmt.Printf("Error while adding default values yaml. %v", err)
		}

		if output {
			conf, _  := yaml.Marshal(appConfig)
			fmt.Println(string(conf))
		}

		validationErrors := api.ValidateAppConfig(appConfig);
		if len(validationErrors.Errors) != 0 {
			fmt.Println("Found errors while validating " + file)
			fmt.Printf("%v", validationErrors)
		}
	},
}

func init() {
	RootCmd.AddCommand(validateCmd)
	validateCmd.Flags().StringP("file", "f", "nais.yaml", "path to appconfig")
	validateCmd.Flags().BoolP("output", "o", false, "prints full appconfig including defaults if tr")
}
<commit_msg>Exit with code > 0 in case of error<commit_after>package cmd

import (
	"fmt"
	"github.com/nais/naisd/api"
	"github.com/spf13/cobra"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
)

var validateCmd = &cobra.Command{
	Use:   "validate",
	Short: "Validates nais.yaml",
	Long:  `Validates nais.yaml`,
	Run: func(cmd *cobra.Command, args []string) {

		file, err := cmd.Flags().GetString("file")
		if err != nil {
			fmt.Printf("Error when getting flag: file. %v", err)
			os.Exit(1)
		}
		output, err := cmd.Flags().GetBool("output")
		if err != nil {
			fmt.Printf("Error when getting flag: output. %v", err)
			os.Exit(1)
		}

		naisYaml, err := ioutil.ReadFile(file)
		if err != nil {
			fmt.Printf("Could not read file: "+file+". %v", err)
			os.Exit(1)
		}

		fmt.Println("Validating the file: " + file)

		var appConfig api.NaisAppConfig

		if err := yaml.Unmarshal(naisYaml, &appConfig); err != nil {
			fmt.Printf("Error while unmarshalling yaml. %v", err)
			os.Exit(1)
		}

		if err := api.AddDefaultAppconfigValues(&appConfig, "appName"); err != nil {
			fmt.Printf("Error while adding default values yaml. %v", err)
			os.Exit(1)
		}

		if output {
			conf, _ := yaml.Marshal(appConfig)
			fmt.Println(string(conf))
		}

		validationErrors := api.ValidateAppConfig(appConfig)
		if len(validationErrors.Errors) != 0 {
			fmt.Println("Found errors while validating " + file)
			fmt.Printf("%v", validationErrors)
			os.Exit(1)
		}
	},
}

func init() {
	RootCmd.AddCommand(validateCmd)
	validateCmd.Flags().StringP("file", "f", "nais.yaml", "path to appconfig")
	validateCmd.Flags().BoolP("output", "o", false, "prints full appconfig including defaults if tr")
}
<|endoftext|>


<commit_before>package zmq4chan_test

import (
	"bytes"
	"testing"

	zmqchan "."
)

var (
	messages = [][][]byte{
		[][]byte{
			[]byte("hello"),
			[]byte("world"),
		},
		[][]byte{
			[]byte("123"),
			[]byte("4567"),
			[]byte("89"),
		},
	}

	parts = []zmqchan.Data{
		zmqchan.Data{[]byte("hello"), true},
		zmqchan.Data{[]byte("world"), false},
		zmqchan.Data{[]byte("123"), true},
		zmqchan.Data{[]byte("4567"), true},
		zmqchan.Data{[]byte("89"), false},
	}
)

func TestSendMessageBytes(t *testing.T) {
	var (
		input  = make(chan [][]byte)
		output = zmqchan.SendMessageBytes(input)
	)

	go func() {
		defer close(input)
		for _, message := range messages {
			input <- message
		}
	}()

	i := 0

	for part := range output {
		if bytes.Compare(part.Bytes, parts[i].Bytes) != 0 {
			t.Errorf("i=%d Bytes mismatch", i)
		}
		if part.More != parts[i].More {
			t.Errorf("i=%d More mismatch", i)
		}
		i++
	}

	if i != len(parts) {
		t.Error("too few parts received")
	}
}

func TestRecvMessageBytes(t *testing.T) {
	t.Log("TODO")
}
<commit_msg>RecvMessageBytes test<commit_after>package zmq4chan_test

import (
	"bytes"
	"testing"

	zmqchan "."
)

var (
	messages = [][][]byte{
		[][]byte{
			[]byte("hello"),
			[]byte("world"),
		},
		[][]byte{
			[]byte("123"),
			[]byte("4567"),
			[]byte("89"),
		},
	}

	parts = []zmqchan.Data{
		zmqchan.Data{[]byte("hello"), true},
		zmqchan.Data{[]byte("world"), false},
		zmqchan.Data{[]byte("123"), true},
		zmqchan.Data{[]byte("4567"), true},
		zmqchan.Data{[]byte("89"), false},
	}
)

func TestSendMessageBytes(t *testing.T) {
	input := make(chan [][]byte)
	output := zmqchan.SendMessageBytes(input)

	go func() {
		defer close(input)

		for _, message := range messages {
			input <- message
		}
	}()

	i := 0

	for part := range output {
		if bytes.Compare(part.Bytes, parts[i].Bytes) != 0 {
			t.Errorf("i=%d Bytes mismatch", i)
		}

		if part.More != parts[i].More {
			t.Errorf("i=%d More mismatch", i)
		}

		i++
	}

	if i != len(parts) {
		t.Error("incorrect number of parts received")
	}
}

func TestRecvMessageBytes(t *testing.T) {
	output := make(chan [][]byte)
	input := zmqchan.RecvMessageBytes(output)

	go func() {
		defer close(input)

		for _, part := range parts {
			input <- part
		}
	}()

	i := 0

	for message := range output {
		if len(message) != len(messages[i]) {
			t.Fatalf("i=%d length mismatch", i)
		}

		for j, part := range message {
			if bytes.Compare(part, messages[i][j]) != 0 {
				t.Errorf("i=%d part=%d mismatch", i, j)
			}
		}

		i++
	}

	if i != len(messages) {
		t.Error("incorrect number of messages received")
	}
}
<|endoftext|>


<commit_before>package proxy

import (
	"testing"
)

func TestUrlNormalization(t *testing.T) {
	shouldEqual(normalizeUrl(""), "", t)
	shouldEqual(normalizeUrl("://"), "", t)
	shouldEqual(normalizeUrl("http://"), "", t)
	shouldEqual(normalizeUrl("https://"), "", t)

	shouldEqual(normalizeUrl("192.168.1.1"), "http://192.168.1.1/", t)
	shouldEqual(normalizeUrl("://192.168.1.1"), "http://192.168.1.1/", t)
	shouldEqual(normalizeUrl("http://192.168.1.1"), "http://192.168.1.1/", t)
	shouldEqual(normalizeUrl("https://192.168.1.1"), "https://192.168.1.1/", t)

	shouldEqual(normalizeUrl("example.com"), "http://example.com/", t)
	shouldEqual(normalizeUrl("example.com"), "http://example.com/", t)
	shouldEqual(normalizeUrl("example.com/dir/"), "http://example.com/dir/", t)
}

func TestHttpScheme(t *testing.T) {
	shouldEqual(httpScheme("http"), "http", t)
	shouldEqual(httpScheme("https"), "https", t)
	shouldEqual(httpScheme("ws"), "http", t)
	shouldEqual(httpScheme("wss"), "https", t)

	shouldEqual(httpScheme(""), "http", t)
	shouldEqual(httpScheme("abcd"), "http", t)
}

func TestWebsocketScheme(t *testing.T) {
	shouldEqual(websocketScheme("ws"), "ws", t)
	shouldEqual(websocketScheme("wss"), "wss", t)
	shouldEqual(websocketScheme("https"), "wss", t)
	shouldEqual(websocketScheme("https"), "wss", t)

	shouldEqual(websocketScheme(""), "ws", t)
	shouldEqual(websocketScheme("abcd"), "ws", t)
}

// Utility function for shorter string compares
func shouldEqual(value, expected string, t *testing.T) {
	if value != expected {
		t.Errorf("Value is '%s', expected '%s'", value, expected)
	}
}
<commit_msg>Add tests for hostnames with ports<commit_after>package proxy

import (
	"testing"
)

func TestUrlNormalization(t *testing.T) {
	shouldEqual(normalizeUrl(""), "", t)
	shouldEqual(normalizeUrl("://"), "", t)
	shouldEqual(normalizeUrl("http://"), "", t)
	shouldEqual(normalizeUrl("https://"), "", t)

	shouldEqual(normalizeUrl("192.168.1.1"), "http://192.168.1.1/", t)
	shouldEqual(normalizeUrl("://192.168.1.1"), "http://192.168.1.1/", t)
	shouldEqual(normalizeUrl("http://192.168.1.1"), "http://192.168.1.1/", t)
	shouldEqual(normalizeUrl("https://192.168.1.1"), "https://192.168.1.1/", t)

	shouldEqual(normalizeUrl("example.com"), "http://example.com/", t)
	shouldEqual(normalizeUrl("example.com"), "http://example.com/", t)
	shouldEqual(normalizeUrl("example.com/dir/"), "http://example.com/dir/", t)

	shouldEqual(normalizeUrl("localhost:3000"), "http://localhost:3000/", t)
	shouldEqual(normalizeUrl("example.com:3000"), "http://example.com:3000/", t)
	shouldEqual(normalizeUrl("https://example.com:3000"), "https://example.com:3000/", t)
	shouldEqual(normalizeUrl("192.168.1.1:3000"), "http://192.168.1.1:3000/", t)
}

func TestHttpScheme(t *testing.T) {
	shouldEqual(httpScheme("http"), "http", t)
	shouldEqual(httpScheme("https"), "https", t)
	shouldEqual(httpScheme("ws"), "http", t)
	shouldEqual(httpScheme("wss"), "https", t)

	shouldEqual(httpScheme(""), "http", t)
	shouldEqual(httpScheme("abcd"), "http", t)
}

func TestWebsocketScheme(t *testing.T) {
	shouldEqual(websocketScheme("ws"), "ws", t)
	shouldEqual(websocketScheme("wss"), "wss", t)
	shouldEqual(websocketScheme("https"), "wss", t)
	shouldEqual(websocketScheme("https"), "wss", t)

	shouldEqual(websocketScheme(""), "ws", t)
	shouldEqual(websocketScheme("abcd"), "ws", t)
}

// Utility function for shorter string compares
func shouldEqual(value, expected string, t *testing.T) {
	if value != expected {
		t.Errorf("Value is '%s', expected '%s'", value, expected)
	}
}
<|endoftext|>


<commit_before>package tusd_test

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"

	"github.com/golang/mock/gomock"
	"github.com/tus/tusd"
)

type FullDataStore interface {
	tusd.DataStore
	tusd.TerminaterDataStore
	tusd.ConcaterDataStore
	tusd.GetReaderDataStore
	tusd.FinisherDataStore
}

type Locker interface {
	tusd.LockerDataStore
}

type httpTest struct {
	Name string

	Method string
	URL    string

	ReqBody   io.Reader
	ReqHeader map[string]string

	Code      int
	ResBody   string
	ResHeader map[string]string
}

func (test *httpTest) Run(handler http.Handler, t *testing.T) *httptest.ResponseRecorder {
	req, _ := http.NewRequest(test.Method, test.URL, test.ReqBody)

	// Add headers
	for key, value := range test.ReqHeader {
		req.Header.Set(key, value)
	}

	req.Host = "tus.io"
	w := httptest.NewRecorder()
	handler.ServeHTTP(w, req)

	if w.Code != test.Code {
		t.Errorf("Expected %v %s as status code (got %v %s)", test.Code, http.StatusText(test.Code), w.Code, http.StatusText(w.Code))
	}

	for key, value := range test.ResHeader {
		header := w.HeaderMap.Get(key)

		if value != header {
			t.Errorf("Expected '%s' as '%s' (got '%s')", value, key, header)
		}
	}

	if test.ResBody != "" && string(w.Body.Bytes()) != test.ResBody {
		t.Errorf("Expected '%s' as body (got '%s'", test.ResBody, string(w.Body.Bytes()))
	}

	return w
}

type readerMatcher struct {
	expect string
}

func NewReaderMatcher(expect string) gomock.Matcher {
	return readerMatcher{
		expect: expect,
	}
}

func (m readerMatcher) Matches(x interface{}) bool {
	input, ok := x.(io.Reader)
	if !ok {
		return false
	}

	bytes, err := ioutil.ReadAll(input)
	if err != nil {
		panic(err)
	}

	readStr := string(bytes)
	return reflect.DeepEqual(m.expect, readStr)
}

func (m readerMatcher) String() string {
	return fmt.Sprintf("reads to %s", m.expect)
}
<commit_msg>Add comments to test utilities<commit_after>package tusd_test

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"

	"github.com/golang/mock/gomock"
	"github.com/tus/tusd"
)

//go:generate mockgen -package tusd_test -source utils_test.go -aux_files tusd=datastore.go -destination=handler_mock_test.go

// FullDataStore is an interface combining most interfaces for data stores.
// This is used by mockgen(1) to generate a mocked data store used for testing
// (see https://github.com/golang/mock). The only interface excluded is
// LockerDataStore because else we would have to explicitly expect calls for
// locking in every single test which would result in more verbose code.
// Therefore it has been moved into its own type definition, the Locker.
type FullDataStore interface {
	tusd.DataStore
	tusd.TerminaterDataStore
	tusd.ConcaterDataStore
	tusd.GetReaderDataStore
	tusd.FinisherDataStore
}

type Locker interface {
	tusd.LockerDataStore
}

type httpTest struct {
	Name string

	Method string
	URL    string

	ReqBody   io.Reader
	ReqHeader map[string]string

	Code      int
	ResBody   string
	ResHeader map[string]string
}

func (test *httpTest) Run(handler http.Handler, t *testing.T) *httptest.ResponseRecorder {
	req, _ := http.NewRequest(test.Method, test.URL, test.ReqBody)

	// Add headers
	for key, value := range test.ReqHeader {
		req.Header.Set(key, value)
	}

	req.Host = "tus.io"
	w := httptest.NewRecorder()
	handler.ServeHTTP(w, req)

	if w.Code != test.Code {
		t.Errorf("Expected %v %s as status code (got %v %s)", test.Code, http.StatusText(test.Code), w.Code, http.StatusText(w.Code))
	}

	for key, value := range test.ResHeader {
		header := w.HeaderMap.Get(key)

		if value != header {
			t.Errorf("Expected '%s' as '%s' (got '%s')", value, key, header)
		}
	}

	if test.ResBody != "" && string(w.Body.Bytes()) != test.ResBody {
		t.Errorf("Expected '%s' as body (got '%s'", test.ResBody, string(w.Body.Bytes()))
	}

	return w
}

type readerMatcher struct {
	expect string
}

// NewReaderMatcher returns a gomock.Matcher which can be used in tests for
// expecting io.Readers as arguments. It will only report an argument x as
// matching if it's an io.Reader which, if fully read, equals the string `expect`.
func NewReaderMatcher(expect string) gomock.Matcher {
	return readerMatcher{
		expect: expect,
	}
}

func (m readerMatcher) Matches(x interface{}) bool {
	input, ok := x.(io.Reader)
	if !ok {
		return false
	}

	bytes, err := ioutil.ReadAll(input)
	if err != nil {
		panic(err)
	}

	readStr := string(bytes)
	return reflect.DeepEqual(m.expect, readStr)
}

func (m readerMatcher) String() string {
	return fmt.Sprintf("reads to %s", m.expect)
}
<|endoftext|>


<commit_before>package orderedmap

import (
	"errors"
	"sort"
	"sync"
)

type OrderedMap struct {
	data  map[string]interface{}
	order []string
	sync.RWMutex
}

// Create a new ordered map object
func New() OrderedMap {
	return OrderedMap{
		data:  make(map[string]interface{}),
		order: make([]string, 0),
	}
}

// Add an object onto the end of the map
func (m *OrderedMap) Add(key string, value interface{}) {
	m.Lock()
	m.data[key] = value
	m.order = append(m.order, key)
	m.Unlock()
}

// Add an object to a specific position in the map
func (m *OrderedMap) Insert(position int, key string, value interface{}) error {
	if position >= len(m.data) {
		return errors.New("Position is larger than the current map size.")
	}

	if position < 0 {
		return errors.New("Position is less than 0.")
	}

	m.Lock()
	m.data[key] = value
	pre := m.order[:position]
	post := m.order[position:]
	m.order = make([]string, len(pre))
	copy(m.order, pre)
	m.order = append(m.order, key)
	m.order = append(m.order, post...)
	m.Unlock()

	return nil
}

// Get a specific object out of the map based on its key
func (m OrderedMap) GetKey(key string) (interface{}, bool) {
	m.RLock()
	data, ok := m.data[key]
	m.RUnlock()
	return data, ok
}

func (m OrderedMap) GetIndex(index int) (interface{}, bool) {
	m.RLock()
	key := m.order[index]
	data, ok := m.data[key]
	m.RUnlock()
	return data, ok
}

// Get a slice of strings containing the current order
func (m OrderedMap) GetOrder() []string {
	m.RLock()
	tmp := make([]string, len(m.order))
	copy(tmp, m.order)
	m.RUnlock()
	return tmp
}

// Set a new order for this map, must contain all keys.
func (m *OrderedMap) SetOrder(order []string) error {
	if !compareOrder(m.order, order) {
		return errors.New("Provided order does not contain the same data as existing.")
	}
	m.Lock()
	copy(m.order, order)
	m.Unlock()
	return nil
}

// Get the index in the order of a specific key
func (m OrderedMap) IndexOf(key string) int {
	m.RLock()
	index := -1
	for i := 0; i < len(m.order); i++ {
		if m.order[i] == key {
			index = i
		}
	}
	m.RUnlock()
	return index
}

// Delete a specific key and all associated data from the map
func (m *OrderedMap) Delete(key string) {
	idx := m.IndexOf(key)

	m.Lock()
	delete(m.data, key)
	tmp := make([]string, len(m.order))
	copy(tmp, m.order)
	m.order = make([]string, len(tmp))

	m.order = append(tmp[:idx], tmp[idx+1:]...)
	m.Unlock()
}

// Get the total size of the map
func (m OrderedMap) Count() int {
	m.RLock()
	cnt := len(m.data)
	m.RUnlock()
	return cnt
}

// Compare two orders and determine if they have the same data even if not in the same order
func compareOrder(f []string, s []string) bool {
	// Check to see if the two slices have the same length, if not they obviously aren't the same
	if len(f) != len(s) {
		return false
	}

	// Let's copy our own slices, so we don't edit the originals
	tmpf := make([]string, len(f))
	tmps := make([]string, len(s))
	copy(tmpf, f)
	copy(tmps, s)

	// Now we'll sort them and go through each index to make sure it's the same
	sort.Strings(tmpf)
	sort.Strings(tmps)

	for i, v := range tmps {
		if tmpf[i] != v {
			return false
		}
	}

	return true
}
<commit_msg>Made lock private<commit_after>package orderedmap

import (
	"errors"
	"sort"
	"sync"
)

type OrderedMap struct {
	data  map[string]interface{}
	order []string
	lock  sync.RWMutex
}

// Create a new ordered map object
func New() OrderedMap {
	return OrderedMap{
		data:  make(map[string]interface{}),
		order: make([]string, 0),
	}
}

// Add an object onto the end of the map
func (m *OrderedMap) Add(key string, value interface{}) {
	m.lock.Lock()
	m.data[key] = value
	m.order = append(m.order, key)
	m.lock.Unlock()
}

// Add an object to a specific position in the map
func (m *OrderedMap) Insert(position int, key string, value interface{}) error {
	if position >= len(m.data) {
		return errors.New("Position is larger than the current map size.")
	}

	if position < 0 {
		return errors.New("Position is less than 0.")
	}

	m.lock.Lock()
	m.data[key] = value
	pre := m.order[:position]
	post := m.order[position:]
	m.order = make([]string, len(pre))
	copy(m.order, pre)
	m.order = append(m.order, key)
	m.order = append(m.order, post...)
	m.lock.Unlock()

	return nil
}

// Get a specific object out of the map based on its key
func (m OrderedMap) GetKey(key string) (interface{}, bool) {
	m.lock.RLock()
	data, ok := m.data[key]
	m.lock.RUnlock()
	return data, ok
}

func (m OrderedMap) GetIndex(index int) (interface{}, bool) {
	m.lock.RLock()
	key := m.order[index]
	data, ok := m.data[key]
	m.lock.RUnlock()
	return data, ok
}

// Get a slice of strings containing the current order
func (m OrderedMap) GetOrder() []string {
	m.lock.RLock()
	tmp := make([]string, len(m.order))
	copy(tmp, m.order)
	m.lock.RUnlock()
	return tmp
}

// Set a new order for this map, must contain all keys.
func (m *OrderedMap) SetOrder(order []string) error {
	if !compareOrder(m.order, order) {
		return errors.New("Provided order does not contain the same data as existing.")
	}
	m.lock.Lock()
	copy(m.order, order)
	m.lock.Unlock()
	return nil
}

// Get the index in the order of a specific key
func (m OrderedMap) IndexOf(key string) int {
	m.lock.RLock()
	index := -1
	for i := 0; i < len(m.order); i++ {
		if m.order[i] == key {
			index = i
		}
	}
	m.lock.RUnlock()
	return index
}

// Delete a specific key and all associated data from the map
func (m *OrderedMap) Delete(key string) {
	idx := m.IndexOf(key)

	m.lock.Lock()
	delete(m.data, key)
	tmp := make([]string, len(m.order))
	copy(tmp, m.order)
	m.order = make([]string, len(tmp))

	m.order = append(tmp[:idx], tmp[idx+1:]...)
	m.lock.Unlock()
}

// Get the total size of the map
func (m OrderedMap) Count() int {
	m.lock.RLock()
	cnt := len(m.data)
	m.lock.RUnlock()
	return cnt
}

// Compare two orders and determine if they have the same data even if not in the same order
func compareOrder(f []string, s []string) bool {
	// Check to see if the two slices have the same length, if not they obviously aren't the same
	if len(f) != len(s) {
		return false
	}

	// Let's copy our own slices, so we don't edit the originals
	tmpf := make([]string, len(f))
	tmps := make([]string, len(s))
	copy(tmpf, f)
	copy(tmps, s)

	// Now we'll sort them and go through each index to make sure it's the same
	sort.Strings(tmpf)
	sort.Strings(tmps)

	for i, v := range tmps {
		if tmpf[i] != v {
			return false
		}
	}

	return true
}
<|endoftext|>


<commit_before>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"strings"
	"time"

	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
)

var conn dbox.IConnection
var count int
var ratioTableName string

var (
	sourcetablename = "salespls-summary"
	calctablename   = "salespls-summary"
	desttablename   = "salespls-summary"
	t0              time.Time
	masters         = toolkit.M{}
	sgaalloc        = map[string]float64{
		"EXP": 0.08,
		"I4":  0.08,
		"I6":  0.105,
	}
)

type plalloc struct {
	ID               string `bson:"_id" json:"_id"`
	Key              string
	Key1, Key2, Key3 string
	Ref1             float64
	Current          float64
	Expect           float64
	Absorbed         float64
}

type allocmap map[string]*plalloc

var (
	plallocs    = allocmap{}
	advtotals   = allocmap{}
	advyears    = allocmap{}
	spgtotals   = allocmap{}
	promototals = allocmap{}
	disctotals  = allocmap{}
)

func main() {
	setinitialconnection()
	prepmastercalc()
	buildratio()
	processTable()
}

func buildratio() {
	connratio, _ := modules.GetDboxIConnection("db_godrej")
	defer connratio.Close()

	csr, _ := connratio.NewQuery().From("rawdatapl_ads30062016").Select().Cursor(nil)
	defer csr.Close()
	i := 0
	count := csr.Count()
	t0 := time.Now()
	mstone := 0
	for {
		mr := toolkit.M{}
		if ef := csr.Fetch(&mr, 1, false); ef != nil {
			break
		}
		i++
		makeProgressLog("Build ads ratio", i, count, 5, &mstone, t0)

		gdrjdate := mr.Get("gdrjdate", toolkit.M{}).(toolkit.M)
		fiscal := gdrjdate.GetString("fiscal")
		month := gdrjdate.GetInt("month")
		brand := mr.GetString("brand")
		value := -mr.GetFloat64("amountinidr")
		keyperiodbrand := toolkit.Sprintf("%s_%d_%s", fiscal, month, brand)
		adjustAllocs(&advtotals, keyperiodbrand, 0, value, 0, 0)
		adjustAllocs(&advyears, fiscal, 0, value, 0, 0)
	}

	ctrx, _ := connratio.NewQuery().From(calctablename).Select().Cursor(nil)
	i = 0
	count = ctrx.Count()
	t0 = time.Now()
	mstone = 0
	for {
		mr := toolkit.M{}
		if ef := ctrx.Fetch(&mr, 1, false); ef != nil {
			break
		}
		i++
		makeProgressLog("Finalizing", i, count, 5, &mstone, t0)

		key := mr.Get("key", toolkit.M{}).(toolkit.M)
		fiscal := key.GetString("date_fiscal")
		month := key.GetInt("date_month")
		brand := key.GetString("product_brand")
		if brand == "" {
			brand = "HIT"
		}

		keyperiodbrand := toolkit.Sprintf("%s_%d_%s", fiscal, month, brand)
		sales := mr.GetFloat64("PL8A")
		adjustAllocs(&advtotals, keyperiodbrand, 0, 0, 0, sales)
	}
}

func processTable() {
	connsave, _ := modules.GetDboxIConnection("db_godrej")
	defer connsave.Close()
	qsave := connsave.NewQuery().SetConfig("multiexec", true).From(desttablename).Save()

	connselect, _ := modules.GetDboxIConnection("db_godrej")
	defer connselect.Close()

	cursor, _ := connselect.NewQuery().
		From(calctablename).Select().Cursor(nil)
	defer cursor.Close()

	deletedids := []string{}

	i := 0
	count := cursor.Count()
	mstone := 0
	t0 = time.Now()
	for {
		mr := toolkit.M{}
		e := cursor.Fetch(&mr, 1, false)
		if e != nil || i >= count {
			break
		}
		i++
		makeProgressLog("Processing", i, count, 5, &mstone, t0)

		key := mr.Get("key", toolkit.M{}).(toolkit.M)
		fiscal := key.GetString("date_fiscal")
		period := key.GetInt("date_month")
		brand := key.GetString("product_brand")
		keyaccountid := key.GetString("customer_customergroup")

		//keyperiod := toolkit.Sprintf("%s_%d", fiscal, period)
		keyperiodaccount := toolkit.Sprintf("%s_%d_%s",
			fiscal, period, keyaccountid)
		keyperiodbrand := toolkit.Sprintf("%s_%d_%s", fiscal, period, brand)

		sales := mr.GetFloat64("PL8A")
		for k, _ := range mr {
			if isPL(k) {
				newv := float64(0)

				//--- discount
				if strings.HasPrefix(k, "PL7A") {
					disctotal := disctotals[keyperiodaccount]
					if disctotal != nil {
						newv = toolkit.Div(sales*disctotal.Expect,
							disctotal.Ref1)
					}
				} else
				//-- advertisement
				if strings.HasPrefix(k, "PL28") {
					advtotal := advtotals[keyperiodbrand]
					if advtotal != nil {
						newv = toolkit.Div(sales*advtotal.Expect, advtotal.Ref1)
						adjustAllocs(&advyears, fiscal, 0, 0, 0, newv)
					}
				} else
				//-- spg
				if strings.HasPrefix(k, "PL31") {
					total := spgtotals[keyperiodbrand]
					if total != nil {
						newv = sales * total.Expect / total.Ref1
					}
				} else
				//-- promo
				if strings.HasPrefix(k, "PL29A") {
					total := promototals[keyperiodbrand]
					if total != nil {
						newv = sales * total.Expect / total.Ref1
					}
				}

				mr.Set(k, newv)
			}
		}

		gdrj.CalcSum(mr, masters)

		isDeleted := true
		for k, v := range mr {
			if strings.HasPrefix(k, "PL") && v != 0 {
				isDeleted = false
			}
		}

		mrid := mr.GetString("_id")
		if isDeleted {
			deletedids = append(deletedids, mrid)
		} else {
			esave := qsave.Exec(toolkit.M{}.Set("data", mr))
			if esave != nil {
				toolkit.Printfn("Error: %s", esave.Error())
				return
			}
		}
	}

	//-- scale
	cursor.ResetFetch()
	i = 0
	count = cursor.Count()
	mstone = 0
	t0 = time.Now()
	for {
		mr := toolkit.M{}
		e := cursor.Fetch(&mr, 1, false)
		if e != nil || i >= count {
			break
		}
		i++
		makeProgressLog("Scale", i, count, 5, &mstone, t0)

		id := mr.GetString("_id")
		key := mr.Get("key", toolkit.M{}).(toolkit.M)
		fiscal := key.GetString("date_fiscal")

		if !toolkit.HasMember(deletedids, id) {
			for k, v := range mr {
				if isPL(k) {
					newv := v.(float64)
					//ads
					if strings.HasPrefix(k, "PL28") {
						total := advyears[fiscal]
						if total != nil {
							newv = toolkit.Div(v.(float64)*total.Expect,
								total.Ref1)
						}
					}
					mr.Set(k, newv)
				}
			}

			esave := qsave.Exec(toolkit.M{}.Set("data", mr))
			if esave != nil {
				toolkit.Printfn("Error: %s", esave.Error())
				return
			}
		}
	}

	toolkit.Printfn("Delete unneccessary data")
	for _, deletedid := range deletedids {
		connsave.NewQuery().From(desttablename).
			Where(dbox.Eq("_id", deletedid)).
			Delete().Exec(nil)
	}
}

func adjustAllocs(allocsmap *allocmap, key string, current, expect, absorbed, ref1 float64) {
	allocs := *allocsmap
	alloc := allocs[key]
	if alloc == nil {
		alloc = new(plalloc)
		alloc.Key = key
		alloc.ID = key
	}
	alloc.Current += current
	alloc.Expect += expect
	alloc.Ref1 += ref1
	alloc.Absorbed += absorbed
	allocs[key] = alloc
	*allocsmap = allocs
}

func makeProgressLog(reference string, i, count, step int, current *int, tstart time.Time) int {
	perstep := count * step / 100
	icurrent := *current
	if icurrent == 0 {
		icurrent = perstep
	}
	pct := i * 100 / count
	if i >= icurrent {
		toolkit.Printfn("Processing %s, %d of %d [%d pct] in %s",
			reference, i, count, pct, time.Since(tstart).String())
		icurrent += perstep
	}
	*current = icurrent
	return icurrent
}

func isPL(id string) bool {
	if strings.HasPrefix(id, "PL7A") ||
		strings.HasPrefix(id, "PL28") ||
		strings.HasPrefix(id, "PL29A") ||
		strings.HasPrefix(id, "PL31") {
		return true
	}
	return false
}

func buildmap(holder interface{},
	fnModel func() orm.IModel,
	filter *dbox.Filter,
	fnIter func(holder interface{}, obj interface{})) interface{} {
	crx, ecrx := gdrj.Find(fnModel(), filter, nil)
	if ecrx != nil {
		toolkit.Printfn("Cursor Error: %s", ecrx.Error())
		os.Exit(100)
	}
	defer crx.Close()
	for {
		s := fnModel()
		e := crx.Fetch(s, 1, false)
		if e != nil {
			break
		}
		fnIter(holder, s)
	}
	return holder
}

func prepmastercalc() {
	toolkit.Println("--> PL MODEL")
	masters.Set("plmodel", buildmap(map[string]*gdrj.PLModel{},
		func() orm.IModel {
			return new(gdrj.PLModel)
		},
		nil,
		func(holder, obj interface{}) {
			h := holder.(map[string]*gdrj.PLModel)
			o := obj.(*gdrj.PLModel)
			h[o.ID] = o
		}).(map[string]*gdrj.PLModel))
}

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}
<commit_msg>dont calc disc act<commit_after>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"strings"
	"time"

	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
)

var conn dbox.IConnection
var count int
var ratioTableName string

var (
	sourcetablename = "salespls-summary"
	calctablename   = "salespls-summary"
	desttablename   = "salespls-summary"
	t0              time.Time
	masters         = toolkit.M{}
	sgaalloc        = map[string]float64{
		"EXP": 0.08,
		"I4":  0.08,
		"I6":  0.105,
	}
)

type plalloc struct {
	ID               string `bson:"_id" json:"_id"`
	Key              string
	Key1, Key2, Key3 string
	Ref1             float64
	Current          float64
	Expect           float64
	Absorbed         float64
}

type allocmap map[string]*plalloc

var (
	plallocs    = allocmap{}
	advtotals   = allocmap{}
	advyears    = allocmap{}
	spgtotals   = allocmap{}
	promototals = allocmap{}
	disctotals  = allocmap{}
)

func main() {
	setinitialconnection()
	prepmastercalc()
	buildratio()
	processTable()
}

func buildratio() {
	connratio, _ := modules.GetDboxIConnection("db_godrej")
	defer connratio.Close()

	csr, _ := connratio.NewQuery().From("rawdatapl_ads30062016").Select().Cursor(nil)
	defer csr.Close()
	i := 0
	count := csr.Count()
	t0 := time.Now()
	mstone := 0
	for {
		mr := toolkit.M{}
		if ef := csr.Fetch(&mr, 1, false); ef != nil {
			break
		}
		i++
		makeProgressLog("Build ads ratio", i, count, 5, &mstone, t0)

		gdrjdate := mr.Get("gdrjdate", toolkit.M{}).(toolkit.M)
		fiscal := gdrjdate.GetString("fiscal")
		month := gdrjdate.GetInt("month")
		brand := mr.GetString("brand")
		value := -mr.GetFloat64("amountinidr")
		keyperiodbrand := toolkit.Sprintf("%s_%d_%s", fiscal, month, brand)
		adjustAllocs(&advtotals, keyperiodbrand, 0, value, 0, 0)
		adjustAllocs(&advyears, fiscal, 0, value, 0, 0)
	}

	ctrx, _ := connratio.NewQuery().From(calctablename).Select().Cursor(nil)
	i = 0
	count = ctrx.Count()
	t0 = time.Now()
	mstone = 0
	for {
		mr := toolkit.M{}
		if ef := ctrx.Fetch(&mr, 1, false); ef != nil {
			break
		}
		i++
		makeProgressLog("Finalizing", i, count, 5, &mstone, t0)

		key := mr.Get("key", toolkit.M{}).(toolkit.M)
		fiscal := key.GetString("date_fiscal")
		month := key.GetInt("date_month")
		brand := key.GetString("product_brand")
		if brand == "" {
			brand = "HIT"
		}

		keyperiodbrand := toolkit.Sprintf("%s_%d_%s", fiscal, month, brand)
		sales := mr.GetFloat64("PL8A")
		adjustAllocs(&advtotals, keyperiodbrand, 0, 0, 0, sales)
	}
}

func processTable() {
	connsave, _ := modules.GetDboxIConnection("db_godrej")
	defer connsave.Close()
	qsave := connsave.NewQuery().SetConfig("multiexec", true).From(desttablename).Save()

	connselect, _ := modules.GetDboxIConnection("db_godrej")
	defer connselect.Close()

	cursor, _ := connselect.NewQuery().
		From(calctablename).Select().Cursor(nil)
	defer cursor.Close()

	deletedids := []string{}

	i := 0
	count := cursor.Count()
	mstone := 0
	t0 = time.Now()
	for {
		mr := toolkit.M{}
		e := cursor.Fetch(&mr, 1, false)
		if e != nil || i >= count {
			break
		}
		i++
		makeProgressLog("Processing", i, count, 5, &mstone, t0)

		key := mr.Get("key", toolkit.M{}).(toolkit.M)
		fiscal := key.GetString("date_fiscal")
		period := key.GetInt("date_month")
		brand := key.GetString("product_brand")
		keyaccountid := key.GetString("customer_customergroup")

		//keyperiod := toolkit.Sprintf("%s_%d", fiscal, period)
		keyperiodaccount := toolkit.Sprintf("%s_%d_%s",
			fiscal, period, keyaccountid)
		keyperiodbrand := toolkit.Sprintf("%s_%d_%s", fiscal, period, brand)

		sales := mr.GetFloat64("PL8A")
		for k, _ := range mr {
			if isPL(k) {
				newv := float64(0)

				//--- discount
				if strings.HasPrefix(k, "PL7A") {
					disctotal := disctotals[keyperiodaccount]
					if disctotal != nil {
						newv = toolkit.Div(sales*disctotal.Expect,
							disctotal.Ref1)
					}
				} else
				//-- advertisement
				if strings.HasPrefix(k, "PL28") {
					advtotal := advtotals[keyperiodbrand]
					if advtotal != nil {
						newv = toolkit.Div(sales*advtotal.Expect, advtotal.Ref1)
						adjustAllocs(&advyears, fiscal, 0, 0, 0, newv)
					}
				} else
				//-- spg
				if strings.HasPrefix(k, "PL31") {
					total := spgtotals[keyperiodbrand]
					if total != nil {
						newv = sales * total.Expect / total.Ref1
					}
				} else
				//-- promo
				if strings.HasPrefix(k, "PL29A") {
					total := promototals[keyperiodbrand]
					if total != nil {
						newv = sales * total.Expect / total.Ref1
					}
				}

				mr.Set(k, newv)
			}
		}

		gdrj.CalcSum(mr, masters)

		isDeleted := true
		for k, v := range mr {
			if strings.HasPrefix(k, "PL") && v != 0 {
				isDeleted = false
			}
		}

		mrid := mr.GetString("_id")
		if isDeleted {
			deletedids = append(deletedids, mrid)
		} else {
			esave := qsave.Exec(toolkit.M{}.Set("data", mr))
			if esave != nil {
				toolkit.Printfn("Error: %s", esave.Error())
				return
			}
		}
	}

	//-- scale
	cursor.ResetFetch()
	i = 0
	count = cursor.Count()
	mstone = 0
	t0 = time.Now()
	for {
		mr := toolkit.M{}
		e := cursor.Fetch(&mr, 1, false)
		if e != nil || i >= count {
			break
		}
		i++
		makeProgressLog("Scale", i, count, 5, &mstone, t0)

		id := mr.GetString("_id")
		key := mr.Get("key", toolkit.M{}).(toolkit.M)
		fiscal := key.GetString("date_fiscal")

		if !toolkit.HasMember(deletedids, id) {
			for k, v := range mr {
				if isPL(k) {
					newv := v.(float64)
					//ads
					if strings.HasPrefix(k, "PL28") {
						total := advyears[fiscal]
						if total != nil {
							newv = toolkit.Div(v.(float64)*total.Expect,
								total.Ref1)
						}
					}
					mr.Set(k, newv)
				}
			}

			esave := qsave.Exec(toolkit.M{}.Set("data", mr))
			if esave != nil {
				toolkit.Printfn("Error: %s", esave.Error())
				return
			}
		}
	}

	toolkit.Printfn("Delete unneccessary data")
	for _, deletedid := range deletedids {
		connsave.NewQuery().From(desttablename).
			Where(dbox.Eq("_id", deletedid)).
			Delete().Exec(nil)
	}
}

func adjustAllocs(allocsmap *allocmap, key string, current, expect, absorbed, ref1 float64) {
	allocs := *allocsmap
	alloc := allocs[key]
	if alloc == nil {
		alloc = new(plalloc)
		alloc.Key = key
		alloc.ID = key
	}
	alloc.Current += current
	alloc.Expect += expect
	alloc.Ref1 += ref1
	alloc.Absorbed += absorbed
	allocs[key] = alloc
	*allocsmap = allocs
}

func makeProgressLog(reference string, i, count, step int, current *int, tstart time.Time) int {
	perstep := count * step / 100
	icurrent := *current
	if icurrent == 0 {
		icurrent = perstep
	}
	pct := i * 100 / count
	if i >= icurrent {
		toolkit.Printfn("Processing %s, %d of %d [%d pct] in %s",
			reference, i, count, pct, time.Since(tstart).String())
		icurrent += perstep
	}
	*current = icurrent
	return icurrent
}

func isPL(id string) bool {
	if
	//strings.HasPrefix(id, "PL7A") ||
	strings.HasPrefix(id, "PL28") ||
		strings.HasPrefix(id, "PL29A") ||
		strings.HasPrefix(id, "PL31") {
		return true
	}
	return false
}

func buildmap(holder interface{},
	fnModel func() orm.IModel,
	filter *dbox.Filter,
	fnIter func(holder interface{}, obj interface{})) interface{} {
	crx, ecrx := gdrj.Find(fnModel(), filter, nil)
	if ecrx != nil {
		toolkit.Printfn("Cursor Error: %s", ecrx.Error())
		os.Exit(100)
	}
	defer crx.Close()
	for {
		s := fnModel()
		e := crx.Fetch(s, 1, false)
		if e != nil {
			break
		}
		fnIter(holder, s)
	}
	return holder
}

func prepmastercalc() {
	toolkit.Println("--> PL MODEL")
	masters.Set("plmodel", buildmap(map[string]*gdrj.PLModel{},
		func() orm.IModel {
			return new(gdrj.PLModel)
		},
		nil,
		func(holder, obj interface{}) {
			h := holder.(map[string]*gdrj.PLModel)
			o := obj.(*gdrj.PLModel)
			h[o.ID] = o
		}).(map[string]*gdrj.PLModel))
}

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}
<|endoftext|>