<commit_before>package Transition

type ArcStandard struct {
	Relations []string
	gold      *Graph
	Oracle    *OracleClassifier
}

func (a *ArcStandard) Transition(from *Configuration, transition string) *Configuration {
	conf := from.Copy()
	// Transition System:
	// LA-r	(S|wi,	wj|B,	A) => (S   ,	wj|B,	A+{(wj,r,wi)})	if: != 0
	// RA-r	(S|wi, 	wj|B,	A) => (S   ,	wi|B, 	A+{(wi,r,wj)})
	// SH	(S   ,	wi|B, 	A) => (S|wi,	   B,	A)
	switch transition[:2] {
	case "LA":
		wi, _ := conf.Stack().Pop()
		if wi == 0 {
			panic("Attempted to LA the root")
		}
		wj, _ := conf.Queue().Peek()
		rel := transition[3:]
		newArc := &DepArc{wj, rel, wi}
		conf.Arcs().Push(newArc)
	case "RA":
		wi, _ := conf.Stack().Pop()
		wj, _ := conf.Queue().Pop()
		rel := transition[3:]
		newArc := &DepArc{wi, rel, wj}
		conf.Queue().Push(wi)
		conf.Arcs().Push(newArc)
	case "SH":
		wi := conf.Queue().Pop()
		conf.Stack().Push(wi)
	}
	conf.SetLastTransition(transition)
	return conf
}

func (a *ArcStandard) TransitionSet() []string {
	all := make([]string, 2*len(a.Relations)+1)
	for i, rel := range a.Relations {
		all[i] = "LA-" + rel
		all[i+len(a.Relations)] = "RA-" + rel
	}
	all[len(a.Relations)*2] = "SHIFT"
}

func (a *ArcStandard) TransitionTypes() []string {
	return [...]string{"LA-*", "RA-*", "SHIFT"}
}

func (a *ArcStandard) Projective() bool {
	return true
}

func (a *ArcStandard) Labeled() bool {
	return true
}

func (a *ArcStandard) Oracle() *Decision {
	if a.gold == nil {
		panic("Oracle can't make a decision without Gold data")
	}
	if a.Oracle == nil {
		a.Oracle = &OracleFunction{a.gold}
	}
	return a.Oracle
}

func (a *ArcStandard) SetGold(g *Gold) {
	a.gold = g
	if a.Oracle() != nil {
		a.Oracle().SetGold(a.gold)
	}
}

type OracleFunction struct {
	gold   *Graph
	arcSet *ArcSet
}

func (o *OracleFunction) SetGold(g *Gold) {
	o.gold = g
	o.arcSet = NewArcSet(g)
}

func (o *OracleFunction) GetTransition(c *Configuration) string {
	// Given Gd=(Vd,Ad) # gold dependencies
	// o(c = (S,B,A)) =
	// LA-r	if	(B[0],r,S[0]) in Ad
	// RA-r	if	(S[0],r,B[0]) in Ad; and for all w,r', if (B[0],r',w) in Ad then (B[0],r',w) in A
	// SH	otherwise
	bTop, bExists := c.Queue().Peek()
	sTop, sExists := c.Stack().Peek()
	if bExists && sExists {
		// test if should Left-Attach
		arcs, exists := o.arcSet.Get(&DepArc{bTop, "", sTop})
		if exists {
			return "LA-" + arcs[0].Relation
		}

		// test if should Right-Attach
		arcs, exists := o.arcSet.Get(&DepArc{sTop, "", bTop})
		if exists {
			reverseArcs, _ := o.arcSet.Get(&DepArc{bTop, "", -1})
			for _, arc := range reverseArcs {
				_, revExists := c.Arcs().Get(arc)
				if !revExists {
					return "SH"
				}
			}
			return "RA-" + arcs[0].Relation
		}
	}
	return "SH"
}
<commit_msg>Added informative comment on ArcStandard<commit_after>package Transition

type ArcStandard struct {
	Relations []string
	gold      *Graph
	Oracle    *OracleClassifier
}

func (a *ArcStandard) Transition(from *Configuration, transition string) *Configuration {
	conf := from.Copy()
	// Transition System:
	// LA-r	(S|wi,	wj|B,	A) => (S   ,	wj|B,	A+{(wj,r,wi)})	if: != 0
	// RA-r	(S|wi, 	wj|B,	A) => (S   ,	wi|B, 	A+{(wi,r,wj)})
	// SH	(S   ,	wi|B, 	A) => (S|wi,	   B,	A)
	switch transition[:2] {
	case "LA":
		wi, _ := conf.Stack().Pop()
		if wi == 0 {
			panic("Attempted to LA the root")
		}
		wj, _ := conf.Queue().Peek()
		rel := transition[3:]
		newArc := &DepArc{wj, rel, wi}
		conf.Arcs().Push(newArc)
	case "RA":
		wi, _ := conf.Stack().Pop()
		wj, _ := conf.Queue().Pop()
		rel := transition[3:]
		newArc := &DepArc{wi, rel, wj}
		conf.Queue().Push(wi)
		conf.Arcs().Push(newArc)
	case "SH":
		wi := conf.Queue().Pop()
		conf.Stack().Push(wi)
	}
	conf.SetLastTransition(transition)
	return conf
}

func (a *ArcStandard) TransitionSet() []string {
	all := make([]string, 2*len(a.Relations)+1)
	for i, rel := range a.Relations {
		all[i] = "LA-" + rel
		all[i+len(a.Relations)] = "RA-" + rel
	}
	all[len(a.Relations)*2] = "SHIFT"
}

func (a *ArcStandard) TransitionTypes() []string {
	return [...]string{"LA-*", "RA-*", "SHIFT"}
}

func (a *ArcStandard) Projective() bool {
	return true
}

func (a *ArcStandard) Labeled() bool {
	return true
}

func (a *ArcStandard) Oracle() *Decision {
	if a.gold == nil {
		panic("Oracle can't make a decision without Gold data")
	}
	if a.Oracle == nil {
		a.Oracle = &OracleFunction{a.gold}
	}
	return a.Oracle
}

func (a *ArcStandard) SetGold(g *Gold) {
	a.gold = g
	if a.Oracle() != nil {
		a.Oracle().SetGold(a.gold)
	}
}

type OracleFunction struct {
	gold   *Graph
	arcSet *ArcSet
}

func (o *OracleFunction) SetGold(g *Gold) {
	o.gold = g
	o.arcSet = NewArcSet(g)
}

func (o *OracleFunction) GetTransition(c *Configuration) string {
	// Given Gd=(Vd,Ad) # gold dependencies
	// o(c = (S,B,A)) =
	// LA-r	if	(B[0],r,S[0]) in Ad
	// RA-r	if	(S[0],r,B[0]) in Ad; and for all w,r', if (B[0],r',w) in Ad then (B[0],r',w) in A
	// SH	otherwise
	bTop, bExists := c.Queue().Peek()
	sTop, sExists := c.Stack().Peek()
	if bExists && sExists {
		// test if should Left-Attach
		arcs, exists := o.arcSet.Get(&DepArc{bTop, "", sTop})
		if exists {
			return "LA-" + arcs[0].Relation
		}

		// test if should Right-Attach
		arcs, exists := o.arcSet.Get(&DepArc{sTop, "", bTop})
		if exists {
			reverseArcs, _ := o.arcSet.Get(&DepArc{bTop, "", -1})
			// for all w,r', if (B[0],r',w) in Ad then (B[0],r',w) in A
			// otherwise, return SH
			for _, arc := range reverseArcs {
				_, revExists := c.Arcs().Get(arc)
				if !revExists {
					return "SH"
				}
			}
			return "RA-" + arcs[0].Relation
		}
	}
	return "SH"
}
<|endoftext|>


<commit_before>// Copyright 2019 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package lib provides utilities for runner.
package lib

import (
	"context"
	"encoding/csv"
	"fmt"
	"io"
	"os"
	"sort"
	"strings"
	"testing"
	"time"

	"gvisor.dev/gvisor/pkg/log"
	"gvisor.dev/gvisor/pkg/test/dockerutil"
	"gvisor.dev/gvisor/pkg/test/testutil"
)

// RunTests is a helper that is called by main. It exists so that we can run
// defered functions before exiting. It returns an exit code that should be
// passed to os.Exit.
func RunTests(lang, image, excludeFile string, partitionNum, totalPartitions, batchSize int, timeout time.Duration) int {
	if partitionNum <= 0 || totalPartitions <= 0 || partitionNum > totalPartitions {
		fmt.Fprintf(os.Stderr, "invalid partition %d of %d", partitionNum, totalPartitions)
		return 1
	}

	// TODO(gvisor.dev/issue/1624): Remove those tests from all exclude lists
	// that only fail with VFS1.

	// Get tests to exclude.
	excludes, err := getExcludes(excludeFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error getting exclude list: %s\n", err.Error())
		return 1
	}

	// Construct the shared docker instance.
	ctx := context.Background()
	d := dockerutil.MakeContainer(ctx, testutil.DefaultLogger(lang))
	defer d.CleanUp(ctx)

	if err := testutil.TouchShardStatusFile(); err != nil {
		fmt.Fprintf(os.Stderr, "error touching status shard file: %v\n", err)
		return 1
	}

	// Get a slice of tests to run. This will also start a single Docker
	// container that will be used to run each test. The final test will
	// stop the Docker container.
	tests, err := getTests(ctx, d, lang, image, partitionNum, totalPartitions, batchSize, timeout, excludes)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
		return 1
	}

	m := testing.MainStart(testDeps{}, tests, nil, nil)
	return m.Run()
}

// getTests executes all tests as table tests.
func getTests(ctx context.Context, d *dockerutil.Container, lang, image string, partitionNum, totalPartitions, batchSize int, timeout time.Duration, excludes map[string]struct{}) ([]testing.InternalTest, error) {
	// Start the container.
	opts := dockerutil.RunOpts{
		Image: fmt.Sprintf("runtimes/%s", image),
	}
	d.CopyFiles(&opts, "/proctor", "test/runtimes/proctor/proctor")
	if err := d.Spawn(ctx, opts, "/proctor/proctor", "--pause"); err != nil {
		return nil, fmt.Errorf("docker run failed: %v", err)
	}

	// Get a list of all tests in the image.
	list, err := d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", lang, "--list")
	if err != nil {
		return nil, fmt.Errorf("docker exec failed: %v", err)
	}

	// Calculate a subset of tests to run corresponding to the current
	// shard.
	tests := strings.Fields(list)
	sort.Strings(tests)

	partitionSize := len(tests) / totalPartitions
	if partitionNum == totalPartitions {
		tests = tests[(partitionNum-1)*partitionSize:]
	} else {
		tests = tests[(partitionNum-1)*partitionSize : partitionNum*partitionSize]
	}

	indices, err := testutil.TestIndicesForShard(len(tests))
	if err != nil {
		return nil, fmt.Errorf("TestsForShard() failed: %v", err)
	}

	var itests []testing.InternalTest
	for i := 0; i < len(indices); i += batchSize {
		var tcs []string
		end := i + batchSize
		if end > len(indices) {
			end = len(indices)
		}
		for _, tc := range indices[i:end] {
			// Add test if not excluded.
			if _, ok := excludes[tests[tc]]; ok {
				log.Infof("Skipping test case %s\n", tests[tc])
				continue
			}
			tcs = append(tcs, tests[tc])
		}
		itests = append(itests, testing.InternalTest{
			Name: strings.Join(tcs, ", "),
			F: func(t *testing.T) {
				var (
					now    = time.Now()
					done   = make(chan struct{})
					output string
					err    error
				)

				state, err := d.Status(ctx)
				if err != nil {
					t.Fatalf("Could not find container status: %v", err)
				}
				if !state.Running {
					t.Fatalf("container is not running: state = %s", state.Status)
				}

				go func() {
					output, err = d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", lang, "--tests", strings.Join(tcs, ","))
					close(done)
				}()

				select {
				case <-done:
					if err == nil {
						fmt.Printf("PASS: (%v) %d tests passed\n", time.Since(now), len(tcs))
						return
					}
					t.Errorf("FAIL: (%v):\nBatch:\n%s\nOutput:\n%s\n", time.Since(now), strings.Join(tcs, "\n"), output)
				case <-time.After(timeout):
					t.Errorf("TIMEOUT: (%v):\nBatch:\n%s\nOutput:\n%s\n", time.Since(now), strings.Join(tcs, "\n"), output)
				}
			},
		})
	}

	return itests, nil
}

// getBlacklist reads the exclude file and returns a set of test names to
// exclude.
func getExcludes(excludeFile string) (map[string]struct{}, error) {
	excludes := make(map[string]struct{})
	if excludeFile == "" {
		return excludes, nil
	}
	f, err := os.Open(excludeFile)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	r := csv.NewReader(f)

	// First line is header. Skip it.
	if _, err := r.Read(); err != nil {
		return nil, err
	}

	for {
		record, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		excludes[record[0]] = struct{}{}
	}
	return excludes, nil
}

// testDeps implements testing.testDeps (an unexported interface), and is
// required to use testing.MainStart.
type testDeps struct{}

func (f testDeps) MatchString(a, b string) (bool, error)       { return a == b, nil }
func (f testDeps) StartCPUProfile(io.Writer) error             { return nil }
func (f testDeps) StopCPUProfile()                             {}
func (f testDeps) WriteProfileTo(string, io.Writer, int) error { return nil }
func (f testDeps) ImportPath() string                          { return "" }
func (f testDeps) StartTestLog(io.Writer)                      {}
func (f testDeps) StopTestLog() error                          { return nil }
<commit_msg>Skip generating an empty (broken) test case.<commit_after>// Copyright 2019 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package lib provides utilities for runner.
package lib

import (
	"context"
	"encoding/csv"
	"fmt"
	"io"
	"os"
	"sort"
	"strings"
	"testing"
	"time"

	"gvisor.dev/gvisor/pkg/log"
	"gvisor.dev/gvisor/pkg/test/dockerutil"
	"gvisor.dev/gvisor/pkg/test/testutil"
)

// RunTests is a helper that is called by main. It exists so that we can run
// defered functions before exiting. It returns an exit code that should be
// passed to os.Exit.
func RunTests(lang, image, excludeFile string, partitionNum, totalPartitions, batchSize int, timeout time.Duration) int {
	if partitionNum <= 0 || totalPartitions <= 0 || partitionNum > totalPartitions {
		fmt.Fprintf(os.Stderr, "invalid partition %d of %d", partitionNum, totalPartitions)
		return 1
	}

	// TODO(gvisor.dev/issue/1624): Remove those tests from all exclude lists
	// that only fail with VFS1.

	// Get tests to exclude.
	excludes, err := getExcludes(excludeFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error getting exclude list: %s\n", err.Error())
		return 1
	}

	// Construct the shared docker instance.
	ctx := context.Background()
	d := dockerutil.MakeContainer(ctx, testutil.DefaultLogger(lang))
	defer d.CleanUp(ctx)

	if err := testutil.TouchShardStatusFile(); err != nil {
		fmt.Fprintf(os.Stderr, "error touching status shard file: %v\n", err)
		return 1
	}

	// Get a slice of tests to run. This will also start a single Docker
	// container that will be used to run each test. The final test will
	// stop the Docker container.
	tests, err := getTests(ctx, d, lang, image, partitionNum, totalPartitions, batchSize, timeout, excludes)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
		return 1
	}

	m := testing.MainStart(testDeps{}, tests, nil, nil)
	return m.Run()
}

// getTests executes all tests as table tests.
func getTests(ctx context.Context, d *dockerutil.Container, lang, image string, partitionNum, totalPartitions, batchSize int, timeout time.Duration, excludes map[string]struct{}) ([]testing.InternalTest, error) {
	// Start the container.
	opts := dockerutil.RunOpts{
		Image: fmt.Sprintf("runtimes/%s", image),
	}
	d.CopyFiles(&opts, "/proctor", "test/runtimes/proctor/proctor")
	if err := d.Spawn(ctx, opts, "/proctor/proctor", "--pause"); err != nil {
		return nil, fmt.Errorf("docker run failed: %v", err)
	}

	// Get a list of all tests in the image.
	list, err := d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", lang, "--list")
	if err != nil {
		return nil, fmt.Errorf("docker exec failed: %v", err)
	}

	// Calculate a subset of tests to run corresponding to the current
	// shard.
	tests := strings.Fields(list)
	sort.Strings(tests)

	partitionSize := len(tests) / totalPartitions
	if partitionNum == totalPartitions {
		tests = tests[(partitionNum-1)*partitionSize:]
	} else {
		tests = tests[(partitionNum-1)*partitionSize : partitionNum*partitionSize]
	}

	indices, err := testutil.TestIndicesForShard(len(tests))
	if err != nil {
		return nil, fmt.Errorf("TestsForShard() failed: %v", err)
	}

	var itests []testing.InternalTest
	for i := 0; i < len(indices); i += batchSize {
		var tcs []string
		end := i + batchSize
		if end > len(indices) {
			end = len(indices)
		}
		for _, tc := range indices[i:end] {
			// Add test if not excluded.
			if _, ok := excludes[tests[tc]]; ok {
				log.Infof("Skipping test case %s\n", tests[tc])
				continue
			}
			tcs = append(tcs, tests[tc])
		}
		if len(tcs) == 0 {
			// No tests to add to this batch.
			continue
		}
		itests = append(itests, testing.InternalTest{
			Name: strings.Join(tcs, ", "),
			F: func(t *testing.T) {
				var (
					now    = time.Now()
					done   = make(chan struct{})
					output string
					err    error
				)

				state, err := d.Status(ctx)
				if err != nil {
					t.Fatalf("Could not find container status: %v", err)
				}
				if !state.Running {
					t.Fatalf("container is not running: state = %s", state.Status)
				}

				go func() {
					output, err = d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", lang, "--tests", strings.Join(tcs, ","))
					close(done)
				}()

				select {
				case <-done:
					if err == nil {
						fmt.Printf("PASS: (%v) %d tests passed\n", time.Since(now), len(tcs))
						return
					}
					t.Errorf("FAIL: (%v):\nBatch:\n%s\nOutput:\n%s\n", time.Since(now), strings.Join(tcs, "\n"), output)
				case <-time.After(timeout):
					t.Errorf("TIMEOUT: (%v):\nBatch:\n%s\nOutput:\n%s\n", time.Since(now), strings.Join(tcs, "\n"), output)
				}
			},
		})
	}

	return itests, nil
}

// getBlacklist reads the exclude file and returns a set of test names to
// exclude.
func getExcludes(excludeFile string) (map[string]struct{}, error) {
	excludes := make(map[string]struct{})
	if excludeFile == "" {
		return excludes, nil
	}
	f, err := os.Open(excludeFile)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	r := csv.NewReader(f)

	// First line is header. Skip it.
	if _, err := r.Read(); err != nil {
		return nil, err
	}

	for {
		record, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		excludes[record[0]] = struct{}{}
	}
	return excludes, nil
}

// testDeps implements testing.testDeps (an unexported interface), and is
// required to use testing.MainStart.
type testDeps struct{}

func (f testDeps) MatchString(a, b string) (bool, error)       { return a == b, nil }
func (f testDeps) StartCPUProfile(io.Writer) error             { return nil }
func (f testDeps) StopCPUProfile()                             {}
func (f testDeps) WriteProfileTo(string, io.Writer, int) error { return nil }
func (f testDeps) ImportPath() string                          { return "" }
func (f testDeps) StartTestLog(io.Writer)                      {}
func (f testDeps) StopTestLog() error                          { return nil }
<|endoftext|>


<commit_before>package routing

// TODO SEE https://github.com/jbenet/node-ipfs/blob/master/submodules/ipfs-routing/index.js
<commit_msg>routing interface<commit_after>package routing

import (
  "time"
  peer "github.com/jbenet/go-ipfs/peer"
  u "github.com/jbenet/go-ipfs/util"
)

// IpfsRouting is the routing module interface
// It is implemented by things like DHTs, etc.
type IpfsRouting interface {

  // Basic Put/Get

  // PutValue adds value corresponding to given Key.
  PutValue(key u.Key, value []byte) (error)

  // GetValue searches for the value corresponding to given Key.
  GetValue(key u.Key, timeout time.Duration) ([]byte, error)

  // Value provider layer of indirection.
  // This is what DSHTs (Coral and MainlineDHT) do to store large values in a DHT.

  // Announce that this node can provide value for given key
  Provide(key u.Key) (error)

  // FindProviders searches for peers who can provide the value for given key.
  FindProviders(key u.Key, timeout time.Duration) (*peer.Peer, error)

  // Find specific Peer

  // FindPeer searches for a peer with given ID.
  FindPeer(id peer.ID, timeout time.Duration) (*peer.Peer, error)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"fmt"
	"io"

	"github.com/golang/protobuf/proto"
	"k8s.io/test-infra/testgrid/config"
	"sigs.k8s.io/yaml"
)

// Config includes config and defaults to apply on unspecified values.
type Config struct {
	config        *config.Configuration
	defaultConfig *config.DefaultConfiguration
}

// MissingFieldError is an error that includes the missing field.
type MissingFieldError struct {
	Field string
}

func (e MissingFieldError) Error() string {
	return fmt.Sprintf("field missing or unset: %s", e.Field)
}

// ReconcileTestGroup sets unfilled currentTestGroup fields to the corresponding defaultTestGroup value.
func ReconcileTestGroup(currentTestGroup *config.TestGroup, defaultTestGroup *config.TestGroup) {
	if currentTestGroup.DaysOfResults == 0 {
		currentTestGroup.DaysOfResults = defaultTestGroup.DaysOfResults
	}

	if currentTestGroup.TestsNamePolicy == config.TestGroup_TESTS_NAME_MIN {
		currentTestGroup.TestsNamePolicy = defaultTestGroup.TestsNamePolicy
	}

	if currentTestGroup.IgnorePending == false {
		currentTestGroup.IgnorePending = defaultTestGroup.IgnorePending
	}

	if currentTestGroup.ColumnHeader == nil {
		currentTestGroup.ColumnHeader = defaultTestGroup.ColumnHeader
	}

	if currentTestGroup.NumColumnsRecent == 0 {
		currentTestGroup.NumColumnsRecent = defaultTestGroup.NumColumnsRecent
	}

	if currentTestGroup.AlertStaleResultsHours == 0 {
		currentTestGroup.AlertStaleResultsHours = defaultTestGroup.AlertStaleResultsHours
	}

	if currentTestGroup.NumFailuresToAlert == 0 {
		currentTestGroup.NumFailuresToAlert = defaultTestGroup.NumFailuresToAlert
	}
	if currentTestGroup.CodeSearchPath == "" {
		currentTestGroup.CodeSearchPath = defaultTestGroup.CodeSearchPath
	}
	if currentTestGroup.NumPassesToDisableAlert == 0 {
		currentTestGroup.NumPassesToDisableAlert = defaultTestGroup.NumPassesToDisableAlert
	}
	// is_external and user_kubernetes_client should always be true
	currentTestGroup.IsExternal = true
	currentTestGroup.UseKubernetesClient = true
}

// ReconcileDashboardTab sets unfilled currentTab fields to the corresponding defaultTab value.
func ReconcileDashboardTab(currentTab *config.DashboardTab, defaultTab *config.DashboardTab) {
	if currentTab.BugComponent == 0 {
		currentTab.BugComponent = defaultTab.BugComponent
	}

	if currentTab.CodeSearchPath == "" {
		currentTab.CodeSearchPath = defaultTab.CodeSearchPath
	}

	if currentTab.NumColumnsRecent == 0 {
		currentTab.NumColumnsRecent = defaultTab.NumColumnsRecent
	}

	if currentTab.OpenTestTemplate == nil {
		currentTab.OpenTestTemplate = defaultTab.OpenTestTemplate
	}

	if currentTab.FileBugTemplate == nil {
		currentTab.FileBugTemplate = defaultTab.FileBugTemplate
	}

	if currentTab.AttachBugTemplate == nil {
		currentTab.AttachBugTemplate = defaultTab.AttachBugTemplate
	}

	if currentTab.ResultsText == "" {
		currentTab.ResultsText = defaultTab.ResultsText
	}

	if currentTab.ResultsUrlTemplate == nil {
		currentTab.ResultsUrlTemplate = defaultTab.ResultsUrlTemplate
	}

	if currentTab.CodeSearchUrlTemplate == nil {
		currentTab.CodeSearchUrlTemplate = defaultTab.CodeSearchUrlTemplate
	}

	if currentTab.AlertOptions == nil {
		currentTab.AlertOptions = defaultTab.AlertOptions
	}
}

// updateDefaults reads the default configuration from yamlData and updates the
// defaultConfig in c.
//
// Returns an error if the defaultConfig is missing from yamlData
func (c *Config) updateDefaults(yamlData []byte) error {
	newDefaults := &config.DefaultConfiguration{}
	err := yaml.Unmarshal(yamlData, newDefaults)
	if err != nil {
		return err
	}

	if newDefaults == nil {
		return errors.New("DefaultConfiguration is nil after unmarshalling")
	}

	if newDefaults.DefaultTestGroup == nil {
		return MissingFieldError{"DefaultTestGroup"}
	}

	if newDefaults.DefaultDashboardTab == nil {
		return MissingFieldError{"DefaultDashboardTab"}
	}

	c.defaultConfig = newDefaults
	return nil
}

// Update reads the config in yamlData and updates the config in c.
// If yamlData does not contain any defaults, an error is returned without effect
func (c *Config) Update(yamlData []byte) error {
	if err := c.updateDefaults(yamlData); err != nil {
		return err
	}

	curConfig := &config.Configuration{}
	if err := yaml.Unmarshal(yamlData, curConfig); err != nil {
		return err
	}

	if c.config == nil {
		c.config = &config.Configuration{}
	}

	for _, testgroup := range curConfig.TestGroups {
		ReconcileTestGroup(testgroup, c.defaultConfig.DefaultTestGroup)
		c.config.TestGroups = append(c.config.TestGroups, testgroup)
	}

	for _, dashboard := range curConfig.Dashboards {
		// validate dashboard tabs
		for _, dashboardtab := range dashboard.DashboardTab {
			ReconcileDashboardTab(dashboardtab, c.defaultConfig.DefaultDashboardTab)
		}
		c.config.Dashboards = append(c.config.Dashboards, dashboard)
	}

	for _, dashboardGroup := range curConfig.DashboardGroups {
		c.config.DashboardGroups = append(c.config.DashboardGroups, dashboardGroup)
	}

	return nil
}

// validate checks that a configuration is well-formed, having test groups and dashboards set.
func (c *Config) validate() error {
	if c.config == nil {
		return errors.New("Configuration unset")
	}
	if len(c.config.TestGroups) == 0 {
		return MissingFieldError{"TestGroups"}
	}
	if len(c.config.Dashboards) == 0 {
		return MissingFieldError{"Dashboards"}
	}

	return nil
}

// MarshalText writes a text version of the parsed configuration to the supplied io.Writer.
// Returns an error if config is invalid or writing failed.
func (c *Config) MarshalText(w io.Writer) error {
	if err := c.validate(); err != nil {
		return err
	}
	return proto.MarshalText(w, c.config)
}

// MarshalBytes returns the wire-encoded protobuf data for the parsed configuration.
// Returns an error if config is invalid or encoding failed.
func (c *Config) MarshalBytes() ([]byte, error) {
	if err := c.validate(); err != nil {
		return nil, err
	}
	return proto.Marshal(c.config)
}

// Raw returns the raw protocol buffer for the parsed configuration after validation.
// Returns an error if validation fails.
func (c *Config) Raw() (*config.Configuration, error) {
	if err := c.validate(); err != nil {
		return nil, err
	}
	return c.config, nil
}
<commit_msg>Restoration of global testgrid default behavior<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"fmt"
	"io"

	"github.com/golang/protobuf/proto"
	"k8s.io/test-infra/testgrid/config"
	"sigs.k8s.io/yaml"
)

// Config includes config and defaults to apply on unspecified values.
type Config struct {
	config        *config.Configuration
	defaultConfig *config.DefaultConfiguration
}

// MissingFieldError is an error that includes the missing field.
type MissingFieldError struct {
	Field string
}

func (e MissingFieldError) Error() string {
	return fmt.Sprintf("field missing or unset: %s", e.Field)
}

// ReconcileTestGroup sets unfilled currentTestGroup fields to the corresponding defaultTestGroup value.
func ReconcileTestGroup(currentTestGroup *config.TestGroup, defaultTestGroup *config.TestGroup) {
	if currentTestGroup.DaysOfResults == 0 {
		currentTestGroup.DaysOfResults = defaultTestGroup.DaysOfResults
	}

	if currentTestGroup.TestsNamePolicy == config.TestGroup_TESTS_NAME_MIN {
		currentTestGroup.TestsNamePolicy = defaultTestGroup.TestsNamePolicy
	}

	if currentTestGroup.IgnorePending == false {
		currentTestGroup.IgnorePending = defaultTestGroup.IgnorePending
	}

	if currentTestGroup.ColumnHeader == nil {
		currentTestGroup.ColumnHeader = defaultTestGroup.ColumnHeader
	}

	if currentTestGroup.NumColumnsRecent == 0 {
		currentTestGroup.NumColumnsRecent = defaultTestGroup.NumColumnsRecent
	}

	if currentTestGroup.AlertStaleResultsHours == 0 {
		currentTestGroup.AlertStaleResultsHours = defaultTestGroup.AlertStaleResultsHours
	}

	if currentTestGroup.NumFailuresToAlert == 0 {
		currentTestGroup.NumFailuresToAlert = defaultTestGroup.NumFailuresToAlert
	}
	if currentTestGroup.CodeSearchPath == "" {
		currentTestGroup.CodeSearchPath = defaultTestGroup.CodeSearchPath
	}
	if currentTestGroup.NumPassesToDisableAlert == 0 {
		currentTestGroup.NumPassesToDisableAlert = defaultTestGroup.NumPassesToDisableAlert
	}
	// is_external and user_kubernetes_client should always be true
	currentTestGroup.IsExternal = true
	currentTestGroup.UseKubernetesClient = true
}

// ReconcileDashboardTab sets unfilled currentTab fields to the corresponding defaultTab value.
func ReconcileDashboardTab(currentTab *config.DashboardTab, defaultTab *config.DashboardTab) {
	if currentTab.BugComponent == 0 {
		currentTab.BugComponent = defaultTab.BugComponent
	}

	if currentTab.CodeSearchPath == "" {
		currentTab.CodeSearchPath = defaultTab.CodeSearchPath
	}

	if currentTab.NumColumnsRecent == 0 {
		currentTab.NumColumnsRecent = defaultTab.NumColumnsRecent
	}

	if currentTab.OpenTestTemplate == nil {
		currentTab.OpenTestTemplate = defaultTab.OpenTestTemplate
	}

	if currentTab.FileBugTemplate == nil {
		currentTab.FileBugTemplate = defaultTab.FileBugTemplate
	}

	if currentTab.AttachBugTemplate == nil {
		currentTab.AttachBugTemplate = defaultTab.AttachBugTemplate
	}

	if currentTab.ResultsText == "" {
		currentTab.ResultsText = defaultTab.ResultsText
	}

	if currentTab.ResultsUrlTemplate == nil {
		currentTab.ResultsUrlTemplate = defaultTab.ResultsUrlTemplate
	}

	if currentTab.CodeSearchUrlTemplate == nil {
		currentTab.CodeSearchUrlTemplate = defaultTab.CodeSearchUrlTemplate
	}

	if currentTab.AlertOptions == nil {
		currentTab.AlertOptions = defaultTab.AlertOptions
	}
}

// updateDefaults reads any default configuration from yamlData and updates the
// defaultConfig in c.
//
// Returns an error if the defaultConfig remains unset.
func (c *Config) updateDefaults(yamlData []byte) error {
	newDefaults := &config.DefaultConfiguration{}
	err := yaml.Unmarshal(yamlData, newDefaults)
	if err != nil {
		return err
	}

	if c.defaultConfig == nil {
		c.defaultConfig = newDefaults
	} else {
		if newDefaults.DefaultTestGroup != nil {
			c.defaultConfig.DefaultTestGroup = newDefaults.DefaultTestGroup
		}
		if newDefaults.DefaultDashboardTab != nil {
			c.defaultConfig.DefaultDashboardTab = newDefaults.DefaultDashboardTab
		}
	}

	if c.defaultConfig.DefaultTestGroup == nil {
		return MissingFieldError{"DefaultTestGroup"}
	}
	if c.defaultConfig.DefaultDashboardTab == nil {
		return MissingFieldError{"DefaultDashboardTab"}
	}
	return nil
}

// Update reads the config in yamlData and updates the config in c.
// If yamlData does not contain any defaults, the defaults from a
// previous call to Update are used instead.
func (c *Config) Update(yamlData []byte) error {
	if err := c.updateDefaults(yamlData); err != nil {
		return err
	}

	curConfig := &config.Configuration{}
	if err := yaml.Unmarshal(yamlData, curConfig); err != nil {
		return err
	}

	if c.config == nil {
		c.config = &config.Configuration{}
	}

	for _, testgroup := range curConfig.TestGroups {
		ReconcileTestGroup(testgroup, c.defaultConfig.DefaultTestGroup)
		c.config.TestGroups = append(c.config.TestGroups, testgroup)
	}

	for _, dashboard := range curConfig.Dashboards {
		// validate dashboard tabs
		for _, dashboardtab := range dashboard.DashboardTab {
			ReconcileDashboardTab(dashboardtab, c.defaultConfig.DefaultDashboardTab)
		}
		c.config.Dashboards = append(c.config.Dashboards, dashboard)
	}

	for _, dashboardGroup := range curConfig.DashboardGroups {
		c.config.DashboardGroups = append(c.config.DashboardGroups, dashboardGroup)
	}

	return nil
}

// validate checks that a configuration is well-formed, having test groups and dashboards set.
func (c *Config) validate() error {
	if c.config == nil {
		return errors.New("Configuration unset")
	}
	if len(c.config.TestGroups) == 0 {
		return MissingFieldError{"TestGroups"}
	}
	if len(c.config.Dashboards) == 0 {
		return MissingFieldError{"Dashboards"}
	}

	return nil
}

// MarshalText writes a text version of the parsed configuration to the supplied io.Writer.
// Returns an error if config is invalid or writing failed.
func (c *Config) MarshalText(w io.Writer) error {
	if err := c.validate(); err != nil {
		return err
	}
	return proto.MarshalText(w, c.config)
}

// MarshalBytes returns the wire-encoded protobuf data for the parsed configuration.
// Returns an error if config is invalid or encoding failed.
func (c *Config) MarshalBytes() ([]byte, error) {
	if err := c.validate(); err != nil {
		return nil, err
	}
	return proto.Marshal(c.config)
}

// Raw returns the raw protocol buffer for the parsed configuration after validation.
// Returns an error if validation fails.
func (c *Config) Raw() (*config.Configuration, error) {
	if err := c.validate(); err != nil {
		return nil, err
	}
	return c.config, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package storage

import (
	"github.com/juju/cmd"
	"github.com/juju/errors"
	"github.com/juju/utils/keyvalues"
	"launchpad.net/gnuflag"
)

const PoolCreateCommandDoc = `
Create or define a storage pool.

Pools are a mechanism for administrators to define sources of storage that
they will use to satisfy service storage requirements.

A single pool might be used for storage from units of many different services -
it is a resource from which different stores may be drawn.

A pool describes provider-specific parameters for creating storage,
such as performance (e.g. IOPS), media type (e.g. magnetic vs. SSD),
or durability.

For many providers, there will be a shared resource
where storage can be requested (e.g. EBS in amazon).
Creating pools there maps provider specific settings
into named resources that can be used during deployment.

Pools defined at the environment level are easily reused across services.

options:
    -e, --environment (= "")
        juju environment to operate in
    -o, --output (= "")
        specify an output file
    <name>
        pool name
    <provider type>
        pool provider type
    <key>=<value> (<key>=<value> ...)
        pool configuration attributes as space-separated pairs,
        for e.g. tags, size, path, etc...
`

// PoolCreateCommand lists storage pools.
type PoolCreateCommand struct {
	PoolCommandBase
	poolName string
	// TODO(anastasiamac 2015-01-29) type will need to become optional
	// if type is unspecified, use the environment's default provider type
	provider string
	attrs    map[string]interface{}
}

// Init implements Command.Init.
func (c *PoolCreateCommand) Init(args []string) (err error) {
	if len(args) < 3 {
		return errors.New("pool creation requires names, provider type and attrs for configuration")
	}

	c.poolName = args[0]
	c.provider = args[1]

	options, err := keyvalues.Parse(args[2:], false)
	if err != nil {
		return err
	}

	if len(options) == 0 {
		return errors.New("pool creation requires attrs for configuration")
	}
	c.attrs = make(map[string]interface{})
	for key, value := range options {
		c.attrs[key] = value
	}
	return nil
}

// Info implements Command.Info.
func (c *PoolCreateCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "create",
		Purpose: "create storage pool",
		Doc:     PoolCreateCommandDoc,
	}
}

// SetFlags implements Command.SetFlags.
func (c *PoolCreateCommand) SetFlags(f *gnuflag.FlagSet) {
	c.StorageCommandBase.SetFlags(f)
}

// Run implements Command.Run.
func (c *PoolCreateCommand) Run(ctx *cmd.Context) (err error) {
	api, err := getPoolCreateAPI(c)
	if err != nil {
		return err
	}
	defer api.Close()

	return api.CreatePool(c.poolName, c.provider, c.attrs)
}

var (
	getPoolCreateAPI = (*PoolCreateCommand).getPoolCreateAPI
)

// PoolCreateAPI defines the API methods that pool create command uses.
type PoolCreateAPI interface {
	Close() error
	CreatePool(pname, ptype string, pconfig map[string]interface{}) error
}

func (c *PoolCreateCommand) getPoolCreateAPI() (PoolCreateAPI, error) {
	return c.NewStorageAPI()
}
<commit_msg>cmd/juju/storage: add args to "pool create" usage<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package storage

import (
	"github.com/juju/cmd"
	"github.com/juju/errors"
	"github.com/juju/utils/keyvalues"
	"launchpad.net/gnuflag"
)

const PoolCreateCommandDoc = `
Create or define a storage pool.

Pools are a mechanism for administrators to define sources of storage that
they will use to satisfy service storage requirements.

A single pool might be used for storage from units of many different services -
it is a resource from which different stores may be drawn.

A pool describes provider-specific parameters for creating storage,
such as performance (e.g. IOPS), media type (e.g. magnetic vs. SSD),
or durability.

For many providers, there will be a shared resource
where storage can be requested (e.g. EBS in amazon).
Creating pools there maps provider specific settings
into named resources that can be used during deployment.

Pools defined at the environment level are easily reused across services.

options:
    -e, --environment (= "")
        juju environment to operate in
    -o, --output (= "")
        specify an output file
    <name>
        pool name
    <provider type>
        pool provider type
    <key>=<value> (<key>=<value> ...)
        pool configuration attributes as space-separated pairs,
        for e.g. tags, size, path, etc...
`

// PoolCreateCommand lists storage pools.
type PoolCreateCommand struct {
	PoolCommandBase
	poolName string
	// TODO(anastasiamac 2015-01-29) type will need to become optional
	// if type is unspecified, use the environment's default provider type
	provider string
	attrs    map[string]interface{}
}

// Init implements Command.Init.
func (c *PoolCreateCommand) Init(args []string) (err error) {
	if len(args) < 3 {
		return errors.New("pool creation requires names, provider type and attrs for configuration")
	}

	c.poolName = args[0]
	c.provider = args[1]

	options, err := keyvalues.Parse(args[2:], false)
	if err != nil {
		return err
	}

	if len(options) == 0 {
		return errors.New("pool creation requires attrs for configuration")
	}
	c.attrs = make(map[string]interface{})
	for key, value := range options {
		c.attrs[key] = value
	}
	return nil
}

// Info implements Command.Info.
func (c *PoolCreateCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "create",
		Args:    "<name> <provider> [<key>=<value> [<key>=<value>...]]",
		Purpose: "create storage pool",
		Doc:     PoolCreateCommandDoc,
	}
}

// SetFlags implements Command.SetFlags.
func (c *PoolCreateCommand) SetFlags(f *gnuflag.FlagSet) {
	c.StorageCommandBase.SetFlags(f)
}

// Run implements Command.Run.
func (c *PoolCreateCommand) Run(ctx *cmd.Context) (err error) {
	api, err := getPoolCreateAPI(c)
	if err != nil {
		return err
	}
	defer api.Close()

	return api.CreatePool(c.poolName, c.provider, c.attrs)
}

var (
	getPoolCreateAPI = (*PoolCreateCommand).getPoolCreateAPI
)

// PoolCreateAPI defines the API methods that pool create command uses.
type PoolCreateAPI interface {
	Close() error
	CreatePool(pname, ptype string, pconfig map[string]interface{}) error
}

func (c *PoolCreateCommand) getPoolCreateAPI() (PoolCreateAPI, error) {
	return c.NewStorageAPI()
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package manager

import (
	"fmt"
	"log"
	"regexp"
	"time"

	"github.com/kubernetes/helm/cmd/manager/repository"
	"github.com/kubernetes/helm/pkg/chart"
	"github.com/kubernetes/helm/pkg/common"
	"github.com/kubernetes/helm/pkg/repo"
)

// Manager manages a persistent set of Deployments.
type Manager interface {
	// Deployments
	ListDeployments() ([]common.Deployment, error)
	GetDeployment(name string) (*common.Deployment, error)
	CreateDeployment(t *common.Template) (*common.Deployment, error)
	DeleteDeployment(name string, forget bool) (*common.Deployment, error)
	PutDeployment(name string, t *common.Template) (*common.Deployment, error)

	// Manifests
	ListManifests(deploymentName string) (map[string]*common.Manifest, error)
	GetManifest(deploymentName string, manifest string) (*common.Manifest, error)
	Expand(t *common.Template) (*common.Manifest, error)

	// Charts
	ListCharts() ([]string, error)
	ListChartInstances(chartName string) ([]*common.ChartInstance, error)
	GetRepoForChart(chartName string) (string, error)
	GetMetadataForChart(chartName string) (*chart.Chartfile, error)
	GetChart(chartName string) (*chart.Chart, error)

	// Repo Charts
	ListRepoCharts(repoURL string, regex *regexp.Regexp) ([]string, error)
	GetChartForRepo(repoURL, chartName string) (*chart.Chart, error)

	// Credentials
	CreateCredential(name string, c *repo.Credential) error
	GetCredential(name string) (*repo.Credential, error)

	// Chart Repositories
	ListRepos() (map[string]string, error)
	AddRepo(addition repo.IRepo) error
	RemoveRepo(name string) error
	GetRepo(URL string) (repo.IRepo, error)
}

type manager struct {
	expander     Expander
	deployer     Deployer
	repository   repository.Repository
	repoProvider repo.IRepoProvider
	service      repo.IRepoService
	//TODO: add chart repo service
	credentialProvider repo.ICredentialProvider
}

// NewManager returns a new initialized Manager.
func NewManager(expander Expander,
	deployer Deployer,
	repository repository.Repository,
	repoProvider repo.IRepoProvider,
	service repo.IRepoService,
	credentialProvider repo.ICredentialProvider) Manager {
	return &manager{expander, deployer, repository, repoProvider, service, credentialProvider}
}

// ListDeployments returns the list of deployments
func (m *manager) ListDeployments() ([]common.Deployment, error) {
	l, err := m.repository.ListDeployments()
	if err != nil {
		return nil, err
	}
	return l, nil
}

// GetDeployment retrieves the configuration stored for a given deployment
// as well as the current configuration from the cluster.
func (m *manager) GetDeployment(name string) (*common.Deployment, error) {
	d, err := m.repository.GetDeployment(name)
	if err != nil {
		return nil, err
	}

	return d, nil
}

// ListManifests retrieves the manifests for a given deployment
// of each of the deployments in the repository and returns the deployments.
func (m *manager) ListManifests(deploymentName string) (map[string]*common.Manifest, error) {
	l, err := m.repository.ListManifests(deploymentName)
	if err != nil {
		return nil, err
	}

	return l, nil
}

// GetManifest retrieves the specified manifest for a given deployment
func (m *manager) GetManifest(deploymentName string, manifestName string) (*common.Manifest, error) {
	d, err := m.repository.GetManifest(deploymentName, manifestName)
	if err != nil {
		return nil, err
	}

	return d, nil
}

// CreateDeployment expands the supplied template, creates the resulting
// configuration in the cluster, creates a new deployment that tracks it,
// and stores the deployment in the repository. Returns the deployment.
func (m *manager) CreateDeployment(t *common.Template) (*common.Deployment, error) {
	log.Printf("Creating deployment: %s", t.Name)
	_, err := m.repository.CreateDeployment(t.Name)
	if err != nil {
		log.Printf("CreateDeployment failed %v", err)
		return nil, err
	}

	manifest, err := m.createManifest(t)
	if err != nil {
		log.Printf("Manifest creation failed: %v", err)
		m.repository.SetDeploymentState(t.Name, failState(err))
		return nil, err
	}

	if err := m.repository.AddManifest(manifest); err != nil {
		log.Printf("AddManifest failed %v", err)
		m.repository.SetDeploymentState(t.Name, failState(err))
		return nil, err
	}

	actualConfig, err := m.deployer.CreateConfiguration(manifest.ExpandedConfig)
	if err != nil {
		// Deployment failed, mark as failed
		log.Printf("CreateConfiguration failed: %v", err)
		m.repository.SetDeploymentState(t.Name, failState(err))

		// If we failed before being able to create some of the resources, then
		// return the failure as such. Otherwise, we're going to add the manifest
		// and hence resource specific errors down below.
		if actualConfig == nil {
			return nil, err
		}
	} else {
		// May be errors in the resources themselves.
		errs := getResourceErrors(actualConfig)
		if len(errs) > 0 {
			e := fmt.Errorf("Found resource errors during deployment: %v", errs)
			m.repository.SetDeploymentState(t.Name, failState(e))
			return nil, e
		}

		m.repository.SetDeploymentState(t.Name, &common.DeploymentState{Status: common.DeployedStatus})
	}

	// Update the manifest with the actual state of the reified resources
	manifest.ExpandedConfig = actualConfig
	if err := m.repository.SetManifest(manifest); err != nil {
		log.Printf("SetManifest failed %v", err)
		m.repository.SetDeploymentState(t.Name, failState(err))
		return nil, err
	}

	// Finally update the type instances for this deployment.
	m.setChartInstances(t.Name, manifest.Name, manifest.Layout)
	return m.repository.GetValidDeployment(t.Name)
}

func (m *manager) createManifest(t *common.Template) (*common.Manifest, error) {
	et, err := m.expander.ExpandTemplate(t)
	if err != nil {
		log.Printf("Expansion failed %v", err)
		return nil, err
	}

	return &common.Manifest{
		Name:           generateManifestName(),
		Deployment:     t.Name,
		InputConfig:    t,
		ExpandedConfig: et.Config,
		Layout:         et.Layout,
	}, nil
}

func (m *manager) setChartInstances(deploymentName string, manifestName string, layout *common.Layout) {
	m.repository.ClearChartInstancesForDeployment(deploymentName)

	instances := make(map[string][]*common.ChartInstance)
	for i, r := range layout.Resources {
		addChartInstances(&instances, r, deploymentName, manifestName, fmt.Sprintf("$.resources[%d]", i))
	}

	m.repository.AddChartInstances(instances)
}

func addChartInstances(instances *map[string][]*common.ChartInstance, r *common.LayoutResource, deploymentName string, manifestName string, jsonPath string) {
	// Add this resource.
	inst := &common.ChartInstance{
		Name:       r.Name,
		Type:       r.Type,
		Deployment: deploymentName,
		Manifest:   manifestName,
		Path:       jsonPath,
	}

	(*instances)[r.Type] = append((*instances)[r.Type], inst)

	// Add all sub resources if they exist.
	for i, sr := range r.Resources {
		addChartInstances(instances, sr, deploymentName, manifestName, fmt.Sprintf("%s.resources[%d]", jsonPath, i))
	}
}

// DeleteDeployment deletes the configuration for the deployment with
// the supplied identifier from the cluster.repository. If forget is true, then
// the deployment is removed from the repository. Otherwise, it is marked
// as deleted and retained.
func (m *manager) DeleteDeployment(name string, forget bool) (*common.Deployment, error) {
	log.Printf("Deleting deployment: %s", name)
	d, err := m.repository.GetValidDeployment(name)
	if err != nil {
		return nil, err
	}

	// If there's a latest manifest, delete the underlying resources.
	latest, err := m.repository.GetLatestManifest(name)
	if err != nil {
		m.repository.SetDeploymentState(name, failState(err))
		return nil, err
	}

	if latest != nil {
		log.Printf("Deleting resources from the latest manifest")
		// Clear previous state.
		for _, r := range latest.ExpandedConfig.Resources {
			r.State = nil
		}

		if _, err := m.deployer.DeleteConfiguration(latest.ExpandedConfig); err != nil {
			log.Printf("Failed to delete resources from the latest manifest: %v", err)
			m.repository.SetDeploymentState(name, failState(err))
			return nil, err
		}

		// Create an empty manifest since resources have been deleted.
		if !forget {
			manifest := &common.Manifest{Deployment: name, Name: generateManifestName()}
			if err := m.repository.AddManifest(manifest); err != nil {
				log.Printf("Failed to add empty manifest")
				return nil, err
			}
		}
	}

	d, err = m.repository.DeleteDeployment(name, forget)
	if err != nil {
		return nil, err
	}

	// Finally remove the type instances for this deployment.
	m.repository.ClearChartInstancesForDeployment(name)
	return d, nil
}

// PutDeployment replaces the configuration of the deployment with
// the supplied identifier in the cluster, and returns the deployment.
func (m *manager) PutDeployment(name string, t *common.Template) (*common.Deployment, error) {
	_, err := m.repository.GetValidDeployment(name)
	if err != nil {
		return nil, err
	}

	manifest, err := m.createManifest(t)
	if err != nil {
		log.Printf("Manifest creation failed: %v", err)
		m.repository.SetDeploymentState(name, failState(err))
		return nil, err
	}

	actualConfig, err := m.deployer.PutConfiguration(manifest.ExpandedConfig)
	if err != nil {
		m.repository.SetDeploymentState(name, failState(err))
		return nil, err
	}

	manifest.ExpandedConfig = actualConfig
	err = m.repository.AddManifest(manifest)
	if err != nil {
		m.repository.SetDeploymentState(name, failState(err))
		return nil, err
	}

	// Finally update the type instances for this deployment.
	m.setChartInstances(t.Name, manifest.Name, manifest.Layout)
	return m.repository.GetValidDeployment(t.Name)
}

func (m *manager) Expand(t *common.Template) (*common.Manifest, error) {
	et, err := m.expander.ExpandTemplate(t)
	if err != nil {
		log.Printf("Expansion failed %v", err)
		return nil, err
	}

	return &common.Manifest{
		ExpandedConfig: et.Config,
		Layout:         et.Layout,
	}, nil
}

func (m *manager) ListCharts() ([]string, error) {
	return m.repository.ListCharts()
}

func (m *manager) ListChartInstances(chartName string) ([]*common.ChartInstance, error) {
	return m.repository.GetChartInstances(chartName)
}

// GetRepoForChart returns the repository where the referenced chart resides.
func (m *manager) GetRepoForChart(reference string) (string, error) {
	_, r, err := m.repoProvider.GetChartByReference(reference)
	if err != nil {
		return "", err
	}

	return r.GetURL(), nil
}

// GetMetadataForChart returns the metadata for the referenced chart.
func (m *manager) GetMetadataForChart(reference string) (*chart.Chartfile, error) {
	c, _, err := m.repoProvider.GetChartByReference(reference)
	if err != nil {
		return nil, err
	}

	return c.Chartfile(), nil
}

// GetChart returns the referenced chart.
func (m *manager) GetChart(reference string) (*chart.Chart, error) {
	c, _, err := m.repoProvider.GetChartByReference(reference)
	if err != nil {
		return nil, err
	}

	return c, nil
}

// ListRepos returns the list of available repository URLs
func (m *manager) ListRepos() (map[string]string, error) {
	return m.service.ListRepos()
}

// AddRepo adds a repository to the list
func (m *manager) AddRepo(addition repo.IRepo) error {
	return m.service.CreateRepo(addition)
}

// RemoveRepo removes a repository from the list by URL
func (m *manager) RemoveRepo(name string) error {
	url, err := m.service.GetRepoURLByName(name)
	if err != nil {
		return err
	}
	return m.service.DeleteRepo(url)
}

// GetRepo returns the repository with the given URL
func (m *manager) GetRepo(URL string) (repo.IRepo, error) {
	return m.service.GetRepoByURL(URL)
}

func generateManifestName() string {
	return fmt.Sprintf("manifest-%d", time.Now().UTC().UnixNano())
}

func failState(e error) *common.DeploymentState {
	return &common.DeploymentState{
		Status: common.FailedStatus,
		Errors: []string{e.Error()},
	}
}

func getResourceErrors(c *common.Configuration) []string {
	var errs []string
	for _, r := range c.Resources {
		if r.State.Status == common.Failed {
			errs = append(errs, r.State.Errors...)
		}
	}

	return errs
}

// ListRepoCharts lists charts in a given repository whose URLs
// conform to the supplied regular expression, or all charts, if the regular
// expression is nil.
func (m *manager) ListRepoCharts(repoURL string, regex *regexp.Regexp) ([]string, error) {
	r, err := m.repoProvider.GetRepoByURL(repoURL)
	if err != nil {
		return nil, err
	}

	return r.ListCharts(regex)
}

// GetChartForRepo returns a chart by name from a given repository.
func (m *manager) GetChartForRepo(repoURL, chartName string) (*chart.Chart, error) {
	r, err := m.repoProvider.GetRepoByURL(repoURL)
	if err != nil {
		return nil, err
	}

	return r.GetChart(chartName)
}

// CreateCredential creates a credential that can be used to authenticate to repository
func (m *manager) CreateCredential(name string, c *repo.Credential) error {
	return m.credentialProvider.SetCredential(name, c)
}

func (m *manager) GetCredential(name string) (*repo.Credential, error) {
	return m.credentialProvider.GetCredential(name)
}
<commit_msg>Make all repo methods name based.<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package manager

import (
	"fmt"
	"log"
	"regexp"
	"time"

	"github.com/kubernetes/helm/cmd/manager/repository"
	"github.com/kubernetes/helm/pkg/chart"
	"github.com/kubernetes/helm/pkg/common"
	"github.com/kubernetes/helm/pkg/repo"
)

// Manager manages a persistent set of Deployments.
type Manager interface {
	// Deployments
	ListDeployments() ([]common.Deployment, error)
	GetDeployment(name string) (*common.Deployment, error)
	CreateDeployment(t *common.Template) (*common.Deployment, error)
	DeleteDeployment(name string, forget bool) (*common.Deployment, error)
	PutDeployment(name string, t *common.Template) (*common.Deployment, error)

	// Manifests
	ListManifests(deploymentName string) (map[string]*common.Manifest, error)
	GetManifest(deploymentName string, manifest string) (*common.Manifest, error)
	Expand(t *common.Template) (*common.Manifest, error)

	// Charts
	ListCharts() ([]string, error)
	ListChartInstances(chartName string) ([]*common.ChartInstance, error)
	GetRepoForChart(chartName string) (string, error)
	GetMetadataForChart(chartName string) (*chart.Chartfile, error)
	GetChart(chartName string) (*chart.Chart, error)

	// Repo Charts
	ListRepoCharts(repoName string, regex *regexp.Regexp) ([]string, error)
	GetChartForRepo(repoName, chartName string) (*chart.Chart, error)

	// Credentials
	CreateCredential(name string, c *repo.Credential) error
	GetCredential(name string) (*repo.Credential, error)

	// Chart Repositories
	ListRepos() (map[string]string, error)
	AddRepo(addition repo.IRepo) error
	RemoveRepo(repoName string) error
	GetRepo(repoName string) (repo.IRepo, error)
}

type manager struct {
	expander     Expander
	deployer     Deployer
	repository   repository.Repository
	repoProvider repo.IRepoProvider
	service      repo.IRepoService
	//TODO: add chart repo service
	credentialProvider repo.ICredentialProvider
}

// NewManager returns a new initialized Manager.
func NewManager(expander Expander,
	deployer Deployer,
	repository repository.Repository,
	repoProvider repo.IRepoProvider,
	service repo.IRepoService,
	credentialProvider repo.ICredentialProvider) Manager {
	return &manager{expander, deployer, repository, repoProvider, service, credentialProvider}
}

// ListDeployments returns the list of deployments
func (m *manager) ListDeployments() ([]common.Deployment, error) {
	l, err := m.repository.ListDeployments()
	if err != nil {
		return nil, err
	}
	return l, nil
}

// GetDeployment retrieves the configuration stored for a given deployment
// as well as the current configuration from the cluster.
func (m *manager) GetDeployment(name string) (*common.Deployment, error) {
	d, err := m.repository.GetDeployment(name)
	if err != nil {
		return nil, err
	}

	return d, nil
}

// ListManifests retrieves the manifests for a given deployment
// of each of the deployments in the repository and returns the deployments.
func (m *manager) ListManifests(deploymentName string) (map[string]*common.Manifest, error) {
	l, err := m.repository.ListManifests(deploymentName)
	if err != nil {
		return nil, err
	}

	return l, nil
}

// GetManifest retrieves the specified manifest for a given deployment
func (m *manager) GetManifest(deploymentName string, manifestName string) (*common.Manifest, error) {
	d, err := m.repository.GetManifest(deploymentName, manifestName)
	if err != nil {
		return nil, err
	}

	return d, nil
}

// CreateDeployment expands the supplied template, creates the resulting
// configuration in the cluster, creates a new deployment that tracks it,
// and stores the deployment in the repository. Returns the deployment.
func (m *manager) CreateDeployment(t *common.Template) (*common.Deployment, error) {
	log.Printf("Creating deployment: %s", t.Name)
	_, err := m.repository.CreateDeployment(t.Name)
	if err != nil {
		log.Printf("CreateDeployment failed %v", err)
		return nil, err
	}

	manifest, err := m.createManifest(t)
	if err != nil {
		log.Printf("Manifest creation failed: %v", err)
		m.repository.SetDeploymentState(t.Name, failState(err))
		return nil, err
	}

	if err := m.repository.AddManifest(manifest); err != nil {
		log.Printf("AddManifest failed %v", err)
		m.repository.SetDeploymentState(t.Name, failState(err))
		return nil, err
	}

	actualConfig, err := m.deployer.CreateConfiguration(manifest.ExpandedConfig)
	if err != nil {
		// Deployment failed, mark as failed
		log.Printf("CreateConfiguration failed: %v", err)
		m.repository.SetDeploymentState(t.Name, failState(err))

		// If we failed before being able to create some of the resources, then
		// return the failure as such. Otherwise, we're going to add the manifest
		// and hence resource specific errors down below.
		if actualConfig == nil {
			return nil, err
		}
	} else {
		// May be errors in the resources themselves.
		errs := getResourceErrors(actualConfig)
		if len(errs) > 0 {
			e := fmt.Errorf("Found resource errors during deployment: %v", errs)
			m.repository.SetDeploymentState(t.Name, failState(e))
			return nil, e
		}

		m.repository.SetDeploymentState(t.Name, &common.DeploymentState{Status: common.DeployedStatus})
	}

	// Update the manifest with the actual state of the reified resources
	manifest.ExpandedConfig = actualConfig
	if err := m.repository.SetManifest(manifest); err != nil {
		log.Printf("SetManifest failed %v", err)
		m.repository.SetDeploymentState(t.Name, failState(err))
		return nil, err
	}

	// Finally update the type instances for this deployment.
	m.setChartInstances(t.Name, manifest.Name, manifest.Layout)
	return m.repository.GetValidDeployment(t.Name)
}

func (m *manager) createManifest(t *common.Template) (*common.Manifest, error) {
	et, err := m.expander.ExpandTemplate(t)
	if err != nil {
		log.Printf("Expansion failed %v", err)
		return nil, err
	}

	return &common.Manifest{
		Name:           generateManifestName(),
		Deployment:     t.Name,
		InputConfig:    t,
		ExpandedConfig: et.Config,
		Layout:         et.Layout,
	}, nil
}

func (m *manager) setChartInstances(deploymentName string, manifestName string, layout *common.Layout) {
	m.repository.ClearChartInstancesForDeployment(deploymentName)

	instances := make(map[string][]*common.ChartInstance)
	for i, r := range layout.Resources {
		addChartInstances(&instances, r, deploymentName, manifestName, fmt.Sprintf("$.resources[%d]", i))
	}

	m.repository.AddChartInstances(instances)
}

func addChartInstances(instances *map[string][]*common.ChartInstance, r *common.LayoutResource, deploymentName string, manifestName string, jsonPath string) {
	// Add this resource.
	inst := &common.ChartInstance{
		Name:       r.Name,
		Type:       r.Type,
		Deployment: deploymentName,
		Manifest:   manifestName,
		Path:       jsonPath,
	}

	(*instances)[r.Type] = append((*instances)[r.Type], inst)

	// Add all sub resources if they exist.
	for i, sr := range r.Resources {
		addChartInstances(instances, sr, deploymentName, manifestName, fmt.Sprintf("%s.resources[%d]", jsonPath, i))
	}
}

// DeleteDeployment deletes the configuration for the deployment with
// the supplied identifier from the cluster.repository. If forget is true, then
// the deployment is removed from the repository. Otherwise, it is marked
// as deleted and retained.
func (m *manager) DeleteDeployment(name string, forget bool) (*common.Deployment, error) {
	log.Printf("Deleting deployment: %s", name)
	d, err := m.repository.GetValidDeployment(name)
	if err != nil {
		return nil, err
	}

	// If there's a latest manifest, delete the underlying resources.
	latest, err := m.repository.GetLatestManifest(name)
	if err != nil {
		m.repository.SetDeploymentState(name, failState(err))
		return nil, err
	}

	if latest != nil {
		log.Printf("Deleting resources from the latest manifest")
		// Clear previous state.
		for _, r := range latest.ExpandedConfig.Resources {
			r.State = nil
		}

		if _, err := m.deployer.DeleteConfiguration(latest.ExpandedConfig); err != nil {
			log.Printf("Failed to delete resources from the latest manifest: %v", err)
			m.repository.SetDeploymentState(name, failState(err))
			return nil, err
		}

		// Create an empty manifest since resources have been deleted.
		if !forget {
			manifest := &common.Manifest{Deployment: name, Name: generateManifestName()}
			if err := m.repository.AddManifest(manifest); err != nil {
				log.Printf("Failed to add empty manifest")
				return nil, err
			}
		}
	}

	d, err = m.repository.DeleteDeployment(name, forget)
	if err != nil {
		return nil, err
	}

	// Finally remove the type instances for this deployment.
	m.repository.ClearChartInstancesForDeployment(name)
	return d, nil
}

// PutDeployment replaces the configuration of the deployment with
// the supplied identifier in the cluster, and returns the deployment.
func (m *manager) PutDeployment(name string, t *common.Template) (*common.Deployment, error) {
	_, err := m.repository.GetValidDeployment(name)
	if err != nil {
		return nil, err
	}

	manifest, err := m.createManifest(t)
	if err != nil {
		log.Printf("Manifest creation failed: %v", err)
		m.repository.SetDeploymentState(name, failState(err))
		return nil, err
	}

	actualConfig, err := m.deployer.PutConfiguration(manifest.ExpandedConfig)
	if err != nil {
		m.repository.SetDeploymentState(name, failState(err))
		return nil, err
	}

	manifest.ExpandedConfig = actualConfig
	err = m.repository.AddManifest(manifest)
	if err != nil {
		m.repository.SetDeploymentState(name, failState(err))
		return nil, err
	}

	// Finally update the type instances for this deployment.
	m.setChartInstances(t.Name, manifest.Name, manifest.Layout)
	return m.repository.GetValidDeployment(t.Name)
}

func (m *manager) Expand(t *common.Template) (*common.Manifest, error) {
	et, err := m.expander.ExpandTemplate(t)
	if err != nil {
		log.Printf("Expansion failed %v", err)
		return nil, err
	}

	return &common.Manifest{
		ExpandedConfig: et.Config,
		Layout:         et.Layout,
	}, nil
}

func (m *manager) ListCharts() ([]string, error) {
	return m.repository.ListCharts()
}

func (m *manager) ListChartInstances(chartName string) ([]*common.ChartInstance, error) {
	return m.repository.GetChartInstances(chartName)
}

// GetRepoForChart returns the repository where the referenced chart resides.
func (m *manager) GetRepoForChart(reference string) (string, error) {
	_, r, err := m.repoProvider.GetChartByReference(reference)
	if err != nil {
		return "", err
	}

	return r.GetURL(), nil
}

// GetMetadataForChart returns the metadata for the referenced chart.
func (m *manager) GetMetadataForChart(reference string) (*chart.Chartfile, error) {
	c, _, err := m.repoProvider.GetChartByReference(reference)
	if err != nil {
		return nil, err
	}

	return c.Chartfile(), nil
}

// GetChart returns the referenced chart.
func (m *manager) GetChart(reference string) (*chart.Chart, error) {
	c, _, err := m.repoProvider.GetChartByReference(reference)
	if err != nil {
		return nil, err
	}

	return c, nil
}

// ListRepos returns the list of available repository URLs
func (m *manager) ListRepos() (map[string]string, error) {
	return m.service.ListRepos()
}

// AddRepo adds a repository to the list
func (m *manager) AddRepo(addition repo.IRepo) error {
	return m.service.CreateRepo(addition)
}

// RemoveRepo removes a repository from the list by URL
func (m *manager) RemoveRepo(repoName string) error {
	repoURL, err := m.service.GetRepoURLByName(repoName)
	if err != nil {
		return err
	}

	return m.service.DeleteRepo(repoURL)
}

// GetRepo returns the repository with the given URL
func (m *manager) GetRepo(repoName string) (repo.IRepo, error) {
	repoURL, err := m.service.GetRepoURLByName(repoName)
	if err != nil {
		return nil, err
	}

	return m.service.GetRepoByURL(repoURL)
}

func generateManifestName() string {
	return fmt.Sprintf("manifest-%d", time.Now().UTC().UnixNano())
}

func failState(e error) *common.DeploymentState {
	return &common.DeploymentState{
		Status: common.FailedStatus,
		Errors: []string{e.Error()},
	}
}

func getResourceErrors(c *common.Configuration) []string {
	var errs []string
	for _, r := range c.Resources {
		if r.State.Status == common.Failed {
			errs = append(errs, r.State.Errors...)
		}
	}

	return errs
}

// ListRepoCharts lists charts in a given repository whose names
// conform to the supplied regular expression, or all charts, if the regular
// expression is nil.
func (m *manager) ListRepoCharts(repoName string, regex *regexp.Regexp) ([]string, error) {
	repoURL, err := m.service.GetRepoURLByName(repoName)
	if err != nil {
		return nil, err
	}

	r, err := m.repoProvider.GetRepoByURL(repoURL)
	if err != nil {
		return nil, err
	}

	return r.ListCharts(regex)
}

// GetChartForRepo returns a chart by name from a given repository.
func (m *manager) GetChartForRepo(repoName, chartName string) (*chart.Chart, error) {
	repoURL, err := m.service.GetRepoURLByName(repoName)
	if err != nil {
		return nil, err
	}

	r, err := m.repoProvider.GetRepoByURL(repoURL)
	if err != nil {
		return nil, err
	}

	return r.GetChart(chartName)
}

// CreateCredential creates a credential that can be used to authenticate to repository
func (m *manager) CreateCredential(name string, c *repo.Credential) error {
	return m.credentialProvider.SetCredential(name, c)
}

func (m *manager) GetCredential(name string) (*repo.Credential, error) {
	return m.credentialProvider.GetCredential(name)
}
<|endoftext|>


<commit_before>package mc

import (
	"time"

	"crypto/md5"
	"io"
	"math"
	"os"

	"path/filepath"

	"fmt"

	"github.com/materials-commons/gohandy/file"
	"github.com/materials-commons/gohandy/with"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/app/flow"
	"github.com/materials-commons/mcstore/pkg/files"
	"github.com/materials-commons/mcstore/server/mcstore"
)

var _ = fmt.Println

// projectUploader holds the starting state for a project upload. It controls
// how many threads process requests.
type projectUploader struct {
	db         ProjectDB
	numThreads int
}

// upload will upload the project. It starts a parallel walker to process entries.
func (p *projectUploader) upload() error {
	db := p.db
	project := db.Project()

	// create a func to process entries. Since we have some state to close over we create
	// a closure here to call.
	fn := func(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
		uploader := newUploader(p.db, project)
		uploader.retrier.RetryCount = 1
		uploader.uploadEntries(done, entries, result)
	}

	walker := files.PWalker{
		NumParallel: p.numThreads,
		ProcessFn:   fn,
		ProcessDirs: true,
	}

	_, errc := walker.PWalk(project.Path)
	err := <-errc
	return err
}

// uploader holds the state for one upload thread.
type uploader struct {
	db        ProjectDB
	serverAPI *mcstore.ServerAPI
	project   *Project
	retrier   with.Retrier
}

// newUploader creates a new uploader. It creates a clone of the database.
func newUploader(db ProjectDB, project *Project) *uploader {
	return &uploader{
		db:        db.Clone(),
		project:   project,
		serverAPI: mcstore.NewServerAPI(),
	}
}

// uploadEntries waits on the entries and done channels. When  entry comes in on
// the entries channel it will process it. If an something comes in on the done
// channel then it will stop processing and return from the go routine.
func (u *uploader) uploadEntries(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
	for entry := range entries {
		select {
		case result <- u.uploadEntry(entry):
		case <-done:
			return
		}
	}
}

// uploadEntry identifies the type of entry it is processing (file or directory) and
// processes it appropriately.
func (u *uploader) uploadEntry(entry files.TreeEntry) string {
	switch {
	case entry.Finfo.IsDir():
		u.handleDirEntry(entry)
	default:
		u.handleFileEntry(entry)
	}
	return ""
}

// handleDirEntry handles processing of directories. It checks if the directory already exists in the
// local database. If it doesn't it will create the directory on the server and insert the directory
// into its local database.
func (u *uploader) handleDirEntry(entry files.TreeEntry) {
	fmt.Println("handleDirEntry", entry.Finfo.Name(), entry.Path)
	path := filepath.ToSlash(entry.Path)

	_, err := u.db.FindDirectory(path)
	switch {
	case err == app.ErrNotFound:
		fmt.Println("   Did not find directory")
		u.createDirectory(entry)
	case err != nil:
		app.Log.Panicf("Local database returned err, panic!: %s", err)
	default:
		// directory already known nothing to do
		return
	}
}

// createDirectory creates a new directory entry on the server.
func (u *uploader) createDirectory(entry files.TreeEntry) {
	dirPath := filepath.ToSlash(entry.Path)
	req := mcstore.DirectoryRequest{
		ProjectName: u.project.Name,
		ProjectID:   u.project.ProjectID,
		Path:        dirPath,
	}
	dirID, _ := u.getDirectory(req)
	dir := &Directory{
		DirectoryID: dirID,
		Path:        dirPath,
	}
	if _, err := u.db.InsertDirectory(dir); err != nil {
		app.Log.Panicf("Local database returned err, panic!: %s", err)
	}
}

// getDirectoryWithRetry will make the server API GetDirectory call. If it fails
// it will retry (dependent on retry settings).
func (u *uploader) getDirectory(req mcstore.DirectoryRequest) (string, error) {
	var (
		dirID string
		err   error
	)
	if dirID, err = u.serverAPI.GetDirectory(req); err != nil {
		fmt.Printf("u.serverAPI.GetDirectory failed %#v/%s\n", req, err)
		return "", err
	}
	fmt.Printf("u.serverAPI.GetDirectory succeeded:%s\n", dirID)
	return dirID, nil
}

// handleFileEntry will handle processing file entries. It will check if the file
// has already been uploaded. If it hasn't it will upload the file to the server.
func (u *uploader) handleFileEntry(entry files.TreeEntry) {
	fmt.Println("handleFileEntry %#v\n", entry)
	if dir := u.getDirByPath(filepath.Dir(entry.Path)); dir == nil {
		app.Log.Exitf("Should have found dir %s", filepath.Dir(entry.Path))
	} else {
		fmt.Printf("handleFileEntry dir = %#v\n", dir)
		file := u.getFileByName(entry.Finfo.Name(), dir.ID)
		switch {
		case file == nil:
			u.uploadFile(entry, file, dir)
		case entry.Finfo.ModTime().Unix() > file.MTime.Unix():
			u.uploadFile(entry, file, dir)
		default:
			// nothing to do
		}
	}
}

// getDirByPath looks up a path in the local database. It normalizes
// the file separator.
func (u *uploader) getDirByPath(path string) *Directory {
	path = filepath.ToSlash(path)
	dir, err := u.db.FindDirectory(path)
	switch {
	case err == app.ErrNotFound:
		return nil
	case err != nil:
		app.Log.Panicf("Local database returned err, panic!: %s", err)
		return nil
	default:
		// directory already known nothing to do
		return dir
	}
}

// getFileByName looks up a file by name and directory id in the local database.
func (u *uploader) getFileByName(name string, dirID int64) *File {
	f, err := u.db.FindFile(name, dirID)
	switch {
	case err == app.ErrNotFound:
		return nil
	case err != nil:
		app.Log.Panicf("Local database returned err, panic!: %s", err)
		return nil
	default:
		return f
	}
}

// uploadFile will upload the file blocks to the server. It will create an upload request
// and then start uploading the blocks. When it completes it will update the local database
// state for the file.
func (u *uploader) uploadFile(entry files.TreeEntry, file *File, dir *Directory) {
	uploadResponse, checksum := u.getUploadResponse(dir.DirectoryID, entry)
	requestID := uploadResponse.RequestID

	var _ = checksum

	// TODO: do something with the starting block (its ignored for now)
	var n int
	var err error
	chunkNumber := 1

	f, _ := os.Open(entry.Path)
	defer f.Close()
	buf := make([]byte, 1024*1024)
	totalChunks := numChunks(entry.Finfo.Size())
	var uploadResp *mcstore.UploadChunkResponse
	for {
		n, err = f.Read(buf)
		if n != 0 {
			// send bytes
			req := &flow.Request{
				FlowChunkNumber:  int32(chunkNumber),
				FlowTotalChunks:  totalChunks,
				FlowChunkSize:    int32(n),
				FlowTotalSize:    entry.Finfo.Size(),
				FlowIdentifier:   requestID,
				FlowFileName:     entry.Finfo.Name(),
				FlowRelativePath: "",
				ProjectID:        u.project.ProjectID,
				DirectoryID:      dir.DirectoryID,
				Chunk:            buf[:n],
			}
			uploadResp, _ = u.sendFlowData(req)
			if uploadResp.Done {
				break
			}
			chunkNumber++
		}
		if err != nil {
			break
		}
	}

	if !uploadResp.Done {
		app.Log.Errorf("uploadResp not done %#v\n", uploadResp)
		return
	}

	if err != nil && err != io.EOF {
		app.Log.Errorf("Unable to complete read on file for upload: %s", entry.Path)
	}
	//else {
	//		// done, add or update the entry in the database.
	//		if file == nil {
	//			// create new entry
	//			newFile := File{
	//				FileID:     uploadResp.FileID,
	//				Name:       entry.Finfo.Name(),
	//				Checksum:   checksum,
	//				Size:       entry.Finfo.Size(),
	//				MTime:      entry.Finfo.ModTime(),
	//				LastUpload: time.Now(),
	//				Directory:  dir.ID,
	//			}
	//			u.db.InsertFile(&newFile)
	//		} else {
	//			// update existing entry
	//			file.MTime = entry.Finfo.ModTime()
	//			file.LastUpload = time.Now()
	//			if file.Checksum != checksum {
	//				// Existing file uploaded but we created a new version on the server.
	//				// We could get here if a previous upload did not complete.
	//				file.Checksum = checksum
	//				file.FileID = uploadResp.FileID
	//				file.Size = entry.Finfo.Size()
	//			}
	//			u.db.UpdateFile(file)
	//		}
	//	}
}

// getUploadResponse sends an upload request to the server and gets the response.
func (u *uploader) getUploadResponse(directoryID string, entry files.TreeEntry) (*mcstore.CreateUploadResponse, string) {
	checksum, _ := file.HashStr(md5.New(), entry.Path)
	chunkSize := int32(1024 * 1024)
	uploadReq := mcstore.CreateUploadRequest{
		ProjectID:   u.project.ProjectID,
		DirectoryID: directoryID,
		FileName:    entry.Finfo.Name(),
		FileSize:    entry.Finfo.Size(),
		ChunkSize:   chunkSize,
		FileMTime:   entry.Finfo.ModTime().Format(time.RFC1123),
		Checksum:    checksum,
	}
	fmt.Printf("%#v\n", uploadReq)
	resp, err := u.createUploadRequest(uploadReq)
	fmt.Println("getUploadResponse err", err)
	return resp, checksum
}

// createUploadRequestWithRetry will make the server API CreateUploadRequest call. If it fails
// it will retry (dependent on retry settings).
func (u *uploader) createUploadRequest(uploadReq mcstore.CreateUploadRequest) (*mcstore.CreateUploadResponse, error) {
	var (
		resp *mcstore.CreateUploadResponse
		err  error
	)
	if resp, err = u.serverAPI.CreateUploadRequest(uploadReq); err != nil {
		return nil, err
	}

	return resp, nil
}

// numChunks determines the number of chunks that will be sent to the server.
func numChunks(size int64) int32 {
	d := float64(size) / float64(1024*1024)
	n := int(math.Ceil(d))
	return int32(n)
}

// sendFlowDataWithRetry will make the server API SendFlowData call. If it fails
// it will retry (dependent on retry settings).
func (u *uploader) sendFlowData(req *flow.Request) (*mcstore.UploadChunkResponse, error) {
	var (
		resp *mcstore.UploadChunkResponse
		err  error
	)
	if resp, err = u.serverAPI.SendFlowData(req); err != nil {
		return nil, err
	}
	return resp, nil
}
<commit_msg>Remove debug print statements. Add back in the code (that was commented out) for tracking file state on the client.<commit_after>package mc

import (
	"time"

	"crypto/md5"
	"io"
	"math"
	"os"

	"path/filepath"

	"fmt"

	"github.com/materials-commons/gohandy/file"
	"github.com/materials-commons/gohandy/with"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/app/flow"
	"github.com/materials-commons/mcstore/pkg/files"
	"github.com/materials-commons/mcstore/server/mcstore"
)

var _ = fmt.Println

// projectUploader holds the starting state for a project upload. It controls
// how many threads process requests.
type projectUploader struct {
	db         ProjectDB
	numThreads int
}

// upload will upload the project. It starts a parallel walker to process entries.
func (p *projectUploader) upload() error {
	db := p.db
	project := db.Project()

	// create a func to process entries. Since we have some state to close over we create
	// a closure here to call.
	fn := func(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
		uploader := newUploader(p.db, project)
		uploader.retrier.RetryCount = 1
		uploader.uploadEntries(done, entries, result)
	}

	walker := files.PWalker{
		NumParallel: p.numThreads,
		ProcessFn:   fn,
		ProcessDirs: true,
	}

	_, errc := walker.PWalk(project.Path)
	err := <-errc
	return err
}

// uploader holds the state for one upload thread.
type uploader struct {
	db        ProjectDB
	serverAPI *mcstore.ServerAPI
	project   *Project
	retrier   with.Retrier
}

// newUploader creates a new uploader. It creates a clone of the database.
func newUploader(db ProjectDB, project *Project) *uploader {
	return &uploader{
		db:        db.Clone(),
		project:   project,
		serverAPI: mcstore.NewServerAPI(),
	}
}

// uploadEntries waits on the entries and done channels. When  entry comes in on
// the entries channel it will process it. If an something comes in on the done
// channel then it will stop processing and return from the go routine.
func (u *uploader) uploadEntries(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
	for entry := range entries {
		select {
		case result <- u.uploadEntry(entry):
		case <-done:
			return
		}
	}
}

// uploadEntry identifies the type of entry it is processing (file or directory) and
// processes it appropriately.
func (u *uploader) uploadEntry(entry files.TreeEntry) string {
	switch {
	case entry.Finfo.IsDir():
		u.handleDirEntry(entry)
	default:
		u.handleFileEntry(entry)
	}
	return ""
}

// handleDirEntry handles processing of directories. It checks if the directory already exists in the
// local database. If it doesn't it will create the directory on the server and insert the directory
// into its local database.
func (u *uploader) handleDirEntry(entry files.TreeEntry) {
	path := filepath.ToSlash(entry.Path)

	_, err := u.db.FindDirectory(path)
	switch {
	case err == app.ErrNotFound:
		u.createDirectory(entry)
	case err != nil:
		app.Log.Panicf("Local database returned err, panic!: %s", err)
	default:
		// directory already known nothing to do
		return
	}
}

// createDirectory creates a new directory entry on the server.
func (u *uploader) createDirectory(entry files.TreeEntry) {
	dirPath := filepath.ToSlash(entry.Path)
	req := mcstore.DirectoryRequest{
		ProjectName: u.project.Name,
		ProjectID:   u.project.ProjectID,
		Path:        dirPath,
	}
	dirID, _ := u.getDirectory(req)
	dir := &Directory{
		DirectoryID: dirID,
		Path:        dirPath,
	}
	if _, err := u.db.InsertDirectory(dir); err != nil {
		app.Log.Panicf("Local database returned err, panic!: %s", err)
	}
}

// getDirectoryWithRetry will make the server API GetDirectory call. If it fails
// it will retry (dependent on retry settings).
func (u *uploader) getDirectory(req mcstore.DirectoryRequest) (string, error) {
	var (
		dirID string
		err   error
	)
	if dirID, err = u.serverAPI.GetDirectory(req); err != nil {
		fmt.Printf("u.serverAPI.GetDirectory failed %#v/%s\n", req, err)
		return "", err
	}
	return dirID, nil
}

// handleFileEntry will handle processing file entries. It will check if the file
// has already been uploaded. If it hasn't it will upload the file to the server.
func (u *uploader) handleFileEntry(entry files.TreeEntry) {
	if dir := u.getDirByPath(filepath.Dir(entry.Path)); dir == nil {
		app.Log.Exitf("Should have found dir %s", filepath.Dir(entry.Path))
	} else {
		file := u.getFileByName(entry.Finfo.Name(), dir.ID)
		switch {
		case file == nil:
			fmt.Println("file == nil uploading:", entry.Finfo.Name())
			u.uploadFile(entry, file, dir)
		case entry.Finfo.ModTime().Unix() > file.MTime.Unix():
			fmt.Println("modtime different uploading:", entry.Finfo.Name())
			u.uploadFile(entry, file, dir)
		default:
			fmt.Println("file already uploaded:", entry.Finfo.Name())
			// nothing to do
		}
	}
}

// getDirByPath looks up a path in the local database. It normalizes
// the file separator.
func (u *uploader) getDirByPath(path string) *Directory {
	path = filepath.ToSlash(path)
	dir, err := u.db.FindDirectory(path)
	switch {
	case err == app.ErrNotFound:
		return nil
	case err != nil:
		app.Log.Panicf("Local database returned err, panic!: %s", err)
		return nil
	default:
		// directory already known nothing to do
		return dir
	}
}

// getFileByName looks up a file by name and directory id in the local database.
func (u *uploader) getFileByName(name string, dirID int64) *File {
	f, err := u.db.FindFile(name, dirID)
	switch {
	case err == app.ErrNotFound:
		return nil
	case err != nil:
		app.Log.Panicf("Local database returned err, panic!: %s", err)
		return nil
	default:
		return f
	}
}

// uploadFile will upload the file blocks to the server. It will create an upload request
// and then start uploading the blocks. When it completes it will update the local database
// state for the file.
func (u *uploader) uploadFile(entry files.TreeEntry, file *File, dir *Directory) {
	uploadResponse, checksum := u.getUploadResponse(dir.DirectoryID, entry)
	requestID := uploadResponse.RequestID

	var _ = checksum

	// TODO: do something with the starting block (its ignored for now)
	var n int
	var err error
	chunkNumber := 1

	f, _ := os.Open(entry.Path)
	defer f.Close()
	buf := make([]byte, 1024*1024)
	totalChunks := numChunks(entry.Finfo.Size())
	var uploadResp *mcstore.UploadChunkResponse
	for {
		n, err = f.Read(buf)
		if n != 0 {
			// send bytes
			req := &flow.Request{
				FlowChunkNumber:  int32(chunkNumber),
				FlowTotalChunks:  totalChunks,
				FlowChunkSize:    int32(n),
				FlowTotalSize:    entry.Finfo.Size(),
				FlowIdentifier:   requestID,
				FlowFileName:     entry.Finfo.Name(),
				FlowRelativePath: "",
				ProjectID:        u.project.ProjectID,
				DirectoryID:      dir.DirectoryID,
				Chunk:            buf[:n],
			}
			uploadResp, _ = u.sendFlowData(req)
			if uploadResp.Done {
				break
			}
			chunkNumber++
		}
		if err != nil {
			break
		}
	}

	if !uploadResp.Done {
		app.Log.Errorf("uploadResp not done %#v\n", uploadResp)
		return
	}

	if err != nil && err != io.EOF {
		app.Log.Errorf("Unable to complete read on file for upload: %s", entry.Path)
	} else {
		// done, add or update the entry in the database.
		if file == nil {
			// create new entry
			newFile := File{
				FileID:     uploadResp.FileID,
				Name:       entry.Finfo.Name(),
				Checksum:   checksum,
				Size:       entry.Finfo.Size(),
				MTime:      entry.Finfo.ModTime(),
				LastUpload: time.Now(),
				Directory:  dir.ID,
			}
			u.db.InsertFile(&newFile)
		} else {
			// update existing entry
			file.MTime = entry.Finfo.ModTime()
			file.LastUpload = time.Now()
			if file.Checksum != checksum {
				// Existing file uploaded but we created a new version on the server.
				// We could get here if a previous upload did not complete.
				file.Checksum = checksum
				file.FileID = uploadResp.FileID
				file.Size = entry.Finfo.Size()
			}
			u.db.UpdateFile(file)
		}
	}
}

// getUploadResponse sends an upload request to the server and gets the response.
func (u *uploader) getUploadResponse(directoryID string, entry files.TreeEntry) (*mcstore.CreateUploadResponse, string) {
	checksum, _ := file.HashStr(md5.New(), entry.Path)
	chunkSize := int32(1024 * 1024)
	uploadReq := mcstore.CreateUploadRequest{
		ProjectID:   u.project.ProjectID,
		DirectoryID: directoryID,
		FileName:    entry.Finfo.Name(),
		FileSize:    entry.Finfo.Size(),
		ChunkSize:   chunkSize,
		FileMTime:   entry.Finfo.ModTime().Format(time.RFC1123),
		Checksum:    checksum,
	}
	resp, _ := u.createUploadRequest(uploadReq)
	return resp, checksum
}

// createUploadRequestWithRetry will make the server API CreateUploadRequest call. If it fails
// it will retry (dependent on retry settings).
func (u *uploader) createUploadRequest(uploadReq mcstore.CreateUploadRequest) (*mcstore.CreateUploadResponse, error) {
	var (
		resp *mcstore.CreateUploadResponse
		err  error
	)
	if resp, err = u.serverAPI.CreateUploadRequest(uploadReq); err != nil {
		return nil, err
	}

	return resp, nil
}

// numChunks determines the number of chunks that will be sent to the server.
func numChunks(size int64) int32 {
	d := float64(size) / float64(1024*1024)
	n := int(math.Ceil(d))
	return int32(n)
}

// sendFlowDataWithRetry will make the server API SendFlowData call. If it fails
// it will retry (dependent on retry settings).
func (u *uploader) sendFlowData(req *flow.Request) (*mcstore.UploadChunkResponse, error) {
	var (
		resp *mcstore.UploadChunkResponse
		err  error
	)
	if resp, err = u.serverAPI.SendFlowData(req); err != nil {
		return nil, err
	}
	return resp, nil
}
<|endoftext|>


<commit_before>package buildkite

import "fmt"
import "strings"

// You can overriden buildVersion at compile time by using:
//
//  go run -ldflags "-X github.com/buildkite/agent/buildkite.buildVersion abc" *.go --version
//
// On CI, the binaries are always build with the buildVersion variable set.

var baseVersion string = "1.0-beta.21"
var buildVersion string = ""

func Version() string {
	// Only output the build version if a pre-release
	if strings.Contains(baseVersion, "beta") || strings.Contains(baseVersion, "alpha") {
		// Use a default buildVersion if one doesn't exist
		actualBuildVersion := buildVersion
		if actualBuildVersion == "" {
			actualBuildVersion = "x"
		}

		return fmt.Sprintf("%s.%s", baseVersion, actualBuildVersion)
	} else {
		return baseVersion
	}
}
<commit_msg>Bumped to 1.0-beta.22<commit_after>package buildkite

import "fmt"
import "strings"

// You can overriden buildVersion at compile time by using:
//
//  go run -ldflags "-X github.com/buildkite/agent/buildkite.buildVersion abc" *.go --version
//
// On CI, the binaries are always build with the buildVersion variable set.

var baseVersion string = "1.0-beta.22"
var buildVersion string = ""

func Version() string {
	// Only output the build version if a pre-release
	if strings.Contains(baseVersion, "beta") || strings.Contains(baseVersion, "alpha") {
		// Use a default buildVersion if one doesn't exist
		actualBuildVersion := buildVersion
		if actualBuildVersion == "" {
			actualBuildVersion = "x"
		}

		return fmt.Sprintf("%s.%s", baseVersion, actualBuildVersion)
	} else {
		return baseVersion
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ssa

// dse does dead-store elimination on the Function.
// Dead stores are those which are unconditionally followed by
// another store to the same location, with no intervening load.
// This implementation only works within a basic block.  TODO: use something more global.
func dse(f *Func) {
	var stores []*Value
	loadUse := newSparseSet(f.NumValues())
	storeUse := newSparseSet(f.NumValues())
	shadowed := newSparseSet(f.NumValues())
	for _, b := range f.Blocks {
		// Find all the stores in this block.  Categorize their uses:
		//  loadUse contains stores which are used by a subsequent load.
		//  storeUse contains stores which are used by a subsequent store.
		loadUse.clear()
		storeUse.clear()
		stores = stores[:0]
		for _, v := range b.Values {
			if v.Op == OpPhi {
				// Ignore phis - they will always be first and can't be eliminated
				continue
			}
			if v.Type.IsMemory() {
				stores = append(stores, v)
				for _, a := range v.Args {
					if a.Block == b && a.Type.IsMemory() {
						storeUse.add(a.ID)
						if v.Op != OpStore && v.Op != OpZero {
							// CALL, DUFFCOPY, etc. are both
							// reads and writes.
							loadUse.add(a.ID)
						}
					}
				}
			} else {
				for _, a := range v.Args {
					if a.Block == b && a.Type.IsMemory() {
						loadUse.add(a.ID)
					}
				}
			}
		}
		if len(stores) == 0 {
			continue
		}

		// find last store in the block
		var last *Value
		for _, v := range stores {
			if storeUse.contains(v.ID) {
				continue
			}
			if last != nil {
				b.Fatalf("two final stores - simultaneous live stores %s %s", last, v)
			}
			last = v
		}
		if last == nil {
			b.Fatalf("no last store found - cycle?")
		}

		// Walk backwards looking for dead stores.  Keep track of shadowed addresses.
		// An "address" is an SSA Value which encodes both the address and size of
		// the write.  This code will not remove dead stores to the same address
		// of different types.
		shadowed.clear()
		v := last

	walkloop:
		if loadUse.contains(v.ID) {
			// Someone might be reading this memory state.
			// Clear all shadowed addresses.
			shadowed.clear()
		}
		if v.Op == OpStore || v.Op == OpZero {
			if shadowed.contains(v.Args[0].ID) {
				// Modify store into a copy
				if v.Op == OpStore {
					// store addr value mem
					v.SetArgs1(v.Args[2])
				} else {
					// zero addr mem
					sz := v.Args[0].Type.Elem().Size()
					if v.AuxInt != sz {
						f.Fatalf("mismatched zero/store sizes: %d and %d [%s]",
							v.AuxInt, sz, v.LongString())
					}
					v.SetArgs1(v.Args[1])
				}
				v.Aux = nil
				v.AuxInt = 0
				v.Op = OpCopy
			} else {
				shadowed.add(v.Args[0].ID)
			}
		}
		// walk to previous store
		if v.Op == OpPhi {
			continue // At start of block.  Move on to next block.
		}
		for _, a := range v.Args {
			if a.Block == b && a.Type.IsMemory() {
				v = a
				goto walkloop
			}
		}
	}
}
<commit_msg>[dev.ssa] cmd/compile/internal/ssa: don't treat vardef/varkill as reads<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ssa

// dse does dead-store elimination on the Function.
// Dead stores are those which are unconditionally followed by
// another store to the same location, with no intervening load.
// This implementation only works within a basic block.  TODO: use something more global.
func dse(f *Func) {
	var stores []*Value
	loadUse := newSparseSet(f.NumValues())
	storeUse := newSparseSet(f.NumValues())
	shadowed := newSparseSet(f.NumValues())
	for _, b := range f.Blocks {
		// Find all the stores in this block.  Categorize their uses:
		//  loadUse contains stores which are used by a subsequent load.
		//  storeUse contains stores which are used by a subsequent store.
		loadUse.clear()
		storeUse.clear()
		stores = stores[:0]
		for _, v := range b.Values {
			if v.Op == OpPhi {
				// Ignore phis - they will always be first and can't be eliminated
				continue
			}
			if v.Type.IsMemory() {
				stores = append(stores, v)
				for _, a := range v.Args {
					if a.Block == b && a.Type.IsMemory() {
						storeUse.add(a.ID)
						if v.Op != OpStore && v.Op != OpZero && v.Op != OpVarDef && v.Op != OpVarKill {
							// CALL, DUFFCOPY, etc. are both
							// reads and writes.
							loadUse.add(a.ID)
						}
					}
				}
			} else {
				for _, a := range v.Args {
					if a.Block == b && a.Type.IsMemory() {
						loadUse.add(a.ID)
					}
				}
			}
		}
		if len(stores) == 0 {
			continue
		}

		// find last store in the block
		var last *Value
		for _, v := range stores {
			if storeUse.contains(v.ID) {
				continue
			}
			if last != nil {
				b.Fatalf("two final stores - simultaneous live stores %s %s", last, v)
			}
			last = v
		}
		if last == nil {
			b.Fatalf("no last store found - cycle?")
		}

		// Walk backwards looking for dead stores.  Keep track of shadowed addresses.
		// An "address" is an SSA Value which encodes both the address and size of
		// the write.  This code will not remove dead stores to the same address
		// of different types.
		shadowed.clear()
		v := last

	walkloop:
		if loadUse.contains(v.ID) {
			// Someone might be reading this memory state.
			// Clear all shadowed addresses.
			shadowed.clear()
		}
		if v.Op == OpStore || v.Op == OpZero {
			if shadowed.contains(v.Args[0].ID) {
				// Modify store into a copy
				if v.Op == OpStore {
					// store addr value mem
					v.SetArgs1(v.Args[2])
				} else {
					// zero addr mem
					sz := v.Args[0].Type.Elem().Size()
					if v.AuxInt != sz {
						f.Fatalf("mismatched zero/store sizes: %d and %d [%s]",
							v.AuxInt, sz, v.LongString())
					}
					v.SetArgs1(v.Args[1])
				}
				v.Aux = nil
				v.AuxInt = 0
				v.Op = OpCopy
			} else {
				shadowed.add(v.Args[0].ID)
			}
		}
		// walk to previous store
		if v.Op == OpPhi {
			continue // At start of block.  Move on to next block.
		}
		for _, a := range v.Args {
			if a.Block == b && a.Type.IsMemory() {
				v = a
				goto walkloop
			}
		}
	}
}
<|endoftext|>


<commit_before>package models

import (
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestChannelparticipantNewChannelParticipant(t *testing.T) {
	Convey("while testing New Channel Participant", t, func() {
		Convey("Function call should return ChannelParticipant", func() {
			So(NewChannelParticipant(), ShouldNotBeNil)
		})
	})
}

func TestChannelparticipantGetId(t *testing.T) {
	Convey("While testing get id", t, func() {
		Convey("Initialized struct", func() {
			Convey("should return given id", func() {
				c := ChannelParticipant{Id: 45}
				So(c.GetId(), ShouldEqual, 45)
			})
		})
		Convey("Uninitialized struct", func() {
			Convey("should return 0", func() {
				So(NewChannelParticipant().GetId(), ShouldEqual, 0)
			})
			So(NewChannelParticipant, ShouldNotBeNil)
		})
	})
}
<commit_msg>Social/test: test is added for channelparticipant<commit_after>package models

import (
	"testing"
	"time"

	. "github.com/smartystreets/goconvey/convey"
)

func TestChannelparticipantNewChannelParticipant(t *testing.T) {
	Convey("while testing New Channel Participant", t, func() {
		Convey("Function call should return ChannelParticipant", func() {
			So(NewChannelParticipant(), ShouldNotBeNil)
		})
	})
}

func TestChannelparticipantGetId(t *testing.T) {
	Convey("While testing get id", t, func() {
		Convey("Initialized struct", func() {
			Convey("should return given id", func() {
				c := ChannelParticipant{Id: 45}
				So(c.GetId(), ShouldEqual, 45)
			})
		})
		Convey("Uninitialized struct", func() {
			Convey("should return 0", func() {
				So(NewChannelParticipant().GetId(), ShouldEqual, 0)
			})
			So(NewChannelParticipant, ShouldNotBeNil)
		})
	})
}

func TestChannelParticipantBeforeUpdate(t *testing.T) {
	Convey("While testing before update", t, func() {
		Convey("LastSeenAt should be updated", func() {
			lastSeenAt := time.Now().UTC()

			c := NewChannelParticipant()
			c.LastSeenAt = lastSeenAt

			// call before update
			err := c.BeforeUpdate()

			// make sure err is nil
			So(err, ShouldBeNil)
			// check preset last seen at is not same after calling
			// before update function
			So(c.LastSeenAt, ShouldNotEqual, lastSeenAt)
		})
	})
}
<|endoftext|>