<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sparc64

import "cmd/internal/obj"

// General purpose registers, kept in the low bits of Prog.Reg.
const (
	// integer
	REG_R0 = obj.RBaseSPARC64 + iota
	REG_R1
	REG_R2
	REG_R3
	REG_R4
	REG_R5
	REG_R6
	REG_R7
	REG_R8
	REG_R9
	REG_R10
	REG_R11
	REG_R12
	REG_R13
	REG_R14
	REG_R15
	REG_R16
	REG_R17
	REG_R18
	REG_R19
	REG_R20
	REG_R21
	REG_R22
	REG_R23
	REG_R24
	REG_R25
	REG_R26
	REG_R27
	REG_R28
	REG_R29
	REG_R30
	REG_R31

	// single-precision floating point
	REG_F0
	REG_F1
	REG_F2
	REG_F3
	REG_F4
	REG_F5
	REG_F6
	REG_F7
	REG_F8
	REG_F9
	REG_F10
	REG_F11
	REG_F12
	REG_F13
	REG_F14
	REG_F15
	REG_F16
	REG_F17
	REG_F18
	REG_F19
	REG_F20
	REG_F21
	REG_F22
	REG_F23
	REG_F24
	REG_F25
	REG_F26
	REG_F27
	REG_F28
	REG_F29
	REG_F30
	REG_F31

	// double-precision floating point; the first half is aliased to
	// single-precision registers, that is: Dn is aliased to Fn, Fn+1,
	// where n ≤ 30.
	REG_D0
	REG_D32
	REG_D2
	REG_D34
	REG_D4
	REG_D36
	REG_D6
	REG_D38
	REG_D8
	REG_D40
	REG_D10
	REG_D42
	REG_D12
	REG_D44
	REG_D14
	REG_D46
	REG_D16
	REG_D48
	REG_D18
	REG_D50
	REG_D20
	REG_D52
	REG_D22
	REG_D54
	REG_D24
	REG_D56
	REG_D26
	REG_D58
	REG_D28
	REG_D60
	REG_D30
	REG_D62
)

const (
	// floating-point condition-code registers
	REG_FCC0 = REG_R0 + 256 + iota
	REG_FCC1
	REG_FCC2
	REG_FCC3
)

const (
	// integer condition-code flags
	REG_ICC = REG_R0 + 384
	REG_XCC = REG_R0 + 384 + 2
)

const (
	REG_SPECIAL = REG_R0 + 512

	REG_CCR  = REG_SPECIAL + 2
	REG_TICK = REG_SPECIAL + 4
	REG_RPC  = REG_SPECIAL + 5

	REG_LAST = REG_R0 + 1024
)

// Register assignments:
const (
	REG_ZR  = REG_R0
	REG_RSP = REG_R14
	REG_LR  = REG_R15
	REG_TMP = REG_R22
	REG_GOT = REG_R23
	REG_RFP = REG_R30
	REG_PLR = REG_R31
)

const (
	StackBias             = 0x7ff  // craziness
	WindowSaveAreaSize    = 16 * 8 // only slots for RFP and PLR used
	ArgumentsSaveAreaSize = 0      // normally 6*8 on SPARC, but we cheat
	MinStackFrameSize     = WindowSaveAreaSize + ArgumentsSaveAreaSize
)

// Prog.mark
const (
	FOLL = 1 << iota
	LABEL
	LEAF
)

const (
	ClassUnknown = iota

	ClassReg        // R1..R31
	ClassFloatReg   // F0..F31
	ClassDoubleReg  // D0..D62
	ClassCond       // ICC, XCC
	ClassFloatCond  // FCC0..FCC3
	ClassSpecialReg // TICK, CCR, etc

	ClassZero       // $0 or ZR
	ClassConst5     // unsigned 5-bit constant
	ClassConst6     // unsigned 6-bit constant
	ClassConst13    // signed 13-bit constant
	ClassConst31_   // signed 32-bit constant, negative
	ClassConst31    // signed 32-bit constant, positive or zero
	ClassConst32    // 32-bit constant
	ClassConst      // 64-bit constant
	ClassFloatConst // floating-point constant

	ClassRegReg     // $(Rn+Rm) or $(Rn)(Rm*1)
	ClassRegConst13 // $n(R), n is 13-bit signed
	ClassRegConst   // $n(R), n large

	ClassIndirRegReg // (Rn+Rm) or (Rn)(Rm*1)
	ClassIndir0      // (R)
	ClassIndir13     // n(R), n is 13-bit signed
	ClassIndir       // n(R), n large

	ClassShortBranch // n(PC) branch target, n is 21-bit signed, mod 4

	ClassAddr // $sym(SB)
	ClassMem  // sym(SB)

	ClassTextSize
	ClassNone
)

var cnames = []string{
	ClassUnknown:     "ClassUnknown",
	ClassReg:         "ClassReg",
	ClassFloatReg:    "ClassFloatReg",
	ClassDoubleReg:   "ClassDoubleReg",
	ClassCond:        "ClassCond",
	ClassFloatCond:   "ClassFloatCond",
	ClassSpecialReg:  "ClassSpecialReg",
	ClassZero:        "ClassZero",
	ClassConst5:      "ClassConst5",
	ClassConst6:      "ClassConst6",
	ClassConst13:     "ClassConst13",
	ClassConst31_:    "ClassConst31-",
	ClassConst31:     "ClassConst31+",
	ClassConst32:     "ClassConst32",
	ClassConst:       "ClassConst",
	ClassFloatConst:  "ClassFloatConst",
	ClassRegReg:      "ClassRegReg",
	ClassRegConst13:  "ClassRegConst13",
	ClassRegConst:    "ClassRegConst",
	ClassIndirRegReg: "ClassIndirRegReg",
	ClassIndir0:      "ClassIndir0",
	ClassIndir13:     "ClassIndir13",
	ClassIndir:       "ClassIndir",
	ClassShortBranch: "ClassShortBranch",
	ClassAddr:        "ClassAddr",
	ClassMem:         "ClassMem",
	ClassTextSize:    "ClassTextSize",
	ClassNone:        "ClassNone",
}

//go:generate go run ../stringer.go -i $GOFILE -o anames.go -p sparc64

const (
	AADD = obj.ABaseSPARC64 + obj.A_ARCHSPECIFIC + iota
	AADDCC
	AADDC
	AADDCCC
	AAND
	AANDCC
	AANDN
	AANDNCC

	// These are the two-operand SPARCv9 32-, and 64-bit, branch
	// on integer condition codes with prediction (BPcc), not the
	// single-operand SPARCv8 32-bit branch on integer condition
	// codes (Bicc).
	ABN
	ABNE
	ABE
	ABG
	ABLE
	ABGE
	ABL
	ABGU
	ABLEU
	ABCC
	ABCS
	ABPOS
	ABNEG
	ABVC
	ABVS

	ABRZ
	ABRLEZ
	ABRLZ
	ABRNZ
	ABRGZ
	ABRGEZ
	ACASW
	ACASD
	AFABSS
	AFABSD
	AFADDS
	AFADDD
	AFBA
	AFBN
	AFBU
	AFBG
	AFBUG
	AFBL
	AFBUL
	AFBLG
	AFBNE
	AFBE
	AFBUE
	AFBGE
	AFBUGE
	AFBLE
	AFBULE
	AFBO
	AFCMPS
	AFCMPD
	AFDIVS
	AFDIVD
	AFITOS
	AFITOD
	AFLUSH
	AFMOVS // the SPARC64 instruction, and alias for loads and stores
	AFMOVD // the SPARC64 instruction, and alias for loads and stores
	AFMULS
	AFMULD
	AFSMULD
	AFNEGS
	AFNEGD
	AFSQRTS
	AFSQRTD
	AFSTOX
	AFDTOX
	AFSTOI
	AFDTOI
	AFSTOD
	AFDTOS
	AFSUBS
	AFSUBD
	AFXTOS
	AFXTOD
	AJMPL
	ALDSB
	ALDSH
	ALDSW
	ALDUB
	ALDUH
	ALDUW
	ALDD
	ALDSF
	ALDDF
	AMEMBAR
	AMULD
	ASDIVD
	AUDIVD
	AOR
	AORCC
	AORN
	AORNCC
	ARD
	ASETHI
	ASLLW
	ASRLW
	ASRAW
	ASLLD
	ASRLD
	ASRAD
	ASTB
	ASTH
	ASTW
	ASTD
	ASTSF
	ASTDF
	ASUB
	ASUBCC
	ASUBC
	ASUBCCC
	AXOR
	AXORCC
	AXNOR
	AXNORCC

	// Pseudo-instructions, aliases to SPARC64 instructions and
	// synthetic instructions.
	AMOVB
	AMOVSB
	AMOVH
	AMOVSH
	AMOVW
	AMOVSW
	AMOVD // also the SPARC64 synthetic instruction
	ARNOP // SETHI $0, ZR

	AWORD
	ADWORD
)
<commit_msg>cmd/internal/obj/sparc64: REG_TMP is R1 (%g1)<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sparc64

import "cmd/internal/obj"

// General purpose registers, kept in the low bits of Prog.Reg.
const (
	// integer
	REG_R0 = obj.RBaseSPARC64 + iota
	REG_R1
	REG_R2
	REG_R3
	REG_R4
	REG_R5
	REG_R6
	REG_R7
	REG_R8
	REG_R9
	REG_R10
	REG_R11
	REG_R12
	REG_R13
	REG_R14
	REG_R15
	REG_R16
	REG_R17
	REG_R18
	REG_R19
	REG_R20
	REG_R21
	REG_R22
	REG_R23
	REG_R24
	REG_R25
	REG_R26
	REG_R27
	REG_R28
	REG_R29
	REG_R30
	REG_R31

	// single-precision floating point
	REG_F0
	REG_F1
	REG_F2
	REG_F3
	REG_F4
	REG_F5
	REG_F6
	REG_F7
	REG_F8
	REG_F9
	REG_F10
	REG_F11
	REG_F12
	REG_F13
	REG_F14
	REG_F15
	REG_F16
	REG_F17
	REG_F18
	REG_F19
	REG_F20
	REG_F21
	REG_F22
	REG_F23
	REG_F24
	REG_F25
	REG_F26
	REG_F27
	REG_F28
	REG_F29
	REG_F30
	REG_F31

	// double-precision floating point; the first half is aliased to
	// single-precision registers, that is: Dn is aliased to Fn, Fn+1,
	// where n ≤ 30.
	REG_D0
	REG_D32
	REG_D2
	REG_D34
	REG_D4
	REG_D36
	REG_D6
	REG_D38
	REG_D8
	REG_D40
	REG_D10
	REG_D42
	REG_D12
	REG_D44
	REG_D14
	REG_D46
	REG_D16
	REG_D48
	REG_D18
	REG_D50
	REG_D20
	REG_D52
	REG_D22
	REG_D54
	REG_D24
	REG_D56
	REG_D26
	REG_D58
	REG_D28
	REG_D60
	REG_D30
	REG_D62
)

const (
	// floating-point condition-code registers
	REG_FCC0 = REG_R0 + 256 + iota
	REG_FCC1
	REG_FCC2
	REG_FCC3
)

const (
	// integer condition-code flags
	REG_ICC = REG_R0 + 384
	REG_XCC = REG_R0 + 384 + 2
)

const (
	REG_SPECIAL = REG_R0 + 512

	REG_CCR  = REG_SPECIAL + 2
	REG_TICK = REG_SPECIAL + 4
	REG_RPC  = REG_SPECIAL + 5

	REG_LAST = REG_R0 + 1024
)

// Register assignments:
const (
	REG_ZR  = REG_R0
	REG_TMP = REG_R1
	REG_RSP = REG_R14
	REG_LR  = REG_R15
	REG_GOT = REG_R23
	REG_RFP = REG_R30
	REG_PLR = REG_R31
)

const (
	StackBias             = 0x7ff  // craziness
	WindowSaveAreaSize    = 16 * 8 // only slots for RFP and PLR used
	ArgumentsSaveAreaSize = 0      // normally 6*8 on SPARC, but we cheat
	MinStackFrameSize     = WindowSaveAreaSize + ArgumentsSaveAreaSize
)

// Prog.mark
const (
	FOLL = 1 << iota
	LABEL
	LEAF
)

const (
	ClassUnknown = iota

	ClassReg        // R1..R31
	ClassFloatReg   // F0..F31
	ClassDoubleReg  // D0..D62
	ClassCond       // ICC, XCC
	ClassFloatCond  // FCC0..FCC3
	ClassSpecialReg // TICK, CCR, etc

	ClassZero       // $0 or ZR
	ClassConst5     // unsigned 5-bit constant
	ClassConst6     // unsigned 6-bit constant
	ClassConst13    // signed 13-bit constant
	ClassConst31_   // signed 32-bit constant, negative
	ClassConst31    // signed 32-bit constant, positive or zero
	ClassConst32    // 32-bit constant
	ClassConst      // 64-bit constant
	ClassFloatConst // floating-point constant

	ClassRegReg     // $(Rn+Rm) or $(Rn)(Rm*1)
	ClassRegConst13 // $n(R), n is 13-bit signed
	ClassRegConst   // $n(R), n large

	ClassIndirRegReg // (Rn+Rm) or (Rn)(Rm*1)
	ClassIndir0      // (R)
	ClassIndir13     // n(R), n is 13-bit signed
	ClassIndir       // n(R), n large

	ClassShortBranch // n(PC) branch target, n is 21-bit signed, mod 4

	ClassAddr // $sym(SB)
	ClassMem  // sym(SB)

	ClassTextSize
	ClassNone
)

var cnames = []string{
	ClassUnknown:     "ClassUnknown",
	ClassReg:         "ClassReg",
	ClassFloatReg:    "ClassFloatReg",
	ClassDoubleReg:   "ClassDoubleReg",
	ClassCond:        "ClassCond",
	ClassFloatCond:   "ClassFloatCond",
	ClassSpecialReg:  "ClassSpecialReg",
	ClassZero:        "ClassZero",
	ClassConst5:      "ClassConst5",
	ClassConst6:      "ClassConst6",
	ClassConst13:     "ClassConst13",
	ClassConst31_:    "ClassConst31-",
	ClassConst31:     "ClassConst31+",
	ClassConst32:     "ClassConst32",
	ClassConst:       "ClassConst",
	ClassFloatConst:  "ClassFloatConst",
	ClassRegReg:      "ClassRegReg",
	ClassRegConst13:  "ClassRegConst13",
	ClassRegConst:    "ClassRegConst",
	ClassIndirRegReg: "ClassIndirRegReg",
	ClassIndir0:      "ClassIndir0",
	ClassIndir13:     "ClassIndir13",
	ClassIndir:       "ClassIndir",
	ClassShortBranch: "ClassShortBranch",
	ClassAddr:        "ClassAddr",
	ClassMem:         "ClassMem",
	ClassTextSize:    "ClassTextSize",
	ClassNone:        "ClassNone",
}

//go:generate go run ../stringer.go -i $GOFILE -o anames.go -p sparc64

const (
	AADD = obj.ABaseSPARC64 + obj.A_ARCHSPECIFIC + iota
	AADDCC
	AADDC
	AADDCCC
	AAND
	AANDCC
	AANDN
	AANDNCC

	// These are the two-operand SPARCv9 32-, and 64-bit, branch
	// on integer condition codes with prediction (BPcc), not the
	// single-operand SPARCv8 32-bit branch on integer condition
	// codes (Bicc).
	ABN
	ABNE
	ABE
	ABG
	ABLE
	ABGE
	ABL
	ABGU
	ABLEU
	ABCC
	ABCS
	ABPOS
	ABNEG
	ABVC
	ABVS

	ABRZ
	ABRLEZ
	ABRLZ
	ABRNZ
	ABRGZ
	ABRGEZ
	ACASW
	ACASD
	AFABSS
	AFABSD
	AFADDS
	AFADDD
	AFBA
	AFBN
	AFBU
	AFBG
	AFBUG
	AFBL
	AFBUL
	AFBLG
	AFBNE
	AFBE
	AFBUE
	AFBGE
	AFBUGE
	AFBLE
	AFBULE
	AFBO
	AFCMPS
	AFCMPD
	AFDIVS
	AFDIVD
	AFITOS
	AFITOD
	AFLUSH
	AFMOVS // the SPARC64 instruction, and alias for loads and stores
	AFMOVD // the SPARC64 instruction, and alias for loads and stores
	AFMULS
	AFMULD
	AFSMULD
	AFNEGS
	AFNEGD
	AFSQRTS
	AFSQRTD
	AFSTOX
	AFDTOX
	AFSTOI
	AFDTOI
	AFSTOD
	AFDTOS
	AFSUBS
	AFSUBD
	AFXTOS
	AFXTOD
	AJMPL
	ALDSB
	ALDSH
	ALDSW
	ALDUB
	ALDUH
	ALDUW
	ALDD
	ALDSF
	ALDDF
	AMEMBAR
	AMULD
	ASDIVD
	AUDIVD
	AOR
	AORCC
	AORN
	AORNCC
	ARD
	ASETHI
	ASLLW
	ASRLW
	ASRAW
	ASLLD
	ASRLD
	ASRAD
	ASTB
	ASTH
	ASTW
	ASTD
	ASTSF
	ASTDF
	ASUB
	ASUBCC
	ASUBC
	ASUBCCC
	AXOR
	AXORCC
	AXNOR
	AXNORCC

	// Pseudo-instructions, aliases to SPARC64 instructions and
	// synthetic instructions.
	AMOVB
	AMOVSB
	AMOVH
	AMOVSH
	AMOVW
	AMOVSW
	AMOVD // also the SPARC64 synthetic instruction
	ARNOP // SETHI $0, ZR

	AWORD
	ADWORD
)
<|endoftext|>


<commit_before>package commands

import (
	"io"
	"os"
	"os/exec"
	"sync"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	checkoutCmd = &cobra.Command{
		Use: "checkout",
		Run: checkoutCommand,
	}
)

func checkoutCommand(cmd *cobra.Command, args []string) {
	requireInRepo()

	// Parameters are filters
	// firstly convert any pathspecs to the root of the repo, in case this is being executed in a sub-folder
	var rootedpaths []string

	inchan := make(chan string, 1)
	outchan, err := lfs.ConvertCwdFilesRelativeToRepo(inchan)
	if err != nil {
		Panic(err, "Could not checkout")
	}
	for _, arg := range args {
		inchan <- arg
		rootedpaths = append(rootedpaths, <-outchan)
	}
	close(inchan)
	checkoutWithIncludeExclude(rootedpaths, nil)
}

func init() {
	RootCmd.AddCommand(checkoutCmd)
}

// Checkout from items reported from the fetch process (in parallel)
func checkoutAllFromFetchChan(c chan *lfs.WrappedPointer) {
	tracerx.Printf("starting fetch/parallel checkout")
	checkoutFromFetchChan(nil, nil, c)
}

func checkoutFromFetchChan(include []string, exclude []string, in chan *lfs.WrappedPointer) {
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not checkout")
	}
	// Need to ScanTree to identify multiple files with the same content (fetch will only report oids once)
	pointers, err := lfs.ScanTree(ref.Sha)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}

	// Map oid to multiple pointers
	mapping := make(map[string][]*lfs.WrappedPointer)
	for _, pointer := range pointers {
		if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
			mapping[pointer.Oid] = append(mapping[pointer.Oid], pointer)
		}
	}

	// Launch git update-index
	c := make(chan *lfs.WrappedPointer)

	var wait sync.WaitGroup
	wait.Add(1)

	go func() {
		checkoutWithChan(c)
		wait.Done()
	}()

	// Feed it from in, which comes from fetch
	for p := range in {
		// Add all of the files for this oid
		for _, fp := range mapping[p.Oid] {
			c <- fp
		}
	}
	close(c)
	wait.Wait()
}

func checkoutWithIncludeExclude(include []string, exclude []string) {
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not checkout")
	}

	pointers, err := lfs.ScanTree(ref.Sha)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}

	var wait sync.WaitGroup
	wait.Add(1)

	c := make(chan *lfs.WrappedPointer, 1)

	go func() {
		checkoutWithChan(c)
		wait.Done()
	}()

	// Count bytes for progress
	var totalBytes int64
	for _, pointer := range pointers {
		totalBytes += pointer.Size
	}
	progress := lfs.NewProgressMeter(len(pointers), totalBytes, false)
	progress.Start()
	totalBytes = 0
	for _, pointer := range pointers {
		totalBytes += pointer.Size
		if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
			progress.Add(pointer.Name)
			c <- pointer
			// not strictly correct (parallel) but we don't have a callback & it's just local
			// plus only 1 slot in channel so it'll block & be close
			progress.TransferBytes("checkout", pointer.Name, pointer.Size, totalBytes, int(pointer.Size))
			progress.FinishTransfer(pointer.Name)
		} else {
			progress.Skip(pointer.Size)
		}
	}
	close(c)
	wait.Wait()
	progress.Finish()

}

func checkoutAll() {
	checkoutWithIncludeExclude(nil, nil)
}

// Populate the working copy with the real content of objects where the file is
// either missing, or contains a matching pointer placeholder, from a list of pointers.
// If the file exists but has other content it is left alone
// Callers of this function MUST NOT Panic or otherwise exit the process
// without waiting for this function to shut down.  If the process exits while
// update-index is in the middle of processing a file the git index can be left
// in a locked state.
func checkoutWithChan(in <-chan *lfs.WrappedPointer) {
	// Get a converter from repo-relative to cwd-relative
	// Since writing data & calling git update-index must be relative to cwd
	repopathchan := make(chan string, 1)
	cwdpathchan, err := lfs.ConvertRepoFilesRelativeToCwd(repopathchan)
	if err != nil {
		Panic(err, "Could not convert file paths")
	}

	// Don't fire up the update-index command until we have at least one file to
	// give it. Otherwise git interprets the lack of arguments to mean param-less update-index
	// which can trigger entire working copy to be re-examined, which triggers clean filters
	// and which has unexpected side effects (e.g. downloading filtered-out files)
	var cmd *exec.Cmd
	var updateIdxStdin io.WriteCloser

	// From this point on, git update-index is running. Code in this loop MUST
	// NOT Panic() or otherwise cause the process to exit. If the process exits
	// while update-index is in the middle of updating, the index can remain in a
	// locked state.

	// As files come in, write them to the wd and update the index
	for pointer := range in {

		// Check the content - either missing or still this pointer (not exist is ok)
		filepointer, err := lfs.DecodePointerFromFile(pointer.Name)
		if err != nil && !os.IsNotExist(err) {
			if lfs.IsNotAPointerError(err) {
				// File has non-pointer content, leave it alone
				continue
			}
			LoggedError(err, "Problem accessing %v", pointer.Name)
			continue
		}

		if filepointer != nil && filepointer.Oid != pointer.Oid {
			// User has probably manually reset a file to another commit
			// while leaving it a pointer; don't mess with this
			continue
		}

		repopathchan <- pointer.Name
		cwdfilepath := <-cwdpathchan

		err = lfs.PointerSmudgeToFile(cwdfilepath, pointer.Pointer, false, nil)
		if err != nil {
			if lfs.IsDownloadDeclinedError(err) {
				// acceptable error, data not local (fetch not run or include/exclude)
				LoggedError(err, "Skipped checkout for %v, content not local. Use fetch to download.", pointer.Name)
			} else {
				LoggedError(err, "Could not checkout file")
				continue
			}
		}

		if cmd == nil {
			// Fire up the update-index command
			cmd = exec.Command("git", "update-index", "-q", "--refresh", "--stdin")
			updateIdxStdin, err = cmd.StdinPipe()
			if err != nil {
				Panic(err, "Could not update the index")
			}

			if err := cmd.Start(); err != nil {
				Panic(err, "Could not update the index")
			}

		}

		updateIdxStdin.Write([]byte(cwdfilepath + "\n"))
	}
	close(repopathchan)

	if cmd != nil && updateIdxStdin != nil {
		updateIdxStdin.Close()
		if err := cmd.Wait(); err != nil {
			Panic(err, "Error updating the git index")
		}
	}
}
<commit_msg>Use LoggedError instead of Panic if update-index fails in checkout #725<commit_after>package commands

import (
	"io"
	"os"
	"os/exec"
	"sync"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	checkoutCmd = &cobra.Command{
		Use: "checkout",
		Run: checkoutCommand,
	}
)

func checkoutCommand(cmd *cobra.Command, args []string) {
	requireInRepo()

	// Parameters are filters
	// firstly convert any pathspecs to the root of the repo, in case this is being executed in a sub-folder
	var rootedpaths []string

	inchan := make(chan string, 1)
	outchan, err := lfs.ConvertCwdFilesRelativeToRepo(inchan)
	if err != nil {
		Panic(err, "Could not checkout")
	}
	for _, arg := range args {
		inchan <- arg
		rootedpaths = append(rootedpaths, <-outchan)
	}
	close(inchan)
	checkoutWithIncludeExclude(rootedpaths, nil)
}

func init() {
	RootCmd.AddCommand(checkoutCmd)
}

// Checkout from items reported from the fetch process (in parallel)
func checkoutAllFromFetchChan(c chan *lfs.WrappedPointer) {
	tracerx.Printf("starting fetch/parallel checkout")
	checkoutFromFetchChan(nil, nil, c)
}

func checkoutFromFetchChan(include []string, exclude []string, in chan *lfs.WrappedPointer) {
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not checkout")
	}
	// Need to ScanTree to identify multiple files with the same content (fetch will only report oids once)
	pointers, err := lfs.ScanTree(ref.Sha)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}

	// Map oid to multiple pointers
	mapping := make(map[string][]*lfs.WrappedPointer)
	for _, pointer := range pointers {
		if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
			mapping[pointer.Oid] = append(mapping[pointer.Oid], pointer)
		}
	}

	// Launch git update-index
	c := make(chan *lfs.WrappedPointer)

	var wait sync.WaitGroup
	wait.Add(1)

	go func() {
		checkoutWithChan(c)
		wait.Done()
	}()

	// Feed it from in, which comes from fetch
	for p := range in {
		// Add all of the files for this oid
		for _, fp := range mapping[p.Oid] {
			c <- fp
		}
	}
	close(c)
	wait.Wait()
}

func checkoutWithIncludeExclude(include []string, exclude []string) {
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not checkout")
	}

	pointers, err := lfs.ScanTree(ref.Sha)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}

	var wait sync.WaitGroup
	wait.Add(1)

	c := make(chan *lfs.WrappedPointer, 1)

	go func() {
		checkoutWithChan(c)
		wait.Done()
	}()

	// Count bytes for progress
	var totalBytes int64
	for _, pointer := range pointers {
		totalBytes += pointer.Size
	}
	progress := lfs.NewProgressMeter(len(pointers), totalBytes, false)
	progress.Start()
	totalBytes = 0
	for _, pointer := range pointers {
		totalBytes += pointer.Size
		if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
			progress.Add(pointer.Name)
			c <- pointer
			// not strictly correct (parallel) but we don't have a callback & it's just local
			// plus only 1 slot in channel so it'll block & be close
			progress.TransferBytes("checkout", pointer.Name, pointer.Size, totalBytes, int(pointer.Size))
			progress.FinishTransfer(pointer.Name)
		} else {
			progress.Skip(pointer.Size)
		}
	}
	close(c)
	wait.Wait()
	progress.Finish()

}

func checkoutAll() {
	checkoutWithIncludeExclude(nil, nil)
}

// Populate the working copy with the real content of objects where the file is
// either missing, or contains a matching pointer placeholder, from a list of pointers.
// If the file exists but has other content it is left alone
// Callers of this function MUST NOT Panic or otherwise exit the process
// without waiting for this function to shut down.  If the process exits while
// update-index is in the middle of processing a file the git index can be left
// in a locked state.
func checkoutWithChan(in <-chan *lfs.WrappedPointer) {
	// Get a converter from repo-relative to cwd-relative
	// Since writing data & calling git update-index must be relative to cwd
	repopathchan := make(chan string, 1)
	cwdpathchan, err := lfs.ConvertRepoFilesRelativeToCwd(repopathchan)
	if err != nil {
		Panic(err, "Could not convert file paths")
	}

	// Don't fire up the update-index command until we have at least one file to
	// give it. Otherwise git interprets the lack of arguments to mean param-less update-index
	// which can trigger entire working copy to be re-examined, which triggers clean filters
	// and which has unexpected side effects (e.g. downloading filtered-out files)
	var cmd *exec.Cmd
	var updateIdxStdin io.WriteCloser

	// From this point on, git update-index is running. Code in this loop MUST
	// NOT Panic() or otherwise cause the process to exit. If the process exits
	// while update-index is in the middle of updating, the index can remain in a
	// locked state.

	// As files come in, write them to the wd and update the index
	for pointer := range in {

		// Check the content - either missing or still this pointer (not exist is ok)
		filepointer, err := lfs.DecodePointerFromFile(pointer.Name)
		if err != nil && !os.IsNotExist(err) {
			if lfs.IsNotAPointerError(err) {
				// File has non-pointer content, leave it alone
				continue
			}
			LoggedError(err, "Problem accessing %v", pointer.Name)
			continue
		}

		if filepointer != nil && filepointer.Oid != pointer.Oid {
			// User has probably manually reset a file to another commit
			// while leaving it a pointer; don't mess with this
			continue
		}

		repopathchan <- pointer.Name
		cwdfilepath := <-cwdpathchan

		err = lfs.PointerSmudgeToFile(cwdfilepath, pointer.Pointer, false, nil)
		if err != nil {
			if lfs.IsDownloadDeclinedError(err) {
				// acceptable error, data not local (fetch not run or include/exclude)
				LoggedError(err, "Skipped checkout for %v, content not local. Use fetch to download.", pointer.Name)
			} else {
				LoggedError(err, "Could not checkout file")
				continue
			}
		}

		if cmd == nil {
			// Fire up the update-index command
			cmd = exec.Command("git", "update-index", "-q", "--refresh", "--stdin")
			updateIdxStdin, err = cmd.StdinPipe()
			if err != nil {
				Panic(err, "Could not update the index")
			}

			if err := cmd.Start(); err != nil {
				Panic(err, "Could not update the index")
			}

		}

		updateIdxStdin.Write([]byte(cwdfilepath + "\n"))
	}
	close(repopathchan)

	if cmd != nil && updateIdxStdin != nil {
		updateIdxStdin.Close()
		if err := cmd.Wait(); err != nil {
			outp, _ := cmd.CombinedOutput()
			LoggedError(err, "Error updating the git index\n%v", string(outp))
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package client

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"reflect"
	"testing"
	"time"

	"github.com/romana/core/common"
	"github.com/romana/core/common/api"
)

var (
	client *Client
	// Keep track of state of some tests
	state int
)

func init() {
	rand.Seed(time.Now().UnixNano())
	testSaver = &TestSaver{}
}

func tearDown(t *testing.T) {
	var err error
	err = client.Store.DeleteTree(client.config.EtcdPrefix)
	if err != nil {
		t.Errorf("Error tearing down: %s", err)
	}
	t.Logf("Tore down %s", client.config.EtcdPrefix)
}

func initClient(t *testing.T, topoConf string) *Client {
	var err error
	cfg := &common.Config{EtcdEndpoints: []string{"localhost:2379"},
		EtcdPrefix: fmt.Sprintf("/romanaTest%d", rand.Int63n(100000)),
	}
	client, err := NewClient(cfg)
	if err != nil {
		t.Fatal(err)
	}
	if topoConf != "" {
		topoReq := api.TopologyUpdateRequest{}
		err = json.Unmarshal([]byte(topoConf), &topoReq)
		if err != nil {
			t.Fatalf("Cannot parse %s: %v", topoConf, err)
		}
		err = client.IPAM.UpdateTopology(topoReq, false)
		if err != nil {
			t.Fatal(err)
		}
	}
	return client
}

// TestWatchHostsWithCallback tests WatchHostsWithCallback -- and since it
// uses WatchHosts internally, implicitly also WatchHosts
/*
func TestWatchHostsWithCallback(t *testing.T) {
	client := initClient(t, "")
	defer tearDown(t)
	errCh := make(chan string)
	okCh := make(chan string)
	err := client.WatchHostsWithCallback(func(hl api.HostList) {
		state++

		var msg string
		fmt.Printf("HostListCallback: Got host list of length %d and revision %d in state %d\n", len(hl.Hosts), hl.Revision, state)
		if state == 1 {
			if len(hl.Hosts) != 0 {
				msg = fmt.Sprintf("HostListCallback: Expected host length at this point to be 0, got %d", len(hl.Hosts))
				fmt.Println(msg)
				errCh <- msg
				return
			}
			if hl.Revision != 0 {
				msg = fmt.Sprintf("HostListCallback: Expected host list revision at this point to be 0, got %d", hl.Revision)
				fmt.Println(msg)
				errCh <- msg
				return
			}
		} else if state == 2 {
			if len(hl.Hosts) != 2 {
				msg = fmt.Sprintf("HostListCallback: Expected host length at this point to be 2, got %d", len(hl.Hosts))
				fmt.Println(msg)
				errCh <- msg
				return
			}
			if hl.Revision != 1 {
				msg = fmt.Sprintf("HostListCallback: Expected host list revision at this point to be 1, got %d", hl.Revision)
				fmt.Println(msg)
				errCh <- msg
				return
			}
			// Here everything is ok!
			okCh <- "OK"
		} else {
			msg = fmt.Sprintf("HostListCallback: Did not expect HostListCallback to be called in state %d", state)
			fmt.Println(state)
			errCh <- msg

		}

	})
	if err != nil {
		t.Fatal(err)
	}

	topoConf := `{
	"networks": [{
		"name": "net1",
		"cidr": "10.0.0.0/8",
		"block_mask": 30
	}],

	"topologies": [{
		"networks": ["net1"],
		"map": [{
			"routing": "bla",
			"groups": [{
					"name": "host1",
					"ip": "192.168.99.10"
				},
				{
					"name": "host2",
					"ip": "192.168.99.11"
				}
			]
		}]
	}]
}`
	topoReq := api.TopologyUpdateRequest{}
	err = json.Unmarshal([]byte(topoConf), &topoReq)
	if err != nil {
		t.Fatalf("Cannot parse %s: %v", topoConf, err)
	}
	err = client.IPAM.UpdateTopology(topoReq)
	if err != nil {
		t.Fatal(err)
	}
	for {
		select {
		case msg := <-errCh:
			t.Fatal(msg)
			break
		case <-okCh:
			return
		}
	}

}
*/

// TestWatchBlocksWithCallback tests WatchBlocksWithCallback -- and since it
// uses WatchBlocks internally, implicitly also WatchBlocks
/*
func TestWatchBlocksWithCallback(t *testing.T) {
	defer tearDown(t)
	errCh := make(chan string)
	okCh := make(chan string)
	topoConf := `{
	"networks": [{
		"name": "net1",
		"cidr": "10.0.0.0/8",
		"block_mask": 31
	}],

	"topologies": [{
		"networks": ["net1"],
		"map": [{
			"routing": "bla",
			"groups": [{
					"name": "host1",
					"ip": "192.168.99.10"
				},
				{
					"name": "host2",
					"ip": "192.168.99.11"
				}
			]
		}]
	}]
}`
	client = initClient(t, topoConf)
	err := client.WatchBlocksWithCallback(func(hl api.IPAMBlocksResponse) {
		state++

		var msg string
		fmt.Printf("BlocksListCallback: Got block list of length %d and revision %d in state %d\n", len(hl.Blocks), hl.Revision, state)
		switch state {
		case 1:
			if len(hl.Blocks) != 0 {
				msg = fmt.Sprintf("BlocksListCallback: Expected block length at this point to be 0, got %d", len(hl.Blocks))
				fmt.Println(msg)
				errCh <- msg
				return
			}
			if hl.Revision != 0 {
				msg = fmt.Sprintf("BlocksListCallback: Expected block list revision at this point to be 0, got %d", hl.Revision)
				fmt.Println(msg)
				errCh <- msg
				return
			}
		case 2, 3:
			if len(hl.Blocks) != 1 {
				msg = fmt.Sprintf("BlocksListCallback: Expected block length at this point to be 1, got %d", len(hl.Blocks))
				fmt.Println(msg)
				errCh <- msg
				return
			}
			if hl.Revision != state-1 {
				msg = fmt.Sprintf("HostListCallback: Expected block list revision at this point to be %d, got %d", state-1, hl.Revision)
				fmt.Println(msg)
				errCh <- msg
				return
			}

		case 4, 5:
			if len(hl.Blocks) != 2 {
				msg = fmt.Sprintf("BlocksListCallback: Expected block length at this point to be 2, got %d", len(hl.Blocks))
				fmt.Println(msg)
				errCh <- msg
				return
			}
			if hl.Revision != state-1 {
				msg = fmt.Sprintf("HostListCallback: Expected block list revision at this point to be %d, got %d", state-1, hl.Revision)
				fmt.Println(msg)
				errCh <- msg
				return
			}
			if state == 5 {
				// Here everything is ok!
				okCh <- "OK"
			}
		default:
			msg = fmt.Sprintf("HostListCallback: Did not expect HostListCallback to be called in state %d", state)
			errCh <- msg

		}

	})
	if err != nil {
		t.Fatal(err)
	}

	_, err = client.IPAM.AllocateIP("addr1", "host1", "t1", "s1")
	if err != nil {
		t.Fatal(err)
	}
	_, err = client.IPAM.AllocateIP("addr2", "host1", "t1", "s1")
	if err != nil {
		t.Fatal(err)
	}
	_, err = client.IPAM.AllocateIP("addr3", "host1", "t1", "s1")
	if err != nil {
		t.Fatal(err)
	}
	_, err = client.IPAM.AllocateIP("addr4", "host1", "t1", "s1")
	if err != nil {
		t.Fatal(err)
	}
	for {
		select {
		case msg := <-errCh:
			t.Fatal(msg)
			break
		case <-okCh:
			return
		}
	}
}
*/

/*
func TestConcurrency(t *testing.T) {
	//	defer tearDown(t)
	client = initClient(t, "")

	barrier := make(chan int)
	cnt := 8
	locker, err := client.Store.NewLocker("/lock")
	if err != nil {
		t.Fatal(err)
	}
	for i := 0; i < cnt; i++ {
		go func(i int) {
			fmt.Printf("%d %d Entered\n", i, getGID())
			defer func(i int) {
				fmt.Printf("%d %d Unlocking\n", i, getGID())
				locker.Unlock()
				fmt.Printf("%d %d Unlocked\n", i, getGID())
			}(i)
			fmt.Printf("%d %d Waiting for lock\n", i, getGID())
			locker.Lock()
			fmt.Printf("%d %d Got lock\n", i, getGID())
			val := fmt.Sprintf("Hello from %d %d", i, getGID())
			err := client.Store.Put(fmt.Sprintf("/test%d", i), []byte(val), nil)
			if err != nil {
				fmt.Println(err)
			}
			barrier <- 1
		}(i)
	}
	finishedCnt := 0

WAIT_FOR_FINISH:
	for {
		select {
		case <-barrier:
			finishedCnt++
			fmt.Printf("%d routines finished\n", finishedCnt)
			if finishedCnt == cnt {
				break WAIT_FOR_FINISH
			}
		}
	}
}
*/

type testcase struct {
	name string
	ipam *IPAM
	want interface{}
}

func loadTestFiles(name string, inFile string, outFile string) (testcase, error) {
	input, err := ioutil.ReadFile(inFile)
	if err != nil {
		return testcase{}, fmt.Errorf("error reading test input file: %s", err)
	}
	output, err := ioutil.ReadFile(outFile)
	if err != nil {
		return testcase{}, fmt.Errorf("error reading test output file: %s", err)
	}

	ipamState := &IPAM{}
	err = json.Unmarshal(input, ipamState)
	if err != nil {
		return testcase{}, fmt.Errorf("failed to unmarshal ipam information: %s", err)
	}
	topologyState := &api.TopologyUpdateRequest{}
	err = json.Unmarshal(output, topologyState)
	if err != nil {
		return testcase{}, fmt.Errorf("failed to unmarshal topology information: %s", err)
	}

	return testcase{name, ipamState, topologyState}, nil
}

func Test_getTopologyFromIPAMState(t *testing.T) {
	var testcases []testcase

	testfiles := map[string][]string{
		"multinetwork": {
			"testdata/TestTopologyGetInputMultiNetworks.json",
			"testdata/TestTopologyGetOutputMultiNetworks.json",
		},
		"kubeadm": {
			"testdata/TestTopologyGetInputKubeadm.json",
			"testdata/TestTopologyGetOutputKubeadm.json",
		},
	}

	for name, file := range testfiles {
		tc, err := loadTestFiles(name, file[0], file[1])
		if err != nil {
			t.Errorf("error loading test files: %s", err)
			continue
		}
		testcases = append(testcases, tc)
	}

	for _, tt := range testcases {
		t.Run(tt.name, func(t *testing.T) {
			// TODO: got and tt.want below are not sorted, so there is a fair chance the
			// testcases here may fail, in which case this needs to be fixed by sorting
			// it first.
			if got := getTopologyFromIPAMState(tt.ipam); !reflect.DeepEqual(got, tt.want) {
				bodyGot, errGot := json.MarshalIndent(got, "", "\t")
				bodyWant, errWant := json.MarshalIndent(tt.want, "", "\t")
				if errGot == nil && errWant == nil {
					t.Errorf("getTopology() = \ngot(%s)\nwant(%s)",
						string(bodyGot), string(bodyWant))
				} else {
					t.Errorf("getTopology() = \ngot(%#v)\nwant(%#v)", got, tt.want)
				}
			}
		})
	}
}
<commit_msg>client: comment unused functions in tests.<commit_after>// Copyright (c) 2017 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package client

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"reflect"
	"testing"
	"time"

	"github.com/romana/core/common"
	"github.com/romana/core/common/api"
)

/*
var (
	client *Client
	// Keep track of state of some tests
	state int
)

func init() {
	rand.Seed(time.Now().UnixNano())
	testSaver = &TestSaver{}
}

func tearDown(t *testing.T) {
	var err error
	err = client.Store.DeleteTree(client.config.EtcdPrefix)
	if err != nil {
		t.Errorf("Error tearing down: %s", err)
	}
	t.Logf("Tore down %s", client.config.EtcdPrefix)
}

func initClient(t *testing.T, topoConf string) *Client {
	var err error
	cfg := &common.Config{EtcdEndpoints: []string{"localhost:2379"},
		EtcdPrefix: fmt.Sprintf("/romanaTest%d", rand.Int63n(100000)),
	}
	client, err := NewClient(cfg)
	if err != nil {
		t.Fatal(err)
	}
	if topoConf != "" {
		topoReq := api.TopologyUpdateRequest{}
		err = json.Unmarshal([]byte(topoConf), &topoReq)
		if err != nil {
			t.Fatalf("Cannot parse %s: %v", topoConf, err)
		}
		err = client.IPAM.UpdateTopology(topoReq, false)
		if err != nil {
			t.Fatal(err)
		}
	}
	return client
}

// TestWatchHostsWithCallback tests WatchHostsWithCallback -- and since it
// uses WatchHosts internally, implicitly also WatchHosts
/*
func TestWatchHostsWithCallback(t *testing.T) {
	client := initClient(t, "")
	defer tearDown(t)
	errCh := make(chan string)
	okCh := make(chan string)
	err := client.WatchHostsWithCallback(func(hl api.HostList) {
		state++

		var msg string
		fmt.Printf("HostListCallback: Got host list of length %d and revision %d in state %d\n", len(hl.Hosts), hl.Revision, state)
		if state == 1 {
			if len(hl.Hosts) != 0 {
				msg = fmt.Sprintf("HostListCallback: Expected host length at this point to be 0, got %d", len(hl.Hosts))
				fmt.Println(msg)
				errCh <- msg
				return
			}
			if hl.Revision != 0 {
				msg = fmt.Sprintf("HostListCallback: Expected host list revision at this point to be 0, got %d", hl.Revision)
				fmt.Println(msg)
				errCh <- msg
				return
			}
		} else if state == 2 {
			if len(hl.Hosts) != 2 {
				msg = fmt.Sprintf("HostListCallback: Expected host length at this point to be 2, got %d", len(hl.Hosts))
				fmt.Println(msg)
				errCh <- msg
				return
			}
			if hl.Revision != 1 {
				msg = fmt.Sprintf("HostListCallback: Expected host list revision at this point to be 1, got %d", hl.Revision)
				fmt.Println(msg)
				errCh <- msg
				return
			}
			// Here everything is ok!
			okCh <- "OK"
		} else {
			msg = fmt.Sprintf("HostListCallback: Did not expect HostListCallback to be called in state %d", state)
			fmt.Println(state)
			errCh <- msg

		}

	})
	if err != nil {
		t.Fatal(err)
	}

	topoConf := `{
	"networks": [{
		"name": "net1",
		"cidr": "10.0.0.0/8",
		"block_mask": 30
	}],

	"topologies": [{
		"networks": ["net1"],
		"map": [{
			"routing": "bla",
			"groups": [{
					"name": "host1",
					"ip": "192.168.99.10"
				},
				{
					"name": "host2",
					"ip": "192.168.99.11"
				}
			]
		}]
	}]
}`
	topoReq := api.TopologyUpdateRequest{}
	err = json.Unmarshal([]byte(topoConf), &topoReq)
	if err != nil {
		t.Fatalf("Cannot parse %s: %v", topoConf, err)
	}
	err = client.IPAM.UpdateTopology(topoReq)
	if err != nil {
		t.Fatal(err)
	}
	for {
		select {
		case msg := <-errCh:
			t.Fatal(msg)
			break
		case <-okCh:
			return
		}
	}

}
*/

// TestWatchBlocksWithCallback tests WatchBlocksWithCallback -- and since it
// uses WatchBlocks internally, implicitly also WatchBlocks
/*
func TestWatchBlocksWithCallback(t *testing.T) {
	defer tearDown(t)
	errCh := make(chan string)
	okCh := make(chan string)
	topoConf := `{
	"networks": [{
		"name": "net1",
		"cidr": "10.0.0.0/8",
		"block_mask": 31
	}],

	"topologies": [{
		"networks": ["net1"],
		"map": [{
			"routing": "bla",
			"groups": [{
					"name": "host1",
					"ip": "192.168.99.10"
				},
				{
					"name": "host2",
					"ip": "192.168.99.11"
				}
			]
		}]
	}]
}`
	client = initClient(t, topoConf)
	err := client.WatchBlocksWithCallback(func(hl api.IPAMBlocksResponse) {
		state++

		var msg string
		fmt.Printf("BlocksListCallback: Got block list of length %d and revision %d in state %d\n", len(hl.Blocks), hl.Revision, state)
		switch state {
		case 1:
			if len(hl.Blocks) != 0 {
				msg = fmt.Sprintf("BlocksListCallback: Expected block length at this point to be 0, got %d", len(hl.Blocks))
				fmt.Println(msg)
				errCh <- msg
				return
			}
			if hl.Revision != 0 {
				msg = fmt.Sprintf("BlocksListCallback: Expected block list revision at this point to be 0, got %d", hl.Revision)
				fmt.Println(msg)
				errCh <- msg
				return
			}
		case 2, 3:
			if len(hl.Blocks) != 1 {
				msg = fmt.Sprintf("BlocksListCallback: Expected block length at this point to be 1, got %d", len(hl.Blocks))
				fmt.Println(msg)
				errCh <- msg
				return
			}
			if hl.Revision != state-1 {
				msg = fmt.Sprintf("HostListCallback: Expected block list revision at this point to be %d, got %d", state-1, hl.Revision)
				fmt.Println(msg)
				errCh <- msg
				return
			}

		case 4, 5:
			if len(hl.Blocks) != 2 {
				msg = fmt.Sprintf("BlocksListCallback: Expected block length at this point to be 2, got %d", len(hl.Blocks))
				fmt.Println(msg)
				errCh <- msg
				return
			}
			if hl.Revision != state-1 {
				msg = fmt.Sprintf("HostListCallback: Expected block list revision at this point to be %d, got %d", state-1, hl.Revision)
				fmt.Println(msg)
				errCh <- msg
				return
			}
			if state == 5 {
				// Here everything is ok!
				okCh <- "OK"
			}
		default:
			msg = fmt.Sprintf("HostListCallback: Did not expect HostListCallback to be called in state %d", state)
			errCh <- msg

		}

	})
	if err != nil {
		t.Fatal(err)
	}

	_, err = client.IPAM.AllocateIP("addr1", "host1", "t1", "s1")
	if err != nil {
		t.Fatal(err)
	}
	_, err = client.IPAM.AllocateIP("addr2", "host1", "t1", "s1")
	if err != nil {
		t.Fatal(err)
	}
	_, err = client.IPAM.AllocateIP("addr3", "host1", "t1", "s1")
	if err != nil {
		t.Fatal(err)
	}
	_, err = client.IPAM.AllocateIP("addr4", "host1", "t1", "s1")
	if err != nil {
		t.Fatal(err)
	}
	for {
		select {
		case msg := <-errCh:
			t.Fatal(msg)
			break
		case <-okCh:
			return
		}
	}
}
*/

/*
func TestConcurrency(t *testing.T) {
	//	defer tearDown(t)
	client = initClient(t, "")

	barrier := make(chan int)
	cnt := 8
	locker, err := client.Store.NewLocker("/lock")
	if err != nil {
		t.Fatal(err)
	}
	for i := 0; i < cnt; i++ {
		go func(i int) {
			fmt.Printf("%d %d Entered\n", i, getGID())
			defer func(i int) {
				fmt.Printf("%d %d Unlocking\n", i, getGID())
				locker.Unlock()
				fmt.Printf("%d %d Unlocked\n", i, getGID())
			}(i)
			fmt.Printf("%d %d Waiting for lock\n", i, getGID())
			locker.Lock()
			fmt.Printf("%d %d Got lock\n", i, getGID())
			val := fmt.Sprintf("Hello from %d %d", i, getGID())
			err := client.Store.Put(fmt.Sprintf("/test%d", i), []byte(val), nil)
			if err != nil {
				fmt.Println(err)
			}
			barrier <- 1
		}(i)
	}
	finishedCnt := 0

WAIT_FOR_FINISH:
	for {
		select {
		case <-barrier:
			finishedCnt++
			fmt.Printf("%d routines finished\n", finishedCnt)
			if finishedCnt == cnt {
				break WAIT_FOR_FINISH
			}
		}
	}
}
*/

type testcase struct {
	name string
	ipam *IPAM
	want interface{}
}

func loadTestFiles(name string, inFile string, outFile string) (testcase, error) {
	input, err := ioutil.ReadFile(inFile)
	if err != nil {
		return testcase{}, fmt.Errorf("error reading test input file: %s", err)
	}
	output, err := ioutil.ReadFile(outFile)
	if err != nil {
		return testcase{}, fmt.Errorf("error reading test output file: %s", err)
	}

	ipamState := &IPAM{}
	err = json.Unmarshal(input, ipamState)
	if err != nil {
		return testcase{}, fmt.Errorf("failed to unmarshal ipam information: %s", err)
	}
	topologyState := &api.TopologyUpdateRequest{}
	err = json.Unmarshal(output, topologyState)
	if err != nil {
		return testcase{}, fmt.Errorf("failed to unmarshal topology information: %s", err)
	}

	return testcase{name, ipamState, topologyState}, nil
}

func Test_getTopologyFromIPAMState(t *testing.T) {
	var testcases []testcase

	testfiles := map[string][]string{
		"multinetwork": {
			"testdata/TestTopologyGetInputMultiNetworks.json",
			"testdata/TestTopologyGetOutputMultiNetworks.json",
		},
		"kubeadm": {
			"testdata/TestTopologyGetInputKubeadm.json",
			"testdata/TestTopologyGetOutputKubeadm.json",
		},
	}

	for name, file := range testfiles {
		tc, err := loadTestFiles(name, file[0], file[1])
		if err != nil {
			t.Errorf("error loading test files: %s", err)
			continue
		}
		testcases = append(testcases, tc)
	}

	for _, tt := range testcases {
		t.Run(tt.name, func(t *testing.T) {
			// TODO: got and tt.want below are not sorted, so there is a fair chance the
			// testcases here may fail, in which case this needs to be fixed by sorting
			// it first.
			if got := getTopologyFromIPAMState(tt.ipam); !reflect.DeepEqual(got, tt.want) {
				bodyGot, errGot := json.MarshalIndent(got, "", "\t")
				bodyWant, errWant := json.MarshalIndent(tt.want, "", "\t")
				if errGot == nil && errWant == nil {
					t.Errorf("getTopology() = \ngot(%s)\nwant(%s)",
						string(bodyGot), string(bodyWant))
				} else {
					t.Errorf("getTopology() = \ngot(%#v)\nwant(%#v)", got, tt.want)
				}
			}
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"log"
	"math"
	"math/rand"
	"time"

	_ "github.com/c4pt0r/mysql"
	"github.com/kyokomi/emoji"
)

func createTypeTestTbls(db *sql.DB) {
	if b, err := isTblExists(db, "int_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE int_test(id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(id), data INT)`)
	} else if err != nil {
		log.Fatal(err)
	}

	if b, err := isTblExists(db, "double_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE double_test(id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(id), data DOUBLE)`)
	} else if err != nil {
		log.Fatal(err)
	}

	if b, err := isTblExists(db, "varchar_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE varchar_test(id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(id), data VARCHAR(1024))`)
	} else if err != nil {
		log.Fatal(err)
	}

	if b, err := isTblExists(db, "text_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE text_test(id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(id), data TEXT)`)
	} else if err != nil {
		log.Fatal(err)
	}

	if b, err := isTblExists(db, "blob_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE blob_test(id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(id), data BLOB)`)
	} else if err != nil {
		log.Fatal(err)
	}

	if b, err := isTblExists(db, "datetime_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE datetime_test(id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(id), data DATETIME)`)
	} else if err != nil {
		log.Fatal(err)
	}

	if b, err := isTblExists(db, "date_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE date_test(id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(id), data DATE)`)
	} else if err != nil {
		log.Fatal(err)
	}

	if b, err := isTblExists(db, "multi_primary_key_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE multi_primary_key_test(id1 INT, id2 INT, PRIMARY KEY(id1, id2), data INT)`)
	} else if err != nil {
		log.Fatal(err)
	}

	if b, err := isTblExists(db, "emoji_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE emoji_test(id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(id), data VARCHAR(1024))`)
	} else if err != nil {
		log.Fatal(err)
	}

}

func dropTypeTestTbls(db *sql.DB) {
	tbls := []string{
		"int_test",
		"double_test",
		"varchar_test",
		"text_test",
		"blob_test",
		"datetime_test",
		"date_test",
	}

	for _, t := range tbls {
		if b, _ := isTblExists(db, t); b {
			mustExec(db, "DROP TABLE "+t)
		}
	}
}

func insertDataAndQueryBack(db *sql.DB, tblName string, val interface{}, ret interface{}, cachedRet interface{}) error {
	r := mustExec(db, "INSERT INTO "+tblName+"(data) VALUES(?)", val)
	id, err := r.LastInsertId()
	if err != nil {
		return err
	}

	res, err := db.Query("SELECT data FROM "+tblName+" WHERE id = ?", id)
	if err != nil {
		return err
	}
	defer res.Close()

	for res.Next() {
		res.Scan(ret)
		if err != nil {
			return err
		}
	}

	cachedRes, err := db.Query("SELECT data FROM "+tblName+" WHERE id = ?", id)
	if err != nil {
		return err
	}
	defer cachedRes.Close()

	for cachedRes.Next() {
		cachedRes.Scan(cachedRet)
		if err != nil {
			return err
		}
	}
	return nil
}

func multiPKeyTest(db *sql.DB) error {
	x := rand.Intn(1024)
	id1 := rand.Intn(1024)
	id2 := rand.Intn(1024)
	tblName := "multi_primary_key_test"

	mustExec(db, "INSERT INTO multi_primary_key_test(id1, id2, data) VALUES(?, ?, ?)", id1, id2, x)

	res, err := db.Query("SELECT data FROM "+tblName+" WHERE id1 = ? and id2 = ?", id1, id2)
	if err != nil {
		return err
	}
	defer res.Close()

	for res.Next() {
		var ret int
		res.Scan(&ret)
		if err != nil {
			return err
		}
		if ret != x {
			return fmt.Errorf("multi pkey test failed %d != %d", x, ret)
		}
	}

	cachedRes, err := db.Query("SELECT data FROM "+tblName+" WHERE id1 = ? and id2 = ?", id1, id2)
	if err != nil {
		return err
	}
	defer cachedRes.Close()

	for cachedRes.Next() {
		var ret int
		cachedRes.Scan(&ret)
		if err != nil {
			return err
		}
		if ret != x {
			return fmt.Errorf("multi pkey test cache failed %d != %d", ret, x)
		}
	}
	return nil
}

func intTest(db *sql.DB) error {
	x := rand.Intn(1024)
	var ret, cret int
	err := insertDataAndQueryBack(db, "int_test", x, &ret, &cret)
	if err != nil {
		return err
	}
	if ret != x {
		return fmt.Errorf("int test failed %d != %d", ret, x)
	}

	if ret != cret {
		return fmt.Errorf("int test cache failed %d != %d", ret, cret)
	}
	return nil
}

func doubleTest(db *sql.DB) error {
	x := rand.Float64()
	var ret, cret float64

	err := insertDataAndQueryBack(db, "double_test", x, &ret, &cret)
	if err != nil {
		return err
	}

	if math.Abs(ret-x) > 1e-7 {
		return fmt.Errorf("double test failed %v != %v", ret, x)
	}

	if math.Abs(ret-cret) > 1e-7 {
		return fmt.Errorf("double test cache failed %v != %v", ret, cret)
	}

	return nil
}

func emojiTest(db *sql.DB) error {
	s := emoji.Sprint("I like a :pizza: and :sushi:!!")
	var ret, cret []byte
	err := insertDataAndQueryBack(db, "emoji_test", s, &ret, &cret)
	if err != nil {
		return err
	}

	if len(ret) != len(s) {
		return fmt.Errorf("emoji test failed %v != %v", ret, []byte(s))
	}

	if len(ret) != len(cret) {
		return fmt.Errorf("emoji cache test failed %v != %v", ret, cret)
	}

	return nil

}

func varcharTest(db *sql.DB) error {
	s := randSeq(1024)
	var ret, cret string
	err := insertDataAndQueryBack(db, "varchar_test", s, &ret, &cret)
	if err != nil {
		return err
	}

	if ret != s {
		return fmt.Errorf("var char test failed %v != %v", ret, s)
	}

	if ret != cret {
		return fmt.Errorf("var char cache test failed %v != %v", ret, cret)
	}

	return nil
}

func textTest(db *sql.DB) error {
	s := randSeq(4096)
	var ret, cret string
	err := insertDataAndQueryBack(db, "text_test", s, &ret, &cret)
	if err != nil {
		return err
	}

	if ret != s {
		return fmt.Errorf("text test failed %v != %v", ret, s)
	}

	if ret != cret {
		return fmt.Errorf("text test cache failed %v != %v", ret, cret)
	}
	return nil
}

func blobTest(db *sql.DB) error {
	blob := []byte{
		0xff, 0x00, 0x01, 0x02, 0x03,
	}
	var ret, cret []byte
	err := insertDataAndQueryBack(db, "blob_test", blob, &ret, &cret)
	if err != nil {
		return err
	}

	s1 := fmt.Sprintf("%x", ret)
	s2 := fmt.Sprintf("%x", blob)
	s3 := fmt.Sprintf("%x", cret)
	if s1 != s2 {
		return fmt.Errorf("blob test failed %v != %v", s1, s2)
	}

	if s1 != s3 {
		return fmt.Errorf("blob test cache failed %v != %v", s1, s3)
	}

	return nil
}

func datetimeTest(db *sql.DB) error {
	d := time.Now()
	ds := d.Format("2006-01-02 15:04:05")
	var ret, cret []byte
	err := insertDataAndQueryBack(db, "datetime_test", d, &ret, &cret)
	if err != nil {
		return err
	}
	if err != nil {
		return err
	}
	if string(ret) != ds {
		return fmt.Errorf("datetime test failed %v != %v", string(ret), ds)
	}

	if string(cret) != string(ret) {
		return fmt.Errorf("datetime test cache failed %v != %v", string(ret), string(cret))
	}

	return nil
}
func dateTest(db *sql.DB) error {
	d := time.Now()
	ds := d.Format("2006-01-02")
	var ret, cret []byte
	err := insertDataAndQueryBack(db, "date_test", d, &ret, &cret)
	if err != nil {
		return err
	}
	if err != nil {
		return err
	}

	if string(ret) != ds {
		return fmt.Errorf("date test failed %v != %v", string(ret), ds)
	}

	if string(ret) != string(cret) {
		return fmt.Errorf("date test cache failed %v != %v", string(ret), string(cret))
	}

	return nil
}
<commit_msg>fix bug<commit_after>package main

import (
	"database/sql"
	"fmt"
	"log"
	"math"
	"math/rand"
	"time"

	_ "github.com/c4pt0r/mysql"
	"github.com/kyokomi/emoji"
)

func createTypeTestTbls(db *sql.DB) {
	if b, err := isTblExists(db, "int_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE int_test(id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(id), data INT)`)
	} else if err != nil {
		log.Fatal(err)
	}

	if b, err := isTblExists(db, "double_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE double_test(id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(id), data DOUBLE)`)
	} else if err != nil {
		log.Fatal(err)
	}

	if b, err := isTblExists(db, "varchar_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE varchar_test(id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(id), data VARCHAR(1024))`)
	} else if err != nil {
		log.Fatal(err)
	}

	if b, err := isTblExists(db, "text_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE text_test(id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(id), data TEXT)`)
	} else if err != nil {
		log.Fatal(err)
	}

	if b, err := isTblExists(db, "blob_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE blob_test(id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(id), data BLOB)`)
	} else if err != nil {
		log.Fatal(err)
	}

	if b, err := isTblExists(db, "datetime_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE datetime_test(id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(id), data DATETIME)`)
	} else if err != nil {
		log.Fatal(err)
	}

	if b, err := isTblExists(db, "date_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE date_test(id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(id), data DATE)`)
	} else if err != nil {
		log.Fatal(err)
	}

	if b, err := isTblExists(db, "multi_primary_key_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE multi_primary_key_test(id1 INT, id2 INT, PRIMARY KEY(id1, id2), data INT)`)
	} else if err != nil {
		log.Fatal(err)
	}

	if b, err := isTblExists(db, "emoji_test"); !b && err == nil {
		mustExec(db, `CREATE TABLE emoji_test(id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(id), data VARCHAR(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci)`)
	} else if err != nil {
		log.Fatal(err)
	}

}

func dropTypeTestTbls(db *sql.DB) {
	tbls := []string{
		"int_test",
		"double_test",
		"varchar_test",
		"text_test",
		"blob_test",
		"datetime_test",
		"date_test",
		"emoji_test",
	}

	for _, t := range tbls {
		if b, _ := isTblExists(db, t); b {
			mustExec(db, "DROP TABLE "+t)
		}
	}
}

func insertDataAndQueryBack(db *sql.DB, tblName string, val interface{}, ret interface{}, cachedRet interface{}) error {
	r := mustExec(db, "INSERT INTO "+tblName+"(data) VALUES(?)", val)
	id, err := r.LastInsertId()
	if err != nil {
		return err
	}

	res, err := db.Query("SELECT data FROM "+tblName+" WHERE id = ?", id)
	if err != nil {
		return err
	}
	defer res.Close()

	for res.Next() {
		res.Scan(ret)
		if err != nil {
			return err
		}
	}

	cachedRes, err := db.Query("SELECT data FROM "+tblName+" WHERE id = ?", id)
	if err != nil {
		return err
	}
	defer cachedRes.Close()

	for cachedRes.Next() {
		cachedRes.Scan(cachedRet)
		if err != nil {
			return err
		}
	}
	return nil
}

func multiPKeyTest(db *sql.DB) error {
	x := rand.Intn(1024)
	id1 := rand.Intn(1024)
	id2 := rand.Intn(1024)
	tblName := "multi_primary_key_test"

	mustExec(db, "INSERT INTO multi_primary_key_test(id1, id2, data) VALUES(?, ?, ?)", id1, id2, x)

	res, err := db.Query("SELECT data FROM "+tblName+" WHERE id1 = ? and id2 = ?", id1, id2)
	if err != nil {
		return err
	}
	defer res.Close()

	for res.Next() {
		var ret int
		res.Scan(&ret)
		if err != nil {
			return err
		}
		if ret != x {
			return fmt.Errorf("multi pkey test failed %d != %d", x, ret)
		}
	}

	cachedRes, err := db.Query("SELECT data FROM "+tblName+" WHERE id1 = ? and id2 = ?", id1, id2)
	if err != nil {
		return err
	}
	defer cachedRes.Close()

	for cachedRes.Next() {
		var ret int
		cachedRes.Scan(&ret)
		if err != nil {
			return err
		}
		if ret != x {
			return fmt.Errorf("multi pkey test cache failed %d != %d", ret, x)
		}
	}
	return nil
}

func intTest(db *sql.DB) error {
	x := rand.Intn(1024)
	var ret, cret int
	err := insertDataAndQueryBack(db, "int_test", x, &ret, &cret)
	if err != nil {
		return err
	}
	if ret != x {
		return fmt.Errorf("int test failed %d != %d", ret, x)
	}

	if ret != cret {
		return fmt.Errorf("int test cache failed %d != %d", ret, cret)
	}
	return nil
}

func doubleTest(db *sql.DB) error {
	x := rand.Float64()
	var ret, cret float64

	err := insertDataAndQueryBack(db, "double_test", x, &ret, &cret)
	if err != nil {
		return err
	}

	if math.Abs(ret-x) > 1e-7 {
		return fmt.Errorf("double test failed %v != %v", ret, x)
	}

	if math.Abs(ret-cret) > 1e-7 {
		return fmt.Errorf("double test cache failed %v != %v", ret, cret)
	}

	return nil
}

func emojiTest(db *sql.DB) error {
	s := emoji.Sprint("I like a :pizza: and :sushi:!!")
	var ret, cret []byte
	err := insertDataAndQueryBack(db, "emoji_test", s, &ret, &cret)
	if err != nil {
		return err
	}

	if len(ret) != len(s) {
		return fmt.Errorf("emoji test failed %v != %v", ret, []byte(s))
	}

	if len(ret) != len(cret) {
		return fmt.Errorf("emoji cache test failed %v != %v", ret, cret)
	}

	return nil

}

func varcharTest(db *sql.DB) error {
	s := randSeq(1024)
	var ret, cret string
	err := insertDataAndQueryBack(db, "varchar_test", s, &ret, &cret)
	if err != nil {
		return err
	}

	if ret != s {
		return fmt.Errorf("var char test failed %v != %v", ret, s)
	}

	if ret != cret {
		return fmt.Errorf("var char cache test failed %v != %v", ret, cret)
	}

	return nil
}

func textTest(db *sql.DB) error {
	s := randSeq(4096)
	var ret, cret string
	err := insertDataAndQueryBack(db, "text_test", s, &ret, &cret)
	if err != nil {
		return err
	}

	if ret != s {
		return fmt.Errorf("text test failed %v != %v", ret, s)
	}

	if ret != cret {
		return fmt.Errorf("text test cache failed %v != %v", ret, cret)
	}
	return nil
}

func blobTest(db *sql.DB) error {
	blob := []byte{
		0xff, 0x00, 0x01, 0x02, 0x03,
	}
	var ret, cret []byte
	err := insertDataAndQueryBack(db, "blob_test", blob, &ret, &cret)
	if err != nil {
		return err
	}

	s1 := fmt.Sprintf("%x", ret)
	s2 := fmt.Sprintf("%x", blob)
	s3 := fmt.Sprintf("%x", cret)
	if s1 != s2 {
		return fmt.Errorf("blob test failed %v != %v", s1, s2)
	}

	if s1 != s3 {
		return fmt.Errorf("blob test cache failed %v != %v", s1, s3)
	}

	return nil
}

func datetimeTest(db *sql.DB) error {
	d := time.Now()
	ds := d.Format("2006-01-02 15:04:05")
	var ret, cret []byte
	err := insertDataAndQueryBack(db, "datetime_test", d, &ret, &cret)
	if err != nil {
		return err
	}
	if err != nil {
		return err
	}
	if string(ret) != ds {
		return fmt.Errorf("datetime test failed %v != %v", string(ret), ds)
	}

	if string(cret) != string(ret) {
		return fmt.Errorf("datetime test cache failed %v != %v", string(ret), string(cret))
	}

	return nil
}
func dateTest(db *sql.DB) error {
	d := time.Now()
	ds := d.Format("2006-01-02")
	var ret, cret []byte
	err := insertDataAndQueryBack(db, "date_test", d, &ret, &cret)
	if err != nil {
		return err
	}
	if err != nil {
		return err
	}

	if string(ret) != ds {
		return fmt.Errorf("date test failed %v != %v", string(ret), ds)
	}

	if string(ret) != string(cret) {
		return fmt.Errorf("date test cache failed %v != %v", string(ret), string(cret))
	}

	return nil
}
<|endoftext|>


<commit_before>package colorutil

import (
	"github.com/lucasb-eyer/go-colorful"
	"google.golang.org/api/slides/v1"
)

// ChartColor1 is the color palette for Google Charts as collected by
// Craig Davis here: https://gist.github.com/there4/2579834
var ChartColor1 = [...]string{
	"#3366CC",
	"#DC3912",
	"#FF9900",
	"#109618",
	"#990099",
	"#3B3EAC",
	"#0099C6",
	"#DD4477",
	"#66AA00",
	"#B82E2E",
	"#316395",
	"#994499",
	"#22AA99",
	"#AAAA11",
	"#6633CC",
	"#E67300",
	"#8B0707",
	"#329262",
	"#5574A6",
	"#3B3EAC",
}

func GoogleSlidesRgbColorParseHex(hexColor string) (slides.RgbColor, error) {
	rgbColor := slides.RgbColor{}
	c, err := colorful.Hex(hexColor)
	if err != nil {
		return rgbColor, err
	}
	return slides.RgbColor{Red: c.R, Green: c.G, Blue: c.B}, nil
}
<commit_msg>update GoogleSlidesRgbColorParseHex<commit_after>package colorutil

import (
	"github.com/lucasb-eyer/go-colorful"
	"google.golang.org/api/slides/v1"
)

// ChartColor1 is the color palette for Google Charts as collected by
// Craig Davis here: https://gist.github.com/there4/2579834
var ChartColor1 = [...]string{
	"#3366CC",
	"#DC3912",
	"#FF9900",
	"#109618",
	"#990099",
	"#3B3EAC",
	"#0099C6",
	"#DD4477",
	"#66AA00",
	"#B82E2E",
	"#316395",
	"#994499",
	"#22AA99",
	"#AAAA11",
	"#6633CC",
	"#E67300",
	"#8B0707",
	"#329262",
	"#5574A6",
	"#3B3EAC",
}

func GoogleSlidesRgbColorParseHex(hexColor string) (*slides.RgbColor, error) {
	c, err := colorful.Hex(hexColor)
	if err != nil {
		return nil, err
	}
	return &slides.RgbColor{Red: c.R, Green: c.G, Blue: c.B}, nil
}
<|endoftext|>


<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package redis

import (
	"time"
	"strconv"
	"net"
	"encoding/json"
	"fmt"

	"github.com/golang/glog"

	"github.com/att-innovate/charmander-scheduler/manager"
)

func InitRedisUpdater(manager manager.Manager) {
	go updateRedis(manager)
}

func updateRedis(manager manager.Manager) {
	for {
		glog.Infoln("update redis")
		if connection := redisAvailable(manager); connection != nil {
			for _, node := range manager.GetNodes() {
				nodeInJSON, _ := json.Marshal(&node)
				key := "charmander:nodes:"+node.Hostname
				sendCommand(connection, "SET", key, fmt.Sprintf("%s", nodeInJSON))
				sendCommand(connection, "EXPIRE", key, "30") //timeout after 30s
			}
			for _, task := range manager.GetTasks() {
				taskInJSON, _ := json.Marshal(&task)
				key := "charmander:tasks:"+task.InternalID
				sendCommand(connection, "SET", key, fmt.Sprintf("%s", taskInJSON))
				sendCommand(connection, "EXPIRE", key, "30") //timeout after 30s
			}
			for _, task := range manager.GetTasks() {
				if task.NotMetered { continue }
				taskInJSON, _ := json.Marshal(&task)
				key := "charmander:tasks-metered:"+task.InternalID
				sendCommand(connection, "SET", key, fmt.Sprintf("%s", taskInJSON))
				sendCommand(connection, "EXPIRE", key, "30") //timeout after 30s
			}
			connection.Close()
		}

		time.Sleep(15 * time.Second)
	}
}

func redisAvailable(manager manager.Manager) net.Conn {
	connection, error := net.DialTimeout("tcp", manager.GetRedisConnectionIPAndPort(), 2 * time.Second)
	if error != nil {
		return nil
	}

	return connection
}

func sendCommand(connection net.Conn, args ...string) {
	buffer := make([]byte, 0, 0)
	buffer = encodeReq(buffer, args)
	connection.Write(buffer)
}

func encodeReq(buf []byte, args []string) []byte {
	buf = append(buf, '*')
	buf = strconv.AppendUint(buf, uint64(len(args)), 10)
	buf = append(buf, '\r', '\n')
	for _, arg := range args {
		buf = append(buf, '$')
		buf = strconv.AppendUint(buf, uint64(len(arg)), 10)
		buf = append(buf, '\r', '\n')
		buf = append(buf, []byte(arg)...)
		buf = append(buf, '\r', '\n')
	}
	return buf
}
<commit_msg>changed sequence of writing information to redis .. write metered tasks before tasks<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package redis

import (
	"time"
	"strconv"
	"net"
	"encoding/json"
	"fmt"

	"github.com/golang/glog"

	"github.com/att-innovate/charmander-scheduler/manager"
)

func InitRedisUpdater(manager manager.Manager) {
	go updateRedis(manager)
}

func updateRedis(manager manager.Manager) {
	for {
		glog.Infoln("update redis")
		if connection := redisAvailable(manager); connection != nil {
			for _, node := range manager.GetNodes() {
				nodeInJSON, _ := json.Marshal(&node)
				key := "charmander:nodes:"+node.Hostname
				sendCommand(connection, "SET", key, fmt.Sprintf("%s", nodeInJSON))
				sendCommand(connection, "EXPIRE", key, "30") //timeout after 30s
			}
			for _, task := range manager.GetTasks() {
				if task.NotMetered { continue }
				taskInJSON, _ := json.Marshal(&task)
				key := "charmander:tasks-metered:"+task.InternalID
				sendCommand(connection, "SET", key, fmt.Sprintf("%s", taskInJSON))
				sendCommand(connection, "EXPIRE", key, "30") //timeout after 30s
			}
			for _, task := range manager.GetTasks() {
				taskInJSON, _ := json.Marshal(&task)
				key := "charmander:tasks:"+task.InternalID
				sendCommand(connection, "SET", key, fmt.Sprintf("%s", taskInJSON))
				sendCommand(connection, "EXPIRE", key, "30") //timeout after 30s
			}
			connection.Close()
		}

		time.Sleep(15 * time.Second)
	}
}

func redisAvailable(manager manager.Manager) net.Conn {
	connection, error := net.DialTimeout("tcp", manager.GetRedisConnectionIPAndPort(), 2 * time.Second)
	if error != nil {
		return nil
	}

	return connection
}

func sendCommand(connection net.Conn, args ...string) {
	buffer := make([]byte, 0, 0)
	buffer = encodeReq(buffer, args)
	connection.Write(buffer)
}

func encodeReq(buf []byte, args []string) []byte {
	buf = append(buf, '*')
	buf = strconv.AppendUint(buf, uint64(len(args)), 10)
	buf = append(buf, '\r', '\n')
	for _, arg := range args {
		buf = append(buf, '$')
		buf = strconv.AppendUint(buf, uint64(len(arg)), 10)
		buf = append(buf, '\r', '\n')
		buf = append(buf, []byte(arg)...)
		buf = append(buf, '\r', '\n')
	}
	return buf
}
<|endoftext|>


<commit_before>package geoffrey

import (
	log "github.com/Sirupsen/logrus"
	"github.com/jriddick/geoffrey/bot"
	"github.com/jriddick/geoffrey/irc"
	"github.com/jriddick/geoffrey/modules/plugin"
	"github.com/yuin/gluamapper"
	"github.com/yuin/gopher-lua"
)

var (
	bots = make(map[string]*bot.Bot)
)

// Geoffrey module handles the creation, start, stop and
// cleanup of bots.
type Geoffrey struct {
	plugins *plugin.Plugin
}

type geoffreyConfig struct {
	Config  bot.Config
	Plugins []string
}

// NewGeoffrey returns a new Geoffrey module
func NewGeoffrey(plugins *plugin.Plugin) *Geoffrey {
	return &Geoffrey{
		plugins: plugins,
	}
}

// Register will register the module to the lua state
func (g *Geoffrey) Register(L *lua.LState) {
	L.RegisterModule("geoffrey", map[string]lua.LGFunction{
		"add": g.Add,
	})
}

// Shutdown will stop all running bots
func (g *Geoffrey) Shutdown() {
	for _, bot := range bots {
		bot.Close()
	}
}

// Add will create and start a new bot
func (g *Geoffrey) Add(L *lua.LState) int {
	// Get the table
	table := L.ToTable(-1)

	// The bot configuration
	var config geoffreyConfig

	// Map the configuration
	if err := gluamapper.Map(table, &config); err != nil {
		log.Fatalln(err)
	}

	// Create the bot
	bots[L.ToString(1)] = bot.NewBot(config.Config, L.NewThread())

	// Get all loaded plugins
	for _, name := range config.Plugins {
		if plugin, ok := g.plugins.Plugins[name]; !ok {
			log.Errorf("Tried to use non-existent plugin '%s'", name)
			continue
		} else {
			for event, handler := range plugin.Bind {
				switch event {
				case "OnMessage":
					event = irc.Message
				case "OnPing":
					event = irc.Ping
				case "OnWelcome":
					event = irc.Welcome
				case "OnJoin":
					event = irc.Join
				case "OnPart":
					event = irc.Part
				case "OnNameList":
					event = irc.Namreply
				case "OnNotice":
					event = irc.Notice
				}

				bots[L.ToString(1)].AddLuaHandler(event, handler)
			}
		}
	}

	// Connect
	bots[L.ToString(1)].Connect()

	// Run the handler
	go bots[L.ToString(1)].Handler()

	return 0
}
<commit_msg>don't call lua all the time in geoffrey.go<commit_after>package geoffrey

import (
	log "github.com/Sirupsen/logrus"
	"github.com/jriddick/geoffrey/bot"
	"github.com/jriddick/geoffrey/irc"
	"github.com/jriddick/geoffrey/modules/plugin"
	"github.com/yuin/gluamapper"
	"github.com/yuin/gopher-lua"
)

var (
	bots = make(map[string]*bot.Bot)
)

// Geoffrey module handles the creation, start, stop and
// cleanup of bots.
type Geoffrey struct {
	plugins *plugin.Plugin
}

type geoffreyConfig struct {
	Config  bot.Config
	Plugins []string
}

// NewGeoffrey returns a new Geoffrey module
func NewGeoffrey(plugins *plugin.Plugin) *Geoffrey {
	return &Geoffrey{
		plugins: plugins,
	}
}

// Register will register the module to the lua state
func (g *Geoffrey) Register(L *lua.LState) {
	L.RegisterModule("geoffrey", map[string]lua.LGFunction{
		"add": g.Add,
	})
}

// Shutdown will stop all running bots
func (g *Geoffrey) Shutdown() {
	for _, bot := range bots {
		bot.Close()
	}
}

// Add will create and start a new bot
func (g *Geoffrey) Add(L *lua.LState) int {
	// Get the table
	table := L.ToTable(-1)

	// Get the name
	name := L.ToString(1)

	// The bot configuration
	var config geoffreyConfig

	// Map the configuration
	if err := gluamapper.Map(table, &config); err != nil {
		log.Fatalln(err)
	}

	// Create the bot
	bots[name] = bot.NewBot(config.Config, L.NewThread())

	// Get all loaded plugins
	for _, name := range config.Plugins {
		if plugin, ok := g.plugins.Plugins[name]; !ok {
			log.Errorf("Tried to use non-existent plugin '%s'", name)
			continue
		} else {
			for event, handler := range plugin.Bind {
				switch event {
				case "OnMessage":
					event = irc.Message
				case "OnPing":
					event = irc.Ping
				case "OnWelcome":
					event = irc.Welcome
				case "OnJoin":
					event = irc.Join
				case "OnPart":
					event = irc.Part
				case "OnNameList":
					event = irc.Namreply
				case "OnNotice":
					event = irc.Notice
				}

				bots[name].AddLuaHandler(event, handler)
			}
		}
	}

	// Connect
	bots[name].Connect()

	// Run the handler
	go bots[name].Handler()

	return 0
}
<|endoftext|>


<commit_before>package backend

import (
	"crypto/sha1"
	"errors"
	"io"
	"time"

	btree "github.com/boltdb/bolt"
	"github.com/golang/snappy"
)

// Backend represents a persistent AzmoDB backend.
type Backend interface {
	// Range performs fn on all values stored in the database at rev.
	// If fn returns an errors the traversal is stopped and the error
	// is returned.
	Range(rev Revision, fn func(key []byte, value []byte) error) error

	// Batch starts a new batch transaction. Starting multiple write
	// batch transactions will cause the calls to block and be
	// serialized until the current write batch transaction finishes.
	Batch(rev Revision) (Batch, error)

	// Last returns the last revision in the database and an error if
	// any.
	Last() (Revision, error)
}

// Batch returns a batch transaction on the database.
type Batch interface {
	// Put sets the value for a key in the database. Put must create a
	// copy of the supplied key and value.
	Put(key []byte, value []byte) error
	Close() error
}

// Revision represents a serialized AzmoDB revision.
type Revision [8]byte

var (
	rootBuckets = [][]byte{dataBucket, metaBucket}
	dataBucket  = []byte("__data__")
	metaBucket  = []byte("__meta__")

	_ Backend = (*DB)(nil)
)

// Option represents a DB option function.
type Option func(*DB) error

// WithCompression configures the DB to use the Snappy compression on all supplied
// values.
func WithCompression() Option {
	return func(db *DB) error {
		db.compress = true
		return nil
	}
}

// WithMaxBatchEntries configures the maximum batch entries.
func WithMaxBatchEntries(entries int) Option {
	return func(db *DB) error {
		db.maxEntries = entries
		return nil
	}
}

// WithMaxBatchSize configures the maximum batch size.
func WithMaxBatchSize(size int) Option {
	return func(db *DB) error {
		db.maxSize = size
		return nil
	}
}

// DB represents the default AzmoDB backend. DB represents a collection of buckets
// persisted to a file on disk. All data access is performed through transactions
// which can be obtained through the DB.
type DB struct {
	root       *btree.DB
	maxEntries int
	maxSize    int
	compress   bool
}

const (
	defaultMaxBatchSize    = 2 << 20
	defaultMaxBatchEntries = 256
)

// Open creates and opens a database at the given path. If the file does
// not exist then it will be created automatically.
//
// Timeout is the amount of time to wait to obtain a file lock. When set
// to zero it will wait indefinitely. This option is only available on
// Darwin and Linux.
func Open(path string, timeout time.Duration, opts ...Option) (*DB, error) {
	db := &DB{
		maxEntries: defaultMaxBatchEntries,
		maxSize:    defaultMaxBatchSize,
		compress:   false,
	}
	for _, opt := range opts {
		if err := opt(db); err != nil {
			return nil, err
		}
	}

	root, err := btree.Open(path, 0600, &btree.Options{
		Timeout: timeout,
	})
	if err != nil {
		return nil, err
	}

	if err = root.Update(func(tx *btree.Tx) (err error) {
		for _, name := range rootBuckets {
			_, err = tx.CreateBucketIfNotExists(name)
			if err != nil {
				break
			}
		}
		return err
	}); err != nil {
		return nil, err
	}

	//db.root = root
	return &DB{root: root}, nil
}

// Close releases all database resources. All batch transactions must be
// closed before closing the database.
func (db *DB) Close() error {
	if db == nil || db.root == nil {
		return errors.New("backend is shut down")
	}

	err := db.root.Close()
	db.root = nil
	return err
}

// WriteTo writes the entire database to a writer.
func (db *DB) WriteTo(w io.Writer) (n int64, err error) {
	err = db.root.View(func(tx *btree.Tx) error {
		n, err = tx.WriteTo(w)
		return err
	})
	return n, err
}

// Range performs fn on all values stored in the database at rev. If fn
// returns an errors the traversal is stopped and the error is returned.
func (db *DB) Range(rev Revision, fn func(key, value []byte) error) error {
	return db.root.View(func(tx *btree.Tx) (err error) {
		meta := tx.Bucket(metaBucket).Bucket(rev[:])
		if meta == nil {
			return errors.New("revision not found")
		}
		data := tx.Bucket(dataBucket)

		c := meta.Cursor()
		for k, sum := c.First(); k != nil; k, sum = c.Next() {
			v := data.Get(sum)
			if v == nil {
				panic("cannot find value for key: " + string(k))
			}
			if db.compress {
				uk, err := snappy.Decode(nil, k)
				if err != nil {
					return err
				}
				uv, err := snappy.Decode(nil, v)
				if err != nil {
					return err
				}
				err = fn(uk, uv)
			} else {
				err = fn(clone(nil, k), clone(nil, v))
			}
			if err != nil {
				break
			}
		}
		return err
	})
}

// Last returns the last revision in the database and an error if any.
func (db *DB) Last() (rev Revision, err error) {
	err = db.root.View(func(tx *btree.Tx) error {
		c := tx.Bucket(metaBucket).Cursor()
		k, _ := c.Last()
		copy(rev[:], k)
		return nil
	})
	return rev, err
}

// Batch starts a new batch transaction. Starting multiple write batch
// transactions will cause the calls to block and be serialized until
// the current write batch transaction finishes.
func (db *DB) Batch(rev Revision) (Batch, error) {
	tx, err := db.root.Begin(true)
	if err != nil {
		return nil, err
	}

	meta, data := tx.Bucket(metaBucket), tx.Bucket(dataBucket)
	meta, err = meta.CreateBucket(rev[:])
	if err != nil {
		return nil, err
	}
	return &batch{
		entries:    make([]*entry, 12),
		maxEntries: db.maxEntries,
		maxSize:    db.maxSize,
		compress:   db.compress,

		meta: meta,
		data: data,
		tx:   tx,
	}, nil
}

func sha1sum(data []byte) [sha1.Size]byte {
	b := [sha1.Size]byte{}
	h := sha1.New()
	h.Write(data)
	h.Sum(b[:0])
	return b
}

type entry struct {
	key   []byte
	value []byte
}

type batch struct {
	entries    []*entry
	index      int
	size       int
	maxEntries int
	maxSize    int
	compress   bool

	meta *btree.Bucket
	data *btree.Bucket
	tx   *btree.Tx
}

func (b *batch) next() *entry {
	e := b.entries[b.index]
	if e == nil {
		e = &entry{}
		b.entries[b.index] = e
	}
	return e
}

func (b *batch) Put(key, value []byte) error {
	e := b.next()
	if b.compress {
		e.key = snappy.Encode(nil, e.key)
		e.value = snappy.Encode(nil, e.value)
	} else {
		e.key = clone(nil, key)
		e.value = clone(nil, value)
	}
	b.size += len(key) + len(value)
	b.index++

	return b.flush(false)
}

func (b *batch) put(key, value []byte) (err error) {
	sum := sha1sum(value)
	if v := b.data.Get(sum[:]); v == nil {
		err = b.data.Put(sum[:], value)
		if err != nil {
			return err
		}
	}
	return b.meta.Put(key, sum[:])
}

func (b *batch) flush(force bool) (err error) {
	if b.index >= b.maxEntries || b.size >= b.maxSize || force {
		for i := 0; i < b.index; i++ {
			e := b.entries[i]
			if err = b.put(e.key, e.value); err != nil {
				return err
			}
			e.key = nil
			e.value = nil
		}
		b.index = 0
		b.size = 0
	}
	return err
}

func (b *batch) Close() error {
	if err := b.flush(true); err != nil {
		return err
	}
	return b.tx.Commit()
}

func clone(dst, src []byte) []byte {
	n := len(src)
	if len(dst) < n {
		dst = make([]byte, n)
	}
	dst = dst[:n]
	copy(dst, src)
	return dst
}
<commit_msg>add backend package doc<commit_after>// Package backend implements the persistent AzmoDB backend key/value
// database.
package backend

import (
	"crypto/sha1"
	"errors"
	"io"
	"time"

	btree "github.com/boltdb/bolt"
	"github.com/golang/snappy"
)

// Backend represents a persistent AzmoDB backend.
type Backend interface {
	// Range performs fn on all values stored in the database at rev.
	// If fn returns an errors the traversal is stopped and the error
	// is returned.
	Range(rev Revision, fn func(key []byte, value []byte) error) error

	// Batch starts a new batch transaction. Starting multiple write
	// batch transactions will cause the calls to block and be
	// serialized until the current write batch transaction finishes.
	Batch(rev Revision) (Batch, error)

	// Last returns the last revision in the database and an error if
	// any.
	Last() (Revision, error)
}

// Batch returns a batch transaction on the database.
type Batch interface {
	// Put sets the value for a key in the database. Put must create a
	// copy of the supplied key and value.
	Put(key []byte, value []byte) error
	Close() error
}

// Revision represents a serialized AzmoDB revision.
type Revision [8]byte

var (
	rootBuckets = [][]byte{dataBucket, metaBucket}
	dataBucket  = []byte("__data__")
	metaBucket  = []byte("__meta__")

	_ Backend = (*DB)(nil)
)

// Option represents a DB option function.
type Option func(*DB) error

// WithCompression configures the DB to use the Snappy compression on all supplied
// values.
func WithCompression() Option {
	return func(db *DB) error {
		db.compress = true
		return nil
	}
}

// WithMaxBatchEntries configures the maximum batch entries.
func WithMaxBatchEntries(entries int) Option {
	return func(db *DB) error {
		db.maxEntries = entries
		return nil
	}
}

// WithMaxBatchSize configures the maximum batch size.
func WithMaxBatchSize(size int) Option {
	return func(db *DB) error {
		db.maxSize = size
		return nil
	}
}

// DB represents the default AzmoDB backend. DB represents a collection of buckets
// persisted to a file on disk. All data access is performed through transactions
// which can be obtained through the DB.
type DB struct {
	root       *btree.DB
	maxEntries int
	maxSize    int
	compress   bool
}

const (
	defaultMaxBatchSize    = 2 << 20
	defaultMaxBatchEntries = 256
)

// Open creates and opens a database at the given path. If the file does
// not exist then it will be created automatically.
//
// Timeout is the amount of time to wait to obtain a file lock. When set
// to zero it will wait indefinitely. This option is only available on
// Darwin and Linux.
func Open(path string, timeout time.Duration, opts ...Option) (*DB, error) {
	db := &DB{
		maxEntries: defaultMaxBatchEntries,
		maxSize:    defaultMaxBatchSize,
		compress:   false,
	}
	for _, opt := range opts {
		if err := opt(db); err != nil {
			return nil, err
		}
	}

	root, err := btree.Open(path, 0600, &btree.Options{
		Timeout: timeout,
	})
	if err != nil {
		return nil, err
	}

	if err = root.Update(func(tx *btree.Tx) (err error) {
		for _, name := range rootBuckets {
			_, err = tx.CreateBucketIfNotExists(name)
			if err != nil {
				break
			}
		}
		return err
	}); err != nil {
		return nil, err
	}

	//db.root = root
	return &DB{root: root}, nil
}

// Close releases all database resources. All batch transactions must be
// closed before closing the database.
func (db *DB) Close() error {
	if db == nil || db.root == nil {
		return errors.New("backend is shut down")
	}

	err := db.root.Close()
	db.root = nil
	return err
}

// WriteTo writes the entire database to a writer.
func (db *DB) WriteTo(w io.Writer) (n int64, err error) {
	err = db.root.View(func(tx *btree.Tx) error {
		n, err = tx.WriteTo(w)
		return err
	})
	return n, err
}

// Range performs fn on all values stored in the database at rev. If fn
// returns an errors the traversal is stopped and the error is returned.
func (db *DB) Range(rev Revision, fn func(key, value []byte) error) error {
	return db.root.View(func(tx *btree.Tx) (err error) {
		meta := tx.Bucket(metaBucket).Bucket(rev[:])
		if meta == nil {
			return errors.New("revision not found")
		}
		data := tx.Bucket(dataBucket)

		c := meta.Cursor()
		for k, sum := c.First(); k != nil; k, sum = c.Next() {
			v := data.Get(sum)
			if v == nil {
				panic("cannot find value for key: " + string(k))
			}
			if db.compress {
				uk, err := snappy.Decode(nil, k)
				if err != nil {
					return err
				}
				uv, err := snappy.Decode(nil, v)
				if err != nil {
					return err
				}
				err = fn(uk, uv)
			} else {
				err = fn(clone(nil, k), clone(nil, v))
			}
			if err != nil {
				break
			}
		}
		return err
	})
}

// Last returns the last revision in the database and an error if any.
func (db *DB) Last() (rev Revision, err error) {
	err = db.root.View(func(tx *btree.Tx) error {
		c := tx.Bucket(metaBucket).Cursor()
		k, _ := c.Last()
		copy(rev[:], k)
		return nil
	})
	return rev, err
}

// Batch starts a new batch transaction. Starting multiple write batch
// transactions will cause the calls to block and be serialized until
// the current write batch transaction finishes.
func (db *DB) Batch(rev Revision) (Batch, error) {
	tx, err := db.root.Begin(true)
	if err != nil {
		return nil, err
	}

	meta, data := tx.Bucket(metaBucket), tx.Bucket(dataBucket)
	meta, err = meta.CreateBucket(rev[:])
	if err != nil {
		return nil, err
	}
	return &batch{
		entries:    make([]*entry, 12),
		maxEntries: db.maxEntries,
		maxSize:    db.maxSize,
		compress:   db.compress,

		meta: meta,
		data: data,
		tx:   tx,
	}, nil
}

func sha1sum(data []byte) [sha1.Size]byte {
	b := [sha1.Size]byte{}
	h := sha1.New()
	h.Write(data)
	h.Sum(b[:0])
	return b
}

type entry struct {
	key   []byte
	value []byte
}

type batch struct {
	entries    []*entry
	index      int
	size       int
	maxEntries int
	maxSize    int
	compress   bool

	meta *btree.Bucket
	data *btree.Bucket
	tx   *btree.Tx
}

func (b *batch) next() *entry {
	e := b.entries[b.index]
	if e == nil {
		e = &entry{}
		b.entries[b.index] = e
	}
	return e
}

func (b *batch) Put(key, value []byte) error {
	e := b.next()
	if b.compress {
		e.key = snappy.Encode(nil, e.key)
		e.value = snappy.Encode(nil, e.value)
	} else {
		e.key = clone(nil, key)
		e.value = clone(nil, value)
	}
	b.size += len(key) + len(value)
	b.index++

	return b.flush(false)
}

func (b *batch) put(key, value []byte) (err error) {
	sum := sha1sum(value)
	if v := b.data.Get(sum[:]); v == nil {
		err = b.data.Put(sum[:], value)
		if err != nil {
			return err
		}
	}
	return b.meta.Put(key, sum[:])
}

func (b *batch) flush(force bool) (err error) {
	if b.index >= b.maxEntries || b.size >= b.maxSize || force {
		for i := 0; i < b.index; i++ {
			e := b.entries[i]
			if err = b.put(e.key, e.value); err != nil {
				return err
			}
			e.key = nil
			e.value = nil
		}
		b.index = 0
		b.size = 0
	}
	return err
}

func (b *batch) Close() error {
	if err := b.flush(true); err != nil {
		return err
	}
	return b.tx.Commit()
}

func clone(dst, src []byte) []byte {
	n := len(src)
	if len(dst) < n {
		dst = make([]byte, n)
	}
	dst = dst[:n]
	copy(dst, src)
	return dst
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cloud

import (
	"bytes"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

var sentinels = []string{
	"Copyright",
	"Google",
	`Licensed under the Apache License, Version 2.0 (the "License");`,
}

func TestLicense(t *testing.T) {
	t.Parallel()
	err := filepath.Walk(".", func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if ext := filepath.Ext(path); ext != ".go" && ext != ".proto" {
			return nil
		}
		if strings.HasSuffix(path, ".pb.go") {
			// .pb.go files are generated from the proto files.
			// .proto files must have license headers.
			return nil
		}
		if path == "bigtable/cmd/cbt/cbtdoc.go" {
			// Automatically generated.
			return nil
		}

		src, err := ioutil.ReadFile(path)
		if err != nil {
			return nil
		}
		src = src[:140] // Ensure all of the sentinel values are at the top of the file.

		// Find license
		for _, sentinel := range sentinels {
			if !bytes.Contains(src, []byte(sentinel)) {
				t.Errorf("%v: license header not present. want %q", path, sentinel)
				return nil
			}
		}

		return nil
	})
	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>license_test: look at more characters<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cloud

import (
	"bytes"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

var sentinels = []string{
	"Copyright",
	"Google",
	`Licensed under the Apache License, Version 2.0 (the "License");`,
}

func TestLicense(t *testing.T) {
	t.Parallel()
	err := filepath.Walk(".", func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if ext := filepath.Ext(path); ext != ".go" && ext != ".proto" {
			return nil
		}
		if strings.HasSuffix(path, ".pb.go") {
			// .pb.go files are generated from the proto files.
			// .proto files must have license headers.
			return nil
		}
		if path == "bigtable/cmd/cbt/cbtdoc.go" {
			// Automatically generated.
			return nil
		}

		src, err := ioutil.ReadFile(path)
		if err != nil {
			return nil
		}
		src = src[:300] // Ensure all of the sentinel values are at the top of the file.

		// Find license
		for _, sentinel := range sentinels {
			if !bytes.Contains(src, []byte(sentinel)) {
				t.Errorf("%v: license header not present. want %q", path, sentinel)
				return nil
			}
		}

		return nil
	})
	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before><commit_msg>test for RegisterWithSet<commit_after><|endoftext|>


<commit_before>package brats_test

import (
	"flag"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/cloudfoundry/libbuildpack/bratshelper"
	"github.com/cloudfoundry/libbuildpack/cutlass"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

func init() {
	flag.StringVar(&cutlass.DefaultMemory, "memory", "128M", "default memory for pushed apps")
	flag.StringVar(&cutlass.DefaultDisk, "disk", "256M", "default disk for pushed apps")
	flag.Parse()
}

var _ = SynchronizedBeforeSuite(func() []byte {
	// Run once
	return bratshelper.InitBpData().Marshal()
}, func(data []byte) {
	// Run on all nodes
	bratshelper.Data.Unmarshal(data)

	cutlass.SeedRandom()
	cutlass.DefaultStdoutStderr = GinkgoWriter
})

var _ = SynchronizedAfterSuite(func() {
	// Run on all nodes
}, func() {
	// Run once
	Expect(cutlass.DeleteOrphanedRoutes()).To(Succeed())
	Expect(cutlass.DeleteBuildpack(strings.Replace(bratshelper.Data.Cached, "_buildpack", "", 1))).To(Succeed())
	Expect(cutlass.DeleteBuildpack(strings.Replace(bratshelper.Data.Uncached, "_buildpack", "", 1))).To(Succeed())
	Expect(os.Remove(bratshelper.Data.CachedFile)).To(Succeed())
})

func TestBrats(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Brats Suite")
}

func CopyBrats(_ string) *cutlass.App {
	dir, err := cutlass.CopyFixture(filepath.Join(bratshelper.Data.BpDir, "fixtures", "brats"))
	Expect(err).ToNot(HaveOccurred())
	return cutlass.New(dir)
}

func PushApp(app *cutlass.App) {
	Expect(app.Push()).To(Succeed())
	Eventually(app.InstanceStates, 20*time.Second).Should(Equal([]string{"RUNNING"}))
}
<commit_msg>Copy CF_HOME per process in brats tests [#153524941]<commit_after>package brats_test

import (
	"flag"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/cloudfoundry/libbuildpack/bratshelper"
	"github.com/cloudfoundry/libbuildpack/cutlass"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

func init() {
	flag.StringVar(&cutlass.DefaultMemory, "memory", "128M", "default memory for pushed apps")
	flag.StringVar(&cutlass.DefaultDisk, "disk", "256M", "default disk for pushed apps")
	flag.Parse()
}

var _ = SynchronizedBeforeSuite(func() []byte {
	// Run once
	return bratshelper.InitBpData().Marshal()
}, func(data []byte) {
	// Run on all nodes
	bratshelper.Data.Unmarshal(data)

	Expect(cutlass.CopyCfHome()).To(Succeed())
	cutlass.SeedRandom()
	cutlass.DefaultStdoutStderr = GinkgoWriter
})

var _ = SynchronizedAfterSuite(func() {
	// Run on all nodes
}, func() {
	// Run once
	Expect(cutlass.DeleteOrphanedRoutes()).To(Succeed())
	Expect(cutlass.DeleteBuildpack(strings.Replace(bratshelper.Data.Cached, "_buildpack", "", 1))).To(Succeed())
	Expect(cutlass.DeleteBuildpack(strings.Replace(bratshelper.Data.Uncached, "_buildpack", "", 1))).To(Succeed())
	Expect(os.Remove(bratshelper.Data.CachedFile)).To(Succeed())
})

func TestBrats(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Brats Suite")
}

func CopyBrats(_ string) *cutlass.App {
	dir, err := cutlass.CopyFixture(filepath.Join(bratshelper.Data.BpDir, "fixtures", "brats"))
	Expect(err).ToNot(HaveOccurred())
	return cutlass.New(dir)
}

func PushApp(app *cutlass.App) {
	Expect(app.Push()).To(Succeed())
	Eventually(app.InstanceStates, 20*time.Second).Should(Equal([]string{"RUNNING"}))
}
<|endoftext|>


<commit_before><commit_msg>Use logStream also for the non-logging case<commit_after><|endoftext|>