<commit_before>package main

// The git commit that was compiled. This will be filled in by the compiler.
var GitCommit string

// The main version number that is being run at the moment.
const Version = "0.3.2"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "dev"
<commit_msg>bump to 0.3.5<commit_after>package main

// The git commit that was compiled. This will be filled in by the compiler.
var GitCommit string

// The main version number that is being run at the moment.
const Version = "0.3.5"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "dev"
<|endoftext|>


<commit_before>package main

import "fmt"

func cmdVersion(args []string, config *appConfig) {
	fmt.Println("0.0.6")
}
<commit_msg>Fix runtime version<commit_after>package main

import "fmt"

func cmdVersion(args []string, config *appConfig) {
	fmt.Println("0.1.0")
}
<|endoftext|>


<commit_before>// Package vrf implements a verifiable random function using the Edwards form
// of Curve25519, SHA3 and the Elligator map.
//
//     E is Curve25519 (in Edwards coordinates), h is SHA3.
//     f is the elligator map (bytes->E) that covers half of E.
//     8 is the cofactor of E, the group order is 8*l for prime l.
//     Setup : the prover publicly commits to a public key (P : E)
//     H : names -> E
//         H(n) = f(h(n))^8
//     VRF : keys -> names -> vrfs
//         VRF_x(n) = h(n, H(n)^x))
//     Prove : keys -> names -> proofs
//         Prove_x(n) = tuple(c=h(n, g^r, H(n)^r), t=r-c*x, ii=H(n)^x)
//             where r = h(x, n) is used as a source of randomness
//     Check : E -> names -> vrfs -> proofs -> bool
//         Check(P, n, vrf, (c,t,ii)) = vrf == h(n, ii)
//                                     && c == h(n, g^t*P^c, H(n)^t*ii^c)
package vrf

import (
	"bytes"
	"crypto/rand"
	"io"

	"golang.org/x/crypto/sha3"

	"github.com/yahoo/coname/ed25519/edwards25519"
	"github.com/yahoo/coname/ed25519/extra25519"
)

const (
	PublicKeySize    = 32
	SecretKeySize    = 64
	Size             = 32
	intermediateSize = 32
	ProofSize        = 32 + 32 + intermediateSize
)

// GenerateKey creates a public/private key pair. rnd is used for randomness.
// If it is nil, `crypto/rand` is used.
func GenerateKey(rnd io.Reader) (pk []byte, sk *[SecretKeySize]byte, err error) {
	if rnd == nil {
		rnd = rand.Reader
	}
	sk = new([SecretKeySize]byte)
	_, err = io.ReadFull(rnd, sk[:32])
	if err != nil {
		return nil, nil, err
	}
	x, _ := expandSecret(sk)

	var pkP edwards25519.ExtendedGroupElement
	edwards25519.GeScalarMultBase(&pkP, x)
	var pkBytes [PublicKeySize]byte
	pkP.ToBytes(&pkBytes)

	copy(sk[32:], pkBytes[:])
	return pkBytes[:], sk, err
}

func expandSecret(sk *[SecretKeySize]byte) (x, skhr *[32]byte) {
	x, skhr = new([32]byte), new([32]byte)
	hash := sha3.NewShake256()
	hash.Write(sk[:32])
	hash.Read(x[:])
	hash.Read(skhr[:])
	x[0] &= 248
	x[31] &= 127
	x[31] |= 64
	return
}

func Compute(m []byte, sk *[SecretKeySize]byte) []byte {
	x, _ := expandSecret(sk)
	var ii edwards25519.ExtendedGroupElement
	var iiB [32]byte
	edwards25519.GeScalarMult(&ii, x, hashToCurve(m))
	ii.ToBytes(&iiB)

	hash := sha3.NewShake256()
	hash.Write(iiB[:]) // const length: Size
	hash.Write(m)
	var vrf [Size]byte
	hash.Read(vrf[:])
	return vrf[:]
}

func hashToCurve(m []byte) *edwards25519.ExtendedGroupElement {
	// H(n) = (f(h(n))^8)
	var hmb [32]byte
	sha3.ShakeSum256(hmb[:], m)
	var hm edwards25519.ExtendedGroupElement
	extra25519.HashToEdwards(&hm, &hmb)
	edwards25519.GeDouble(&hm, &hm)
	edwards25519.GeDouble(&hm, &hm)
	edwards25519.GeDouble(&hm, &hm)
	return &hm
}

// Prove returns the vrf value and a proof such that Verify(pk, m, vrf, proof)
// == true. The vrf value is the same as returned by Compute(m, sk).
func Prove(m []byte, sk *[SecretKeySize]byte) (vrf, proof []byte) {
	x, skhr := expandSecret(sk)
	var cH, rH [64]byte
	var r, c, minusC, t, grB, hrB, iiB [32]byte
	var ii, gr, hr edwards25519.ExtendedGroupElement

	hm := hashToCurve(m)
	edwards25519.GeScalarMult(&ii, x, hm)
	ii.ToBytes(&iiB)

	hash := sha3.NewShake256()
	hash.Write(skhr[:])
	hash.Write(sk[32:]) // public key, as in ed25519
	hash.Write(m)
	hash.Read(rH[:])
	hash.Reset()
	edwards25519.ScReduce(&r, &rH)

	edwards25519.GeScalarMultBase(&gr, &r)
	edwards25519.GeScalarMult(&hr, &r, hm)
	gr.ToBytes(&grB)
	hr.ToBytes(&hrB)

	hash.Write(grB[:])
	hash.Write(hrB[:])
	hash.Write(m)
	hash.Read(cH[:])
	hash.Reset()
	edwards25519.ScReduce(&c, &cH)

	edwards25519.ScNeg(&minusC, &c)
	edwards25519.ScMulAdd(&t, x, &minusC, &r)

	proof = make([]byte, ProofSize)
	copy(proof[:32], c[:])
	copy(proof[32:64], t[:])
	copy(proof[64:96], iiB[:])

	hash.Write(iiB[:]) // const length: Size
	hash.Write(m)
	vrf = make([]byte, Size)
	hash.Read(vrf[:])
	return
}

// Verify returns true iff vrf=Compute(m, sk) for the sk that corresponds to pk.
func Verify(pkBytes, m, vrfBytes, proof []byte) bool {
	if len(proof) != ProofSize || len(vrfBytes) != Size || len(pkBytes) != PublicKeySize {
		return false
	}
	var pk, c, cRef, t, vrf, iiB, ABytes, BBytes [32]byte
	copy(vrf[:], vrfBytes)
	copy(pk[:], pkBytes)
	copy(c[:32], proof[:32])
	copy(t[:32], proof[32:64])
	copy(iiB[:], proof[64:96])

	hash := sha3.NewShake256()
	hash.Write(iiB[:]) // const length
	hash.Write(m)
	var hCheck [Size]byte
	hash.Read(hCheck[:])
	if !bytes.Equal(hCheck[:], vrf[:]) {
		return false
	}
	hash.Reset()

	var P, B, ii, iic edwards25519.ExtendedGroupElement
	var A, hmtP, iicP edwards25519.ProjectiveGroupElement
	P.FromBytesBaseGroup(&pk)
	ii.FromBytesBaseGroup(&iiB)
	edwards25519.GeDoubleScalarMultVartime(&A, &c, &P, &t)
	A.ToBytes(&ABytes)

	hm := hashToCurve(m)
	edwards25519.GeDoubleScalarMultVartime(&hmtP, &t, hm, &[32]byte{})
	edwards25519.GeDoubleScalarMultVartime(&iicP, &c, &ii, &[32]byte{})
	iicP.ToExtended(&iic)
	hmtP.ToExtended(&B)
	edwards25519.GeAdd(&B, &B, &iic)
	B.ToBytes(&BBytes)

	var cH [64]byte
	hash.Write(ABytes[:]) // const length
	hash.Write(BBytes[:]) // const length
	hash.Write(m)
	hash.Read(cH[:])
	edwards25519.ScReduce(&cRef, &cH)
	return cRef == c
}
<commit_msg>VRF: reject non-basegroup points<commit_after>// Package vrf implements a verifiable random function using the Edwards form
// of Curve25519, SHA3 and the Elligator map.
//
//     E is Curve25519 (in Edwards coordinates), h is SHA3.
//     f is the elligator map (bytes->E) that covers half of E.
//     8 is the cofactor of E, the group order is 8*l for prime l.
//     Setup : the prover publicly commits to a public key (P : E)
//     H : names -> E
//         H(n) = f(h(n))^8
//     VRF : keys -> names -> vrfs
//         VRF_x(n) = h(n, H(n)^x))
//     Prove : keys -> names -> proofs
//         Prove_x(n) = tuple(c=h(n, g^r, H(n)^r), t=r-c*x, ii=H(n)^x)
//             where r = h(x, n) is used as a source of randomness
//     Check : E -> names -> vrfs -> proofs -> bool
//         Check(P, n, vrf, (c,t,ii)) = vrf == h(n, ii)
//                                     && c == h(n, g^t*P^c, H(n)^t*ii^c)
package vrf

import (
	"bytes"
	"crypto/rand"
	"io"

	"golang.org/x/crypto/sha3"

	"github.com/yahoo/coname/ed25519/edwards25519"
	"github.com/yahoo/coname/ed25519/extra25519"
)

const (
	PublicKeySize    = 32
	SecretKeySize    = 64
	Size             = 32
	intermediateSize = 32
	ProofSize        = 32 + 32 + intermediateSize
)

// GenerateKey creates a public/private key pair. rnd is used for randomness.
// If it is nil, `crypto/rand` is used.
func GenerateKey(rnd io.Reader) (pk []byte, sk *[SecretKeySize]byte, err error) {
	if rnd == nil {
		rnd = rand.Reader
	}
	sk = new([SecretKeySize]byte)
	_, err = io.ReadFull(rnd, sk[:32])
	if err != nil {
		return nil, nil, err
	}
	x, _ := expandSecret(sk)

	var pkP edwards25519.ExtendedGroupElement
	edwards25519.GeScalarMultBase(&pkP, x)
	var pkBytes [PublicKeySize]byte
	pkP.ToBytes(&pkBytes)

	copy(sk[32:], pkBytes[:])
	return pkBytes[:], sk, err
}

func expandSecret(sk *[SecretKeySize]byte) (x, skhr *[32]byte) {
	x, skhr = new([32]byte), new([32]byte)
	hash := sha3.NewShake256()
	hash.Write(sk[:32])
	hash.Read(x[:])
	hash.Read(skhr[:])
	x[0] &= 248
	x[31] &= 127
	x[31] |= 64
	return
}

func Compute(m []byte, sk *[SecretKeySize]byte) []byte {
	x, _ := expandSecret(sk)
	var ii edwards25519.ExtendedGroupElement
	var iiB [32]byte
	edwards25519.GeScalarMult(&ii, x, hashToCurve(m))
	ii.ToBytes(&iiB)

	hash := sha3.NewShake256()
	hash.Write(iiB[:]) // const length: Size
	hash.Write(m)
	var vrf [Size]byte
	hash.Read(vrf[:])
	return vrf[:]
}

func hashToCurve(m []byte) *edwards25519.ExtendedGroupElement {
	// H(n) = (f(h(n))^8)
	var hmb [32]byte
	sha3.ShakeSum256(hmb[:], m)
	var hm edwards25519.ExtendedGroupElement
	extra25519.HashToEdwards(&hm, &hmb)
	edwards25519.GeDouble(&hm, &hm)
	edwards25519.GeDouble(&hm, &hm)
	edwards25519.GeDouble(&hm, &hm)
	return &hm
}

// Prove returns the vrf value and a proof such that Verify(pk, m, vrf, proof)
// == true. The vrf value is the same as returned by Compute(m, sk).
func Prove(m []byte, sk *[SecretKeySize]byte) (vrf, proof []byte) {
	x, skhr := expandSecret(sk)
	var cH, rH [64]byte
	var r, c, minusC, t, grB, hrB, iiB [32]byte
	var ii, gr, hr edwards25519.ExtendedGroupElement

	hm := hashToCurve(m)
	edwards25519.GeScalarMult(&ii, x, hm)
	ii.ToBytes(&iiB)

	hash := sha3.NewShake256()
	hash.Write(skhr[:])
	hash.Write(sk[32:]) // public key, as in ed25519
	hash.Write(m)
	hash.Read(rH[:])
	hash.Reset()
	edwards25519.ScReduce(&r, &rH)

	edwards25519.GeScalarMultBase(&gr, &r)
	edwards25519.GeScalarMult(&hr, &r, hm)
	gr.ToBytes(&grB)
	hr.ToBytes(&hrB)

	hash.Write(grB[:])
	hash.Write(hrB[:])
	hash.Write(m)
	hash.Read(cH[:])
	hash.Reset()
	edwards25519.ScReduce(&c, &cH)

	edwards25519.ScNeg(&minusC, &c)
	edwards25519.ScMulAdd(&t, x, &minusC, &r)

	proof = make([]byte, ProofSize)
	copy(proof[:32], c[:])
	copy(proof[32:64], t[:])
	copy(proof[64:96], iiB[:])

	hash.Write(iiB[:]) // const length: Size
	hash.Write(m)
	vrf = make([]byte, Size)
	hash.Read(vrf[:])
	return
}

// Verify returns true iff vrf=Compute(m, sk) for the sk that corresponds to pk.
func Verify(pkBytes, m, vrfBytes, proof []byte) bool {
	if len(proof) != ProofSize || len(vrfBytes) != Size || len(pkBytes) != PublicKeySize {
		return false
	}
	var pk, c, cRef, t, vrf, iiB, ABytes, BBytes [32]byte
	copy(vrf[:], vrfBytes)
	copy(pk[:], pkBytes)
	copy(c[:32], proof[:32])
	copy(t[:32], proof[32:64])
	copy(iiB[:], proof[64:96])

	hash := sha3.NewShake256()
	hash.Write(iiB[:]) // const length
	hash.Write(m)
	var hCheck [Size]byte
	hash.Read(hCheck[:])
	if !bytes.Equal(hCheck[:], vrf[:]) {
		return false
	}
	hash.Reset()

	var P, B, ii, iic edwards25519.ExtendedGroupElement
	var A, hmtP, iicP edwards25519.ProjectiveGroupElement
	if !P.FromBytesBaseGroup(&pk) {
		return false
	}
	if !ii.FromBytesBaseGroup(&iiB) {
		return false
	}
	edwards25519.GeDoubleScalarMultVartime(&A, &c, &P, &t)
	A.ToBytes(&ABytes)

	hm := hashToCurve(m)
	edwards25519.GeDoubleScalarMultVartime(&hmtP, &t, hm, &[32]byte{})
	edwards25519.GeDoubleScalarMultVartime(&iicP, &c, &ii, &[32]byte{})
	iicP.ToExtended(&iic)
	hmtP.ToExtended(&B)
	edwards25519.GeAdd(&B, &B, &iic)
	B.ToBytes(&BBytes)

	var cH [64]byte
	hash.Write(ABytes[:]) // const length
	hash.Write(BBytes[:]) // const length
	hash.Write(m)
	hash.Read(cH[:])
	edwards25519.ScReduce(&cRef, &cH)
	return cRef == c
}
<|endoftext|>


<commit_before>/*
Copyright 2014 CoreOS Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package wal

import (
	"errors"
	"fmt"
	"hash/crc32"
	"io"
	"log"
	"os"
	"path"
	"sort"

	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/wal/walpb"
)

const (
	infoType int64 = iota + 1
	entryType
	stateType
	crcType
)

var (
	ErrIdMismatch  = errors.New("wal: unmatch id")
	ErrNotFound    = errors.New("wal: file is not found")
	ErrCRCMismatch = errors.New("wal: crc mismatch")
	crcTable       = crc32.MakeTable(crc32.Castagnoli)
)

// WAL is a logical repersentation of the stable storage.
// WAL is either in read mode or append mode but not both.
// A newly created WAL is in append mode, and ready for appending records.
// A just opened WAL is in read mode, and ready for reading records.
// The WAL will be ready for appending after reading out all the previous records.
type WAL struct {
	dir string // the living directory of the underlay files

	ri      int64    // index of entry to start reading
	decoder *decoder // decoder to decode records

	f       *os.File // underlay file opened for appending, sync
	seq     int64    // current sequence of the wal file
	encoder *encoder // encoder to encode records
}

// Create creates a WAL ready for appending records.
func Create(dirpath string) (*WAL, error) {
	log.Printf("path=%s wal.create", dirpath)
	if Exist(dirpath) {
		return nil, os.ErrExist
	}

	if err := os.MkdirAll(dirpath, 0700); err != nil {
		return nil, err
	}

	p := path.Join(dirpath, fmt.Sprintf("%016x-%016x.wal", 0, 0))
	f, err := os.OpenFile(p, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
	if err != nil {
		return nil, err
	}
	w := &WAL{
		dir:     dirpath,
		seq:     0,
		f:       f,
		encoder: newEncoder(f, 0),
	}
	if err := w.saveCrc(0); err != nil {
		return nil, err
	}
	return w, nil
}

// OpenFromIndex opens the WAL files containing all the entries after
// the given index.
// The returned WAL is ready to read. The WAL cannot be appended to before
// reading out all of its previous records.
func OpenFromIndex(dirpath string, index int64) (*WAL, error) {
	log.Printf("path=%s wal.load index=%d", dirpath, index)
	names, err := readDir(dirpath)
	if err != nil {
		return nil, err
	}
	names = checkWalNames(names)
	if len(names) == 0 {
		return nil, ErrNotFound
	}

	sort.Sort(sort.StringSlice(names))

	nameIndex, ok := searchIndex(names, index)
	if !ok || !isValidSeq(names[nameIndex:]) {
		return nil, ErrNotFound
	}

	// open the wal files for reading
	rcs := make([]io.ReadCloser, 0)
	for _, name := range names[nameIndex:] {
		f, err := os.Open(path.Join(dirpath, name))
		if err != nil {
			return nil, err
		}
		rcs = append(rcs, f)
	}
	rc := MultiReadCloser(rcs...)

	// open the lastest wal file for appending
	last := path.Join(dirpath, names[len(names)-1])
	f, err := os.OpenFile(last, os.O_WRONLY|os.O_APPEND, 0)
	if err != nil {
		rc.Close()
		return nil, err
	}

	// create a WAL ready for reading
	w := &WAL{
		ri:      index,
		decoder: newDecoder(rc),

		f: f,
	}
	return w, nil
}

// ReadAll reads out all records of the current WAL.
// After ReadAll, the WAL will be ready for appending new records.
func (w *WAL) ReadAll() (int64, raftpb.State, []raftpb.Entry, error) {
	var id int64
	var state raftpb.State
	var entries []raftpb.Entry

	rec := &walpb.Record{}
	decoder := w.decoder
	var err error
	for err = decoder.decode(rec); err == nil; err = decoder.decode(rec) {
		switch rec.Type {
		case entryType:
			e := mustUnmarshalEntry(rec.Data)
			if e.Index > w.ri {
				entries = append(entries[:e.Index-w.ri-1], e)
			}
		case stateType:
			state = mustUnmarshalState(rec.Data)
		case infoType:
			i := mustUnmarshalInfo(rec.Data)
			if id != 0 && id != i.Id {
				state.Reset()
				return 0, state, nil, ErrIdMismatch
			}
			id = i.Id
		case crcType:
			crc := decoder.crc.Sum32()
			// current crc of decoder must match the crc of the record.
			// do no need to match 0 crc, since the decoder is a new one at this case.
			if crc != 0 && rec.Validate(crc) != nil {
				state.Reset()
				return 0, state, nil, ErrCRCMismatch
			}
			decoder.updateCRC(rec.Crc)
		default:
			state.Reset()
			return 0, state, nil, fmt.Errorf("unexpected block type %d", rec.Type)
		}
	}
	if err != io.EOF {
		state.Reset()
		return 0, state, nil, err
	}

	// close decoder, disable reading
	w.decoder.close()
	w.ri = 0

	// create encoder (chain crc with the decoder), enable appending
	w.encoder = newEncoder(w.f, w.decoder.lastCRC())
	w.decoder = nil
	return id, state, entries, nil
}

// index should be the index of last log entry.
// Cut closes current file written and creates a new one ready to append.
func (w *WAL) Cut(index int64) error {
	log.Printf("wal.cut index=%d", index)

	// create a new wal file with name sequence + 1
	fpath := path.Join(w.dir, fmt.Sprintf("%016x-%016x.wal", w.seq+1, index))
	f, err := os.OpenFile(fpath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
	if err != nil {
		return err
	}

	w.Sync()
	w.f.Close()

	// update writer and save the previous crc
	w.f = f
	w.seq++
	prevCrc := w.encoder.crc.Sum32()
	w.encoder = newEncoder(w.f, prevCrc)
	return w.saveCrc(prevCrc)
}

func (w *WAL) Sync() error {
	if w.encoder != nil {
		if err := w.encoder.flush(); err != nil {
			return err
		}
	}
	return w.f.Sync()
}

func (w *WAL) Close() {
	log.Printf("path=%s wal.close", w.f.Name())
	if w.f != nil {
		w.Sync()
		w.f.Close()
	}
}

func (w *WAL) SaveInfo(i *raftpb.Info) error {
	log.Printf("path=%s wal.saveInfo id=%d", w.f.Name(), i.Id)
	b, err := i.Marshal()
	if err != nil {
		panic(err)
	}
	rec := &walpb.Record{Type: infoType, Data: b}
	return w.encoder.encode(rec)
}

func (w *WAL) SaveEntry(e *raftpb.Entry) error {
	b, err := e.Marshal()
	if err != nil {
		panic(err)
	}
	rec := &walpb.Record{Type: entryType, Data: b}
	return w.encoder.encode(rec)
}

func (w *WAL) SaveState(s *raftpb.State) error {
	log.Printf("path=%s wal.saveState state=\"%+v\"", w.f.Name(), s)
	b, err := s.Marshal()
	if err != nil {
		panic(err)
	}
	rec := &walpb.Record{Type: stateType, Data: b}
	return w.encoder.encode(rec)
}

func (w *WAL) Save(st raftpb.State, ents []raftpb.Entry) {
	// TODO(xiangli): no more reference operator
	w.SaveState(&st)
	for i := range ents {
		w.SaveEntry(&ents[i])
	}
	w.Sync()
}

func (w *WAL) saveCrc(prevCrc uint32) error {
	return w.encoder.encode(&walpb.Record{Type: crcType, Crc: prevCrc})
}
<commit_msg>wal: named return values for ReadAll.<commit_after>/*
Copyright 2014 CoreOS Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package wal

import (
	"errors"
	"fmt"
	"hash/crc32"
	"io"
	"log"
	"os"
	"path"
	"sort"

	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/wal/walpb"
)

const (
	infoType int64 = iota + 1
	entryType
	stateType
	crcType
)

var (
	ErrIdMismatch  = errors.New("wal: unmatch id")
	ErrNotFound    = errors.New("wal: file is not found")
	ErrCRCMismatch = errors.New("wal: crc mismatch")
	crcTable       = crc32.MakeTable(crc32.Castagnoli)
)

// WAL is a logical repersentation of the stable storage.
// WAL is either in read mode or append mode but not both.
// A newly created WAL is in append mode, and ready for appending records.
// A just opened WAL is in read mode, and ready for reading records.
// The WAL will be ready for appending after reading out all the previous records.
type WAL struct {
	dir string // the living directory of the underlay files

	ri      int64    // index of entry to start reading
	decoder *decoder // decoder to decode records

	f       *os.File // underlay file opened for appending, sync
	seq     int64    // current sequence of the wal file
	encoder *encoder // encoder to encode records
}

// Create creates a WAL ready for appending records.
func Create(dirpath string) (*WAL, error) {
	log.Printf("path=%s wal.create", dirpath)
	if Exist(dirpath) {
		return nil, os.ErrExist
	}

	if err := os.MkdirAll(dirpath, 0700); err != nil {
		return nil, err
	}

	p := path.Join(dirpath, fmt.Sprintf("%016x-%016x.wal", 0, 0))
	f, err := os.OpenFile(p, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
	if err != nil {
		return nil, err
	}
	w := &WAL{
		dir:     dirpath,
		seq:     0,
		f:       f,
		encoder: newEncoder(f, 0),
	}
	if err := w.saveCrc(0); err != nil {
		return nil, err
	}
	return w, nil
}

// OpenFromIndex opens the WAL files containing all the entries after
// the given index.
// The returned WAL is ready to read. The WAL cannot be appended to before
// reading out all of its previous records.
func OpenFromIndex(dirpath string, index int64) (*WAL, error) {
	log.Printf("path=%s wal.load index=%d", dirpath, index)
	names, err := readDir(dirpath)
	if err != nil {
		return nil, err
	}
	names = checkWalNames(names)
	if len(names) == 0 {
		return nil, ErrNotFound
	}

	sort.Sort(sort.StringSlice(names))

	nameIndex, ok := searchIndex(names, index)
	if !ok || !isValidSeq(names[nameIndex:]) {
		return nil, ErrNotFound
	}

	// open the wal files for reading
	rcs := make([]io.ReadCloser, 0)
	for _, name := range names[nameIndex:] {
		f, err := os.Open(path.Join(dirpath, name))
		if err != nil {
			return nil, err
		}
		rcs = append(rcs, f)
	}
	rc := MultiReadCloser(rcs...)

	// open the lastest wal file for appending
	last := path.Join(dirpath, names[len(names)-1])
	f, err := os.OpenFile(last, os.O_WRONLY|os.O_APPEND, 0)
	if err != nil {
		rc.Close()
		return nil, err
	}

	// create a WAL ready for reading
	w := &WAL{
		ri:      index,
		decoder: newDecoder(rc),

		f: f,
	}
	return w, nil
}

// ReadAll reads out all records of the current WAL.
// After ReadAll, the WAL will be ready for appending new records.
func (w *WAL) ReadAll() (id int64, state raftpb.State, entries []raftpb.Entry, err error) {
	rec := &walpb.Record{}
	decoder := w.decoder

	for err = decoder.decode(rec); err == nil; err = decoder.decode(rec) {
		switch rec.Type {
		case entryType:
			e := mustUnmarshalEntry(rec.Data)
			if e.Index > w.ri {
				entries = append(entries[:e.Index-w.ri-1], e)
			}
		case stateType:
			state = mustUnmarshalState(rec.Data)
		case infoType:
			i := mustUnmarshalInfo(rec.Data)
			if id != 0 && id != i.Id {
				state.Reset()
				return 0, state, nil, ErrIdMismatch
			}
			id = i.Id
		case crcType:
			crc := decoder.crc.Sum32()
			// current crc of decoder must match the crc of the record.
			// do no need to match 0 crc, since the decoder is a new one at this case.
			if crc != 0 && rec.Validate(crc) != nil {
				state.Reset()
				return 0, state, nil, ErrCRCMismatch
			}
			decoder.updateCRC(rec.Crc)
		default:
			state.Reset()
			return 0, state, nil, fmt.Errorf("unexpected block type %d", rec.Type)
		}
	}
	if err != io.EOF {
		state.Reset()
		return 0, state, nil, err
	}

	// close decoder, disable reading
	w.decoder.close()
	w.ri = 0

	// create encoder (chain crc with the decoder), enable appending
	w.encoder = newEncoder(w.f, w.decoder.lastCRC())
	w.decoder = nil
	return id, state, entries, nil
}

// index should be the index of last log entry.
// Cut closes current file written and creates a new one ready to append.
func (w *WAL) Cut(index int64) error {
	log.Printf("wal.cut index=%d", index)

	// create a new wal file with name sequence + 1
	fpath := path.Join(w.dir, fmt.Sprintf("%016x-%016x.wal", w.seq+1, index))
	f, err := os.OpenFile(fpath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
	if err != nil {
		return err
	}

	w.Sync()
	w.f.Close()

	// update writer and save the previous crc
	w.f = f
	w.seq++
	prevCrc := w.encoder.crc.Sum32()
	w.encoder = newEncoder(w.f, prevCrc)
	return w.saveCrc(prevCrc)
}

func (w *WAL) Sync() error {
	if w.encoder != nil {
		if err := w.encoder.flush(); err != nil {
			return err
		}
	}
	return w.f.Sync()
}

func (w *WAL) Close() {
	log.Printf("path=%s wal.close", w.f.Name())
	if w.f != nil {
		w.Sync()
		w.f.Close()
	}
}

func (w *WAL) SaveInfo(i *raftpb.Info) error {
	log.Printf("path=%s wal.saveInfo id=%d", w.f.Name(), i.Id)
	b, err := i.Marshal()
	if err != nil {
		panic(err)
	}
	rec := &walpb.Record{Type: infoType, Data: b}
	return w.encoder.encode(rec)
}

func (w *WAL) SaveEntry(e *raftpb.Entry) error {
	b, err := e.Marshal()
	if err != nil {
		panic(err)
	}
	rec := &walpb.Record{Type: entryType, Data: b}
	return w.encoder.encode(rec)
}

func (w *WAL) SaveState(s *raftpb.State) error {
	log.Printf("path=%s wal.saveState state=\"%+v\"", w.f.Name(), s)
	b, err := s.Marshal()
	if err != nil {
		panic(err)
	}
	rec := &walpb.Record{Type: stateType, Data: b}
	return w.encoder.encode(rec)
}

func (w *WAL) Save(st raftpb.State, ents []raftpb.Entry) {
	// TODO(xiangli): no more reference operator
	w.SaveState(&st)
	for i := range ents {
		w.SaveEntry(&ents[i])
	}
	w.Sync()
}

func (w *WAL) saveCrc(prevCrc uint32) error {
	return w.encoder.encode(&walpb.Record{Type: crcType, Crc: prevCrc})
}
<|endoftext|>


<commit_before>package revel

import (
	"gopkg.in/fsnotify.v1"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
)

// Listener is an interface for receivers of filesystem events.
type Listener interface {
	// Refresh is invoked by the watcher on relevant filesystem events.
	// If the listener returns an error, it is served to the user on the current request.
	Refresh() *Error
}

// DiscerningListener allows the receiver to selectively watch files.
type DiscerningListener interface {
	Listener
	WatchDir(info os.FileInfo) bool
	WatchFile(basename string) bool
}

// Watcher allows listeners to register to be notified of changes under a given
// directory.
type Watcher struct {
	// Parallel arrays of watcher/listener pairs.
	watchers     []*fsnotify.Watcher
	listeners    []Listener
	forceRefresh bool
	lastError    int
	notifyMutex  sync.Mutex
}

func NewWatcher() *Watcher {
	return &Watcher{
		forceRefresh: true,
		lastError:    -1,
	}
}

// Listen registers for events within the given root directories (recursively).
func (w *Watcher) Listen(listener Listener, roots ...string) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		ERROR.Fatal(err)
	}

	// Replace the unbuffered Event channel with a buffered one.
	// Otherwise multiple change events only come out one at a time, across
	// multiple page views.  (There appears no way to "pump" the events out of
	// the watcher)
	watcher.Events = make(chan fsnotify.Event, 100)
	watcher.Errors = make(chan error, 10)

	// Walk through all files / directories under the root, adding each to watcher.
	for _, p := range roots {
		// is the directory / file a symlink?
		f, err := os.Lstat(p)
		if err == nil && f.Mode()&os.ModeSymlink == os.ModeSymlink {
			realPath, err := filepath.EvalSymlinks(p)
			if err != nil {
				panic(err)
			}
			p = realPath
		}

		fi, err := os.Stat(p)
		if err != nil {
			ERROR.Println("Failed to stat watched path", p, ":", err)
			continue
		}

		// If it is a file, watch that specific file.
		if !fi.IsDir() {
			err = watcher.Add(p)
			if err != nil {
				ERROR.Println("Failed to watch", p, ":", err)
			}
			continue
		}

		var watcherWalker func(path string, info os.FileInfo, err error) error

		watcherWalker = func(path string, info os.FileInfo, err error) error {
			if err != nil {
				ERROR.Println("Error walking path:", err)
				return nil
			}

			// is it a symlinked template?
			link, err := os.Lstat(path)
			if err == nil && link.Mode()&os.ModeSymlink == os.ModeSymlink {
				TRACE.Println("Watcher symlink: ", path)
				// lookup the actual target & check for goodness
				targetPath, err := filepath.EvalSymlinks(path)
				if err != nil {
					ERROR.Println("Failed to read symlink", err)
					return err
				}
				targetInfo, err := os.Stat(targetPath)
				if err != nil {
					ERROR.Println("Failed to stat symlink target", err)
					return err
				}

				// set the template path to the target of the symlink
				path = targetPath
				info = targetInfo
				filepath.Walk(path, watcherWalker)
			}

			if info.IsDir() {
				if dl, ok := listener.(DiscerningListener); ok {
					if !dl.WatchDir(info) {
						return filepath.SkipDir
					}
				}

				err = watcher.Add(path)
				if err != nil {
					ERROR.Println("Failed to watch", path, ":", err)
				}
			}
			return nil
		}

		// Else, walk the directory tree.
		filepath.Walk(p, watcherWalker)
	}

	if w.eagerRebuildEnabled() {
		// Create goroutine to notify file changes in real time
		go w.NotifyWhenUpdated(listener, watcher)
	}

	w.watchers = append(w.watchers, watcher)
	w.listeners = append(w.listeners, listener)
}

// Wait until receiving a file change event.
// When a watcher receives an event, the watcher notifies it.
func (w *Watcher) NotifyWhenUpdated(listener Listener, watcher *fsnotify.Watcher) {
	for {
		// Do not catch default for performance.
		select {
		case ev := <-watcher.Event:
			if w.rebuildRequired(ev, listener) {
				// Seriarize listener.Refresh() calls.
				w.notifyMutex.Lock()
				listener.Refresh()
				w.notifyMutex.Unlock()
			}
		case <-watcher.Error:
			continue
		}
	}
}

// Notify causes the watcher to forward any change events to listeners.
// It returns the first (if any) error returned.
func (w *Watcher) Notify() *Error {
	// Serialize Notify() calls.
	w.notifyMutex.Lock()
	defer w.notifyMutex.Unlock()

	for i, watcher := range w.watchers {
		listener := w.listeners[i]

		// Pull all pending events / errors from the watcher.
		refresh := false
		for {
			select {
			case ev := <-watcher.Event:
				if w.rebuildRequired(ev, listener) {
					refresh = true
				}
				continue
			case <-watcher.Error:
				continue
			default:
				// No events left to pull
			}
			break
		}

		if w.forceRefresh || refresh || w.lastError == i {
			err := listener.Refresh()
			if err != nil {
				w.lastError = i
				return err
			}
		}
	}

	w.forceRefresh = false
	w.lastError = -1
	return nil
}

// If the rebuild.eager config is set true, watcher rebuilds an application
// every time a source file is changed.
// This feature is available only in dev mode.
func (w *Watcher) eagerRebuildEnabled() bool {
	return Config.BoolDefault("mode.dev", true) &&
		Config.BoolDefault("watch", true) &&
		Config.BoolDefault("rebuild.eager", false)
}

func (w *Watcher) rebuildRequired(ev *fsnotify.FileEvent, listener Listener) bool {
	// Ignore changes to dotfiles.
	if strings.HasPrefix(path.Base(ev.Name), ".") {
		return false
	}

	if dl, ok := listener.(DiscerningListener); ok {
		if !dl.WatchFile(ev.Name) || ev.IsAttrib() {
			return false
		}
	}
	return true
}

var WatchFilter = func(c *Controller, fc []Filter) {
	if MainWatcher != nil {
		err := MainWatcher.Notify()
		if err != nil {
			c.Result = c.RenderError(err)
			return
		}
	}
	fc[0](c, fc[1:])
}
<commit_msg>Fix conflict resolution<commit_after>package revel

import (
	"gopkg.in/fsnotify.v1"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
)

// Listener is an interface for receivers of filesystem events.
type Listener interface {
	// Refresh is invoked by the watcher on relevant filesystem events.
	// If the listener returns an error, it is served to the user on the current request.
	Refresh() *Error
}

// DiscerningListener allows the receiver to selectively watch files.
type DiscerningListener interface {
	Listener
	WatchDir(info os.FileInfo) bool
	WatchFile(basename string) bool
}

// Watcher allows listeners to register to be notified of changes under a given
// directory.
type Watcher struct {
	// Parallel arrays of watcher/listener pairs.
	watchers     []*fsnotify.Watcher
	listeners    []Listener
	forceRefresh bool
	lastError    int
	notifyMutex  sync.Mutex
}

func NewWatcher() *Watcher {
	return &Watcher{
		forceRefresh: true,
		lastError:    -1,
	}
}

// Listen registers for events within the given root directories (recursively).
func (w *Watcher) Listen(listener Listener, roots ...string) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		ERROR.Fatal(err)
	}

	// Replace the unbuffered Event channel with a buffered one.
	// Otherwise multiple change events only come out one at a time, across
	// multiple page views.  (There appears no way to "pump" the events out of
	// the watcher)
	watcher.Events = make(chan fsnotify.Event, 100)
	watcher.Errors = make(chan error, 10)

	// Walk through all files / directories under the root, adding each to watcher.
	for _, p := range roots {
		// is the directory / file a symlink?
		f, err := os.Lstat(p)
		if err == nil && f.Mode()&os.ModeSymlink == os.ModeSymlink {
			realPath, err := filepath.EvalSymlinks(p)
			if err != nil {
				panic(err)
			}
			p = realPath
		}

		fi, err := os.Stat(p)
		if err != nil {
			ERROR.Println("Failed to stat watched path", p, ":", err)
			continue
		}

		// If it is a file, watch that specific file.
		if !fi.IsDir() {
			err = watcher.Add(p)
			if err != nil {
				ERROR.Println("Failed to watch", p, ":", err)
			}
			continue
		}

		var watcherWalker func(path string, info os.FileInfo, err error) error

		watcherWalker = func(path string, info os.FileInfo, err error) error {
			if err != nil {
				ERROR.Println("Error walking path:", err)
				return nil
			}

			// is it a symlinked template?
			link, err := os.Lstat(path)
			if err == nil && link.Mode()&os.ModeSymlink == os.ModeSymlink {
				TRACE.Println("Watcher symlink: ", path)
				// lookup the actual target & check for goodness
				targetPath, err := filepath.EvalSymlinks(path)
				if err != nil {
					ERROR.Println("Failed to read symlink", err)
					return err
				}
				targetInfo, err := os.Stat(targetPath)
				if err != nil {
					ERROR.Println("Failed to stat symlink target", err)
					return err
				}

				// set the template path to the target of the symlink
				path = targetPath
				info = targetInfo
				filepath.Walk(path, watcherWalker)
			}

			if info.IsDir() {
				if dl, ok := listener.(DiscerningListener); ok {
					if !dl.WatchDir(info) {
						return filepath.SkipDir
					}
				}

				err = watcher.Add(path)
				if err != nil {
					ERROR.Println("Failed to watch", path, ":", err)
				}
			}
			return nil
		}

		// Else, walk the directory tree.
		filepath.Walk(p, watcherWalker)
	}

	if w.eagerRebuildEnabled() {
		// Create goroutine to notify file changes in real time
		go w.NotifyWhenUpdated(listener, watcher)
	}

	w.watchers = append(w.watchers, watcher)
	w.listeners = append(w.listeners, listener)
}

// Wait until receiving a file change event.
// When a watcher receives an event, the watcher notifies it.
func (w *Watcher) NotifyWhenUpdated(listener Listener, watcher *fsnotify.Watcher) {
	for {
		// Do not catch default for performance.
		select {
		case ev := <-watcher.Events:
			if w.rebuildRequired(ev, listener) {
				// Seriarize listener.Refresh() calls.
				w.notifyMutex.Lock()
				listener.Refresh()
				w.notifyMutex.Unlock()
			}
		case <-watcher.Errors:
			continue
		}
	}
}

// Notify causes the watcher to forward any change events to listeners.
// It returns the first (if any) error returned.
func (w *Watcher) Notify() *Error {
	// Serialize Notify() calls.
	w.notifyMutex.Lock()
	defer w.notifyMutex.Unlock()

	for i, watcher := range w.watchers {
		listener := w.listeners[i]

		// Pull all pending events / errors from the watcher.
		refresh := false
		for {
			select {
			case ev := <-watcher.Events:
				if w.rebuildRequired(ev, listener) {
					refresh = true
				}
				continue
			case <-watcher.Errors:
				continue
			default:
				// No events left to pull
			}
			break
		}

		if w.forceRefresh || refresh || w.lastError == i {
			err := listener.Refresh()
			if err != nil {
				w.lastError = i
				return err
			}
		}
	}

	w.forceRefresh = false
	w.lastError = -1
	return nil
}

// If the rebuild.eager config is set true, watcher rebuilds an application
// every time a source file is changed.
// This feature is available only in dev mode.
func (w *Watcher) eagerRebuildEnabled() bool {
	return Config.BoolDefault("mode.dev", true) &&
		Config.BoolDefault("watch", true) &&
		Config.BoolDefault("rebuild.eager", false)
}

func (w *Watcher) rebuildRequired(ev fsnotify.Event, listener Listener) bool {
	// Ignore changes to dotfiles.
	if strings.HasPrefix(path.Base(ev.Name), ".") {
		return false
	}

	if dl, ok := listener.(DiscerningListener); ok {
		if !dl.WatchFile(ev.Name) || ev.Op == fsnotify.Chmod {
			return false
		}
	}
	return true
}

var WatchFilter = func(c *Controller, fc []Filter) {
	if MainWatcher != nil {
		err := MainWatcher.Notify()
		if err != nil {
			c.Result = c.RenderError(err)
			return
		}
	}
	fc[0](c, fc[1:])
}
<|endoftext|>


<commit_before>// Package wayback implements a client for Wayback Availability JSON API.
// See its website for details: https://archive.org/help/wayback_api.php
package wayback

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"
	"time"
)

const req = `http://archive.org/wayback/available?url=%s`
const reqTimestamp = `http://archive.org/wayback/available?url=%s&timestamp=%s`

var errNotAvailable = errors.New("no snapshot is available for the given url")

type response struct {
	ArchivedSnapshots struct {
		Closest struct {
			Available bool      `json:"available"`
			URL       string    `json:"url"`
			Timestamp Timestamp `json:"timestamp"`
			Status    string
		} `json:"closest"`
	} `json:"archived_snapshots"`
}

func ndigit(i uint64) (n int) {
	for ; i != 0; i /= 10 {
		n++
	}
	return
}

// TimeLayout is Timestamp's time format for use with time.Parse and
// time.(Time).Format functions.
const TimeLayout = "20060102150405"

// Timestamp represents a timestamp value which format is YYYYMMDDhhmmss.
type Timestamp struct {
	s string
}

// NewTimestamp gives new timestamp value converted from the given time.
func NewTimestamp(t time.Time) Timestamp {
	return Timestamp{s: t.Format(TimeLayout)}
}

var layouts = []string{
	time.ANSIC,
	time.UnixDate,
	time.RubyDate,
	time.RFC822,
	time.RFC850,
	time.RFC1123,
	time.RFC3339,
}

// ParseTimestamp tries to parse the given string into a Timestamp value. If it
// is a digit, it parses it into uint64 and calls NewTimestamp on it. Otherwise
// it tries to parse a time.Time and upon success gives new Timestamp value via
// NewTime function.
//
// TODO(rjeczalik): update doc
func ParseTimestamp(layout, s string) (Timestamp, error) {
	if layout != "" {
		t, err := time.Parse(layout, s)
		if err != nil {
			return Timestamp{}, err
		}
		return NewTimestamp(t), nil
	}
	if t, err := strconv.ParseUint(s, 10, 64); err == nil {
		s = strconv.FormatUint(t, 10)
		switch n, m := len(s), len(TimeLayout); {
		case n < m:
			s = s + strings.Repeat("0", m-n)
		case n > m:
			s = s[:m]
		}
		return Timestamp{s: s}, nil
	}
	for _, l := range layouts {
		if t, err := time.Parse(l, s); err == nil {
			return NewTimestamp(t), nil
		}
	}
	return Timestamp{}, errors.New("invalid time/timestamp value: " + s)
}

// Time converts the timestamp to a time value.
func (t Timestamp) Time() time.Time {
	if t, err := time.Parse(TimeLayout, t.s); err == nil {
		return t
	}
	return time.Time{}
}

// String converts the timestamp to string. It cuts off any following zeros.
func (t Timestamp) String() string {
	i := len(t.s) - 1
	for ; i > 3; i-- {
		if t.s[i] != '0' {
			break
		}
	}
	return t.s[:i+1]
}

// UnamarshalJSON decodes quoted string into a timestamp value.
//
// TODO(rjeczalik): validate
func (t *Timestamp) UnmarshalJSON(p []byte) error {
	s, err := strconv.Unquote(string(p))
	if err != nil {
		return err
	}
	*t = Timestamp{s: s}
	return nil
}

// Client is a client for Wayback Availability JSON API.
type Client struct {
	// ClientHTTP is a HTTP client used to communicate with the Wayback Machine
	// endpoint. If it's nil, the http.DefaultClient is used.
	ClientHTTP *http.Client
}

func (c *Client) clientHTTP() *http.Client {
	if c.ClientHTTP != nil {
		return c.ClientHTTP
	}
	return http.DefaultClient
}

func (c *Client) available(url string) (s string, t time.Time, err error) {
	res, err := c.clientHTTP().Get(url)
	if err != nil {
		return
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		err = errors.New(http.StatusText(res.StatusCode))
		return
	}
	resp := &response{}
	buf := &bytes.Buffer{}
	if err = json.NewDecoder(io.TeeReader(res.Body, buf)).Decode(resp); err != nil {
		return
	}
	if !resp.ArchivedSnapshots.Closest.Available {
		err = errNotAvailable
		return
	}
	s = resp.ArchivedSnapshots.Closest.URL
	t = resp.ArchivedSnapshots.Closest.Timestamp.Time()
	return
}

// Available queries the archive for cached snapshot of website given by the url.
// If it's available, it returns its URL and creation time of the most recent
// snapshot. Otherwise it returns non-nil error.
func (c *Client) Available(url string) (string, time.Time, error) {
	return c.available(fmt.Sprintf(req, url))
}

// AvailableAt queries the archive for cached snapshot of website given by the url
// and creation time. If it's available, the function returns its URL and creation
// time which is the closest to the requested one.
func (c *Client) AvailableAt(url string, timestamp Timestamp) (string, time.Time, error) {
	return c.available(fmt.Sprintf(reqTimestamp, url, timestamp))
}
<commit_msg>Remove unused code<commit_after>// Package wayback implements a client for Wayback Availability JSON API.
// See its website for details: https://archive.org/help/wayback_api.php
package wayback

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"
	"time"
)

const req = `http://archive.org/wayback/available?url=%s`
const reqTimestamp = `http://archive.org/wayback/available?url=%s&timestamp=%s`

var errNotAvailable = errors.New("no snapshot is available for the given url")

type response struct {
	ArchivedSnapshots struct {
		Closest struct {
			Available bool      `json:"available"`
			URL       string    `json:"url"`
			Timestamp Timestamp `json:"timestamp"`
			Status    string
		} `json:"closest"`
	} `json:"archived_snapshots"`
}

// TimeLayout is Timestamp's time format for use with time.Parse and
// time.(Time).Format functions.
const TimeLayout = "20060102150405"

// Timestamp represents a timestamp value which format is YYYYMMDDhhmmss.
type Timestamp struct {
	s string
}

// NewTimestamp gives new timestamp value converted from the given time.
func NewTimestamp(t time.Time) Timestamp {
	return Timestamp{s: t.Format(TimeLayout)}
}

var layouts = []string{
	time.ANSIC,
	time.UnixDate,
	time.RubyDate,
	time.RFC822,
	time.RFC850,
	time.RFC1123,
	time.RFC3339,
}

// ParseTimestamp tries to parse the given string into a Timestamp value. If it
// is a digit, it parses it into uint64 and calls NewTimestamp on it. Otherwise
// it tries to parse a time.Time and upon success gives new Timestamp value via
// NewTime function.
//
// TODO(rjeczalik): update doc
func ParseTimestamp(layout, s string) (Timestamp, error) {
	if layout != "" {
		t, err := time.Parse(layout, s)
		if err != nil {
			return Timestamp{}, err
		}
		return NewTimestamp(t), nil
	}
	if t, err := strconv.ParseUint(s, 10, 64); err == nil {
		s = strconv.FormatUint(t, 10)
		switch n, m := len(s), len(TimeLayout); {
		case n < m:
			s = s + strings.Repeat("0", m-n)
		case n > m:
			s = s[:m]
		}
		return Timestamp{s: s}, nil
	}
	for _, l := range layouts {
		if t, err := time.Parse(l, s); err == nil {
			return NewTimestamp(t), nil
		}
	}
	return Timestamp{}, errors.New("invalid time/timestamp value: " + s)
}

// Time converts the timestamp to a time value.
func (t Timestamp) Time() time.Time {
	if t, err := time.Parse(TimeLayout, t.s); err == nil {
		return t
	}
	return time.Time{}
}

// String converts the timestamp to string. It cuts off any following zeros.
func (t Timestamp) String() string {
	i := len(t.s) - 1
	for ; i > 3; i-- {
		if t.s[i] != '0' {
			break
		}
	}
	return t.s[:i+1]
}

// UnamarshalJSON decodes quoted string into a timestamp value.
//
// TODO(rjeczalik): validate
func (t *Timestamp) UnmarshalJSON(p []byte) error {
	s, err := strconv.Unquote(string(p))
	if err != nil {
		return err
	}
	*t = Timestamp{s: s}
	return nil
}

// Client is a client for Wayback Availability JSON API.
type Client struct {
	// ClientHTTP is a HTTP client used to communicate with the Wayback Machine
	// endpoint. If it's nil, the http.DefaultClient is used.
	ClientHTTP *http.Client
}

func (c *Client) clientHTTP() *http.Client {
	if c.ClientHTTP != nil {
		return c.ClientHTTP
	}
	return http.DefaultClient
}

func (c *Client) available(url string) (s string, t time.Time, err error) {
	res, err := c.clientHTTP().Get(url)
	if err != nil {
		return
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		err = errors.New(http.StatusText(res.StatusCode))
		return
	}
	resp := &response{}
	buf := &bytes.Buffer{}
	if err = json.NewDecoder(io.TeeReader(res.Body, buf)).Decode(resp); err != nil {
		return
	}
	if !resp.ArchivedSnapshots.Closest.Available {
		err = errNotAvailable
		return
	}
	s = resp.ArchivedSnapshots.Closest.URL
	t = resp.ArchivedSnapshots.Closest.Timestamp.Time()
	return
}

// Available queries the archive for cached snapshot of website given by the url.
// If it's available, it returns its URL and creation time of the most recent
// snapshot. Otherwise it returns non-nil error.
func (c *Client) Available(url string) (string, time.Time, error) {
	return c.available(fmt.Sprintf(req, url))
}

// AvailableAt queries the archive for cached snapshot of website given by the url
// and creation time. If it's available, the function returns its URL and creation
// time which is the closest to the requested one.
func (c *Client) AvailableAt(url string, timestamp Timestamp) (string, time.Time, error) {
	return c.available(fmt.Sprintf(reqTimestamp, url, timestamp))
}
<|endoftext|>


<commit_before>package json

import "github.com/gopherjs/gopherjs/js"

func Marshal(v interface{}) ([]byte, error) {
	return MarshalIndent(v, "", "")
}

func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) {
	s, err := MarshalIndentString(v, prefix, indent)
	return []byte(s), err
}

func MarshalString(v interface{}) (string, error) {
	return MarshalIndentString(v, "", "")
}

func MarshalIndentString(v interface{}, prefix, indent string) (string, error) {
	run := true
	str := js.Global.Get("JSON").Call("stringify", v, func(key, value *js.Object) *js.Object {
		if run {
			run = false
			filter(js.InternalObject(v).Get("constructor"), value)
		}
		return value
	}, indent)
	if len(prefix) > 0 {
		str = str.Call("replace", js.Global.Get("RegExp").New("\n", "g"), "\n"+prefix)
	}
	return str.String(), nil
}

const (
	arrayKind  = 17
	mapKind    = 21
	ptrKind    = 22
	sliceKind  = 23
	structKind = 25
)

func filter(t, value *js.Object) {
	switch t.Get("kind").Int() {
	case arrayKind, sliceKind:
		len := value.Length()
		for i := 0; i < len; i++ {
			filter(t.Get("elem"), value.Index(i))
		}
	case mapKind:
		keys := js.Global.Get("Object").Call("keys", value)
		len := keys.Length()
		for i := 0; i < len; i++ {
			filter(t.Get("elem"), value.Get(keys.Index(i).String()))
		}
	case ptrKind:
		filter(t.Get("elem"), value)
	case structKind:
		nf := t.Get("fields").Length()
		for i := 0; i < nf; i++ {
			field := t.Get("fields").Index(i)
			tag := field.Get("tag").Call("replace", js.Global.Get("RegExp").New(".*json:\"([^\"]*)\".*"), "$1") // need to do this better
			if tag.String() == "-" {
				value.Delete(field.Get("name").String())
				continue
			}
			name := field.Get("name")
			filter(field.Get("typ"), value.Get(name.String()))
			if tag.Length() != 0 {
				p := tag.Call("split", ",", 2)
				if p.Index(0).Length() != 0 {
					if p.Index(0) != name {
						value.Set(p.Index(0).String(), value.Get(name.String()))
						value.Delete(name.String())
					}
				}
			}
		}
	}
}
<commit_msg>interfaces now get delved into as well<commit_after>package json

import "github.com/gopherjs/gopherjs/js"

func Marshal(v interface{}) ([]byte, error) {
	return MarshalIndent(v, "", "")
}

func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) {
	s, err := MarshalIndentString(v, prefix, indent)
	return []byte(s), err
}

func MarshalString(v interface{}) (string, error) {
	return MarshalIndentString(v, "", "")
}

func MarshalIndentString(v interface{}, prefix, indent string) (string, error) {
	run := true
	str := js.Global.Get("JSON").Call("stringify", v, func(key, value *js.Object) *js.Object {
		if run {
			run = false
			filter(js.InternalObject(v), js.InternalObject(v).Get("constructor"), value)
		}
		return value
	}, indent)
	if len(prefix) > 0 {
		str = str.Call("replace", js.Global.Get("RegExp").New("\n", "g"), "\n"+prefix)
	}
	return str.String(), nil
}

const (
	arrayKind     = 17
	interfaceKind = 20
	mapKind       = 21
	ptrKind       = 22
	sliceKind     = 23
	structKind    = 25
)

func filter(v, t, value *js.Object) {
	switch t.Get("kind").Int() {
	case arrayKind, sliceKind:
		len := value.Length()
		for i := 0; i < len; i++ {
			filter(v.Get("$array").Index(i), t.Get("elem"), value.Index(i))
		}
	case mapKind:
		keys := js.Global.Get("Object").Call("keys", value)
		len := keys.Length()
		for i := 0; i < len; i++ {
			name := keys.Index(i).String()
			filter(v.Get("$map").Get(name), t.Get("elem"), value.Get(name))
		}
	case ptrKind:
		filter(v, t.Get("elem"), value)
	case structKind:
		nf := t.Get("fields").Length()
		for i := 0; i < nf; i++ {
			field := t.Get("fields").Index(i)
			tag := field.Get("tag").Call("replace", js.Global.Get("RegExp").New(".*json:\"([^\"]*)\".*"), "$1") // need to do this better
			if tag.String() == "-" {
				value.Delete(field.Get("name").String())
				continue
			}
			name := field.Get("name")
			filter(v.Get(name.String()), field.Get("typ"), value.Get(name.String()))
			if tag.Length() != 0 {
				p := tag.Call("split", ",", 2)
				if p.Index(0).Length() != 0 {
					if p.Index(0) != name {
						value.Set(p.Index(0).String(), value.Get(name.String()))
						value.Delete(name.String())
					}
				}
			}
		}
	case interfaceKind:
		filter(v.Get("$val"), v.Get("constructor"), value)
	}
}
<|endoftext|>


<commit_before>package jws

import (
	"crypto/ecdsa"
	"crypto/hmac"
	"crypto/rsa"
	"errors"
	"math/big"

	"github.com/lestrrat/go-jwx/internal/debug"
	"github.com/lestrrat/go-jwx/jwa"
)

type payloadVerifier interface {
	PayloadVerify([]byte, []byte) error
}

func doMessageVerify(alg jwa.SignatureAlgorithm, v payloadVerifier, m *Message) error {
	var err error
	payload, err := m.Payload.Base64Encode()
	if err != nil {
		return err
	}
	for _, sig := range m.Signatures {
		if sig.ProtectedHeader.Algorithm != alg {
			continue
		}

		var phbuf []byte
		if sig.ProtectedHeader.Source.Len() > 0 {
			phbuf, err = sig.ProtectedHeader.Source.Base64Encode()
			if err != nil {
				continue
			}
		} else {
			phbuf, err = sig.ProtectedHeader.Base64Encode()
			if err != nil {
				continue
			}
		}
		siv := append(append(phbuf, '.'), payload...)

		debug.Printf("siv = '%s'", siv)
		if err := v.PayloadVerify(siv, sig.Signature.Bytes()); err != nil {
			debug.Printf("Payload verify failed: %s", err)
			continue
		}

		return nil
	}

	return errors.New("none of the signatures could be verified")
}

func NewRsaVerify(alg jwa.SignatureAlgorithm, key *rsa.PublicKey) (*RsaVerify, error) {
	if key == nil {
		return nil, ErrMissingPublicKey
	}

	switch alg {
	case jwa.RS256, jwa.RS384, jwa.RS512, jwa.PS256, jwa.PS384, jwa.PS512:
	default:
		return nil, ErrUnsupportedAlgorithm
	}

	h, err := rsaHashForAlg(alg)
	if err != nil {
		return nil, ErrUnsupportedAlgorithm
	}

	return &RsaVerify{alg: alg, hash: h, pubkey: key}, nil
}

func (v RsaVerify) PayloadVerify(payload, signature []byte) error {
	pubkey := v.pubkey
	hfunc := v.hash
	h := hfunc.New()
	h.Write(payload)

	var err error
	switch v.alg {
	case jwa.RS256, jwa.RS384, jwa.RS512:
		err = rsa.VerifyPKCS1v15(pubkey, hfunc, h.Sum(nil), signature)
	case jwa.PS256, jwa.PS384, jwa.PS512:
		err = rsa.VerifyPSS(pubkey, hfunc, h.Sum(nil), signature, nil)
	}

	if err != nil {
		return err
	}
	return nil
}

// Verify checks that signature generated for `payload` matches `signature`.
// This fulfills the `Verifier` interface
func (v RsaVerify) Verify(m *Message) error {
	return doMessageVerify(v.alg, v, m)
}

func NewEcdsaVerify(alg jwa.SignatureAlgorithm, key *ecdsa.PublicKey) (*EcdsaVerify, error) {
	if key == nil {
		return nil, ErrMissingPublicKey
	}

	switch alg {
	case jwa.ES256, jwa.ES384, jwa.ES512:
	default:
		return nil, ErrUnsupportedAlgorithm
	}

	h, err := ecdsaHashForAlg(alg)
	if err != nil {
		return nil, ErrUnsupportedAlgorithm
	}

	return &EcdsaVerify{alg: alg, hash: h, pubkey: key}, nil
}

// Verify checks that signature generated for `payload` matches `signature`.
// This fulfills the `Verifier` interface
func (v EcdsaVerify) Verify(m *Message) error {
	return doMessageVerify(v.alg, v, m)
}

func (v EcdsaVerify) PayloadVerify(payload, signature []byte) error {
	pubkey := v.pubkey
	hfunc := v.hash
	keysiz := hfunc.Size()
	if len(signature) != 2*keysiz {
		return ErrInvalidEcdsaSignatureSize
	}

	rv := (&big.Int{}).SetBytes(signature[:keysiz])
	sv := (&big.Int{}).SetBytes(signature[keysiz:])

	h := hfunc.New()
	h.Write(payload)
	signed := h.Sum(nil)

	debug.Printf("payload -> %s, signed -> %x", payload, signed)

	if !ecdsa.Verify(pubkey, signed, rv, sv) {
		return ErrInvalidSignature
	}
	return nil
}

type HmacVerify struct {
	signer *HmacSign
}

func NewHmacVerify(alg jwa.SignatureAlgorithm, key []byte) (*HmacVerify, error) {
	s, err := NewHmacSign(alg, key)
	if err != nil {
		return nil, err
	}
	return &HmacVerify{signer: s}, nil
}

// Verify checks that signature generated for `payload` matches `signature`.
// This fulfills the `Verifier` interface
func (v HmacVerify) Verify(m *Message) error {
	return doMessageVerify(v.signer.SignatureAlgorithm(), v, m)
}

func (v HmacVerify) PayloadVerify(payload, mac []byte) error {
	expected, err := v.signer.PayloadSign(payload)
	if err != nil {
		return err
	}

	if !hmac.Equal(mac, expected) {
		return ErrInvalidSignature
	}
	return nil
}
<commit_msg>Hide methods that do not need to be exposed<commit_after>package jws

import (
	"crypto/ecdsa"
	"crypto/hmac"
	"crypto/rsa"
	"errors"
	"math/big"

	"github.com/lestrrat/go-jwx/internal/debug"
	"github.com/lestrrat/go-jwx/jwa"
)

type payloadVerifier interface {
	payloadVerify([]byte, []byte) error
}

func doMessageVerify(alg jwa.SignatureAlgorithm, v payloadVerifier, m *Message) error {
	var err error
	payload, err := m.Payload.Base64Encode()
	if err != nil {
		return err
	}
	for _, sig := range m.Signatures {
		if sig.ProtectedHeader.Algorithm != alg {
			continue
		}

		var phbuf []byte
		if sig.ProtectedHeader.Source.Len() > 0 {
			phbuf, err = sig.ProtectedHeader.Source.Base64Encode()
			if err != nil {
				continue
			}
		} else {
			phbuf, err = sig.ProtectedHeader.Base64Encode()
			if err != nil {
				continue
			}
		}
		siv := append(append(phbuf, '.'), payload...)

		debug.Printf("siv = '%s'", siv)
		if err := v.payloadVerify(siv, sig.Signature.Bytes()); err != nil {
			debug.Printf("Payload verify failed: %s", err)
			continue
		}

		return nil
	}

	return errors.New("none of the signatures could be verified")
}

func NewRsaVerify(alg jwa.SignatureAlgorithm, key *rsa.PublicKey) (*RsaVerify, error) {
	if key == nil {
		return nil, ErrMissingPublicKey
	}

	switch alg {
	case jwa.RS256, jwa.RS384, jwa.RS512, jwa.PS256, jwa.PS384, jwa.PS512:
	default:
		return nil, ErrUnsupportedAlgorithm
	}

	h, err := rsaHashForAlg(alg)
	if err != nil {
		return nil, ErrUnsupportedAlgorithm
	}

	return &RsaVerify{alg: alg, hash: h, pubkey: key}, nil
}

func (v RsaVerify) payloadVerify(payload, signature []byte) error {
	pubkey := v.pubkey
	hfunc := v.hash
	h := hfunc.New()
	h.Write(payload)

	var err error
	switch v.alg {
	case jwa.RS256, jwa.RS384, jwa.RS512:
		err = rsa.VerifyPKCS1v15(pubkey, hfunc, h.Sum(nil), signature)
	case jwa.PS256, jwa.PS384, jwa.PS512:
		err = rsa.VerifyPSS(pubkey, hfunc, h.Sum(nil), signature, nil)
	}

	if err != nil {
		return err
	}
	return nil
}

// Verify checks that signature generated for `payload` matches `signature`.
// This fulfills the `Verifier` interface
func (v RsaVerify) Verify(m *Message) error {
	return doMessageVerify(v.alg, v, m)
}

func NewEcdsaVerify(alg jwa.SignatureAlgorithm, key *ecdsa.PublicKey) (*EcdsaVerify, error) {
	if key == nil {
		return nil, ErrMissingPublicKey
	}

	switch alg {
	case jwa.ES256, jwa.ES384, jwa.ES512:
	default:
		return nil, ErrUnsupportedAlgorithm
	}

	h, err := ecdsaHashForAlg(alg)
	if err != nil {
		return nil, ErrUnsupportedAlgorithm
	}

	return &EcdsaVerify{alg: alg, hash: h, pubkey: key}, nil
}

// Verify checks that signature generated for `payload` matches `signature`.
// This fulfills the `Verifier` interface
func (v EcdsaVerify) Verify(m *Message) error {
	return doMessageVerify(v.alg, v, m)
}

func (v EcdsaVerify) payloadVerify(payload, signature []byte) error {
	pubkey := v.pubkey
	hfunc := v.hash
	keysiz := hfunc.Size()
	if len(signature) != 2*keysiz {
		return ErrInvalidEcdsaSignatureSize
	}

	rv := (&big.Int{}).SetBytes(signature[:keysiz])
	sv := (&big.Int{}).SetBytes(signature[keysiz:])

	h := hfunc.New()
	h.Write(payload)
	signed := h.Sum(nil)

	debug.Printf("payload -> %s, signed -> %x", payload, signed)

	if !ecdsa.Verify(pubkey, signed, rv, sv) {
		return ErrInvalidSignature
	}
	return nil
}

type HmacVerify struct {
	signer *HmacSign
}

func NewHmacVerify(alg jwa.SignatureAlgorithm, key []byte) (*HmacVerify, error) {
	s, err := NewHmacSign(alg, key)
	if err != nil {
		return nil, err
	}
	return &HmacVerify{signer: s}, nil
}

// Verify checks that signature generated for `payload` matches `signature`.
// This fulfills the `Verifier` interface
func (v HmacVerify) Verify(m *Message) error {
	return doMessageVerify(v.signer.SignatureAlgorithm(), v, m)
}

func (v HmacVerify) payloadVerify(payload, mac []byte) error {
	expected, err := v.signer.PayloadSign(payload)
	if err != nil {
		return err
	}

	if !hmac.Equal(mac, expected) {
		return ErrInvalidSignature
	}
	return nil
}
<|endoftext|>


<commit_before>package kdebug

import (
	"errors"
	"fmt"
	"reflect"
	"runtime"
)

func VarDump(s interface{}) {
	indent := ""
	switch s.(type) {
	case reflect.Value:
		varDump(s.(reflect.Value), indent, "")
	default:
		varDump(reflect.ValueOf(s), indent, "")
	}
	//varDump(reflect.ValueOf(s), indent, "")
}

func varDump(value reflect.Value, indent string, preStr string) {
	var vKind = value.Kind()
	rIndent := indent
	if preStr != "" {
		rIndent = preStr
	}

	switch {
	case vKind == 0:
		fmt.Printf("%s[%s] : %v\n", rIndent, "nil", value)

	//bool + int +uint
	case vKind < 12:
		fmt.Printf("%s[%s] : %v\n", rIndent, value.Type(), value)

	//uintptr
	case vKind == 12:
		//fmt.Printf("%s[%s] ", r_indent, value.Type())
		varDump(value, indent, fmt.Sprintf("%s[%s] --> ", indent, value.Type()))

	//float + complex
	case vKind < 17:
		fmt.Printf("%s[%s] : %v\n", rIndent, value.Type(), value)

	//Array
	case vKind == 17:
		fmt.Printf("%sarray[%s](%d) ==>\n", rIndent, value.Type(), value.Len())
		index := 0
		for {
			if index >= value.Len() {
				break
			}
			//fmt.Printf("%s[%d] :", r_indent+"    ", index)
			varDump(value.Index(index), "    "+indent, fmt.Sprintf("%s[%d] :", rIndent+"    ", index))
			index++
		}

	//chan\func\Interface
	case vKind < 21:
		fmt.Printf("%s[%s]%v", rIndent, value.Type(), value)

	//Map
	case vKind == 21:
		fmt.Printf("%smap[%s](%d) ==>\n", rIndent, value.Type(), value.Len())
		keys := value.MapKeys()
		for _, key := range keys {
			//fmt.Printf("%s[%s] : ", indent+"    ", key)
			varDump(value.MapIndex(key), "    "+indent, fmt.Sprintf("%s\"%s\" : ", indent+"    ", key))
		}

	//ptr
	case vKind == 22:
		//fmt.Printf("%s[%s] -->\n", indent, value.Type())
		varDump(reflect.Indirect(value), indent, fmt.Sprintf("%s[%s] --> ", indent, value.Type()))

	//slice
	case vKind == 23:
		fmt.Printf("%sslice[%s](%d) ==>\n", indent, value.Type(), value.Len())
		//fmt.Printf("%s%s", indent+"    ", t.Field(k).Name)
		index := 0
		for {
			if index >= value.Len() {
				break
			}
			//fmt.Printf("%s[%d] : ", r_indent+"    ", index)
			varDump(value.Index(index), "    "+indent, fmt.Sprintf("%s[%d] : ", indent+"    ", index))
			index++
		}
		fmt.Printf("%s//end %s\n", indent, vKind.String())

	//string
	case vKind == 24:
		fmt.Printf("%s[%s](%d) ==> \"%s\"\n", rIndent, value.Type(), value.Len(), value)

	//Struct
	case vKind == 25:
		fmt.Printf("%sstruct[%s] ==> {\n", rIndent, value.Type())

		t := value.Type()
		for k := 0; k < value.NumField(); k++ {
			varDump(value.Field(k), indent+"    ", fmt.Sprintf("%s%s ", indent+"    ", t.Field(k).Name))
		}
		fmt.Printf("%s} //end %s\n", indent, vKind.String())

	//Unsafeptr
	case vKind == 26:
		//fmt.Printf("%s[%s] -->\n", r_indent, vKind.String())
		varDump(reflect.Indirect(value), indent, fmt.Sprintf("%s[%s] --> ", indent, vKind.String()))

	default:
		fmt.Printf("%s[%s] : %v\n", rIndent, "Unknown", value)
	}

	return
}

func GetFuncName() (string, error) {
	pc, _, _, succ := runtime.Caller(1)
	if !succ {
		return "", errors.New("get current function name failed")
	}

	return runtime.FuncForPC(pc).Name(), nil
}
<commit_msg>修复map和slice输出时，不输出变量名的BUG<commit_after>package kdebug

import (
	"errors"
	"fmt"
	"reflect"
	"runtime"
)

func VarDump(s interface{}) {
	indent := ""
	switch s.(type) {
	case reflect.Value:
		varDump(s.(reflect.Value), indent, "")
	default:
		varDump(reflect.ValueOf(s), indent, "")
	}
	//varDump(reflect.ValueOf(s), indent, "")
}

func varDump(value reflect.Value, indent string, preStr string) {
	var vKind = value.Kind()
	rIndent := indent
	if preStr != "" {
		rIndent = preStr
	}

	switch {
	case vKind == 0:
		fmt.Printf("%s[%s] : %v\n", rIndent, "nil", value)

	//bool + int +uint
	case vKind < 12:
		fmt.Printf("%s[%s] : %v\n", rIndent, value.Type(), value)

	//uintptr
	case vKind == 12:
		//fmt.Printf("%s[%s] ", r_indent, value.Type())
		varDump(value, indent, fmt.Sprintf("%s[%s] --> ", rIndent, value.Type()))

	//float + complex
	case vKind < 17:
		fmt.Printf("%s[%s] : %v\n", rIndent, value.Type(), value)

	//Array
	case vKind == 17:
		fmt.Printf("%sarray[%s](%d) ==>\n", rIndent, value.Type(), value.Len())
		index := 0
		for {
			if index >= value.Len() {
				break
			}
			//fmt.Printf("%s[%d] :", r_indent+"    ", index)
			varDump(value.Index(index), "    "+indent, fmt.Sprintf("%s[%d] :", rIndent+"    ", index))
			index++
		}

	//chan\func\Interface
	case vKind < 21:
		fmt.Printf("%s[%s]%v", rIndent, value.Type(), value)

	//Map
	case vKind == 21:
		fmt.Printf("%smap[%s](%d) ==>\n", rIndent, value.Type(), value.Len())
		keys := value.MapKeys()
		for _, key := range keys {
			//fmt.Printf("%s[%s] : ", indent+"    ", key)
			varDump(value.MapIndex(key), "    "+indent, fmt.Sprintf("%s\"%s\" : ", indent+"    ", key))
		}

	//ptr
	case vKind == 22:
		//fmt.Printf("%s[%s] -->\n", indent, value.Type())
		varDump(reflect.Indirect(value), indent, fmt.Sprintf("%s[%s] --> ", rIndent, value.Type()))

	//slice
	case vKind == 23:
		fmt.Printf("%sslice[%s](%d) ==>\n", rIndent, value.Type(), value.Len())
		//fmt.Printf("%s%s", indent+"    ", t.Field(k).Name)
		index := 0
		for {
			if index >= value.Len() {
				break
			}
			//fmt.Printf("%s[%d] : ", r_indent+"    ", index)
			varDump(value.Index(index), "    "+indent, fmt.Sprintf("%s[%d] : ", indent+"    ", index))
			index++
		}
		fmt.Printf("%s//end %s\n", indent, vKind.String())

	//string
	case vKind == 24:
		fmt.Printf("%s[%s](%d) ==> \"%s\"\n", rIndent, value.Type(), value.Len(), value)

	//Struct
	case vKind == 25:
		fmt.Printf("%sstruct[%s] ==> {\n", rIndent, value.Type())

		t := value.Type()
		for k := 0; k < value.NumField(); k++ {
			varDump(value.Field(k), indent+"    ", fmt.Sprintf("%s%s ", indent+"    ", t.Field(k).Name))
		}
		fmt.Printf("%s} //end %s\n", indent, vKind.String())

	//Unsafeptr
	case vKind == 26:
		//fmt.Printf("%s[%s] -->\n", r_indent, vKind.String())
		varDump(reflect.Indirect(value), indent, fmt.Sprintf("%s[%s] --> ", rIndent, vKind.String()))

	default:
		fmt.Printf("%s[%s] : %v\n", rIndent, "Unknown", value)
	}

	return
}

func GetFuncName() (string, error) {
	pc, _, _, succ := runtime.Caller(1)
	if !succ {
		return "", errors.New("get current function name failed")
	}

	return runtime.FuncForPC(pc).Name(), nil
}
<|endoftext|>


<commit_before>// Copyright 2016 govend. All rights reserved.
// Use of this source code is governed by an Apache 2.0
// license that can be found in the LICENSE file.

package deps

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/govend/govend/deps/semver"
)

// Vendorable ensures the current local setup is conducive to vendoring.
//
// If the current version of Go cannot be parsed, then trust it supports
// vendoring, but display a message if verbose is true.
func Vendorable(verbose bool) error {
	err := checkGopath()
	if err != nil {
		return err
	}

	go15, _ := semver.New("1.5.0")
	go16, _ := semver.New("1.6.0")
	go17, _ := semver.New("1.7.0")

	version, err := semver.New(strings.TrimPrefix(runtime.Version(), "go"))
	if err != nil {
		if verbose {
			fmt.Printf("\n%s\n", err)
		}
		return nil
	}

	if version.LessThan(go15) {
		return errors.New("vendoring requires Go versions 1.5+")
	}

	if version.GreaterThanEqual(go15) && version.LessThan(go16) {
		if os.Getenv("GO15VENDOREXPERIMENT") != "1" {
			return errors.New("Go 1.5.x requires 'GO15VENDOREXPERIMENT=1'")
		}
	}

	if version.GreaterThanEqual(go16) && version.LessThan(go17) {
		if os.Getenv("GO15VENDOREXPERIMENT") == "0" {
			return errors.New("Go 1.6.x cannot vendor with 'GO15VENDOREXPERIMENT=0'")
		}
	}

	return nil
}

// checkGopath checks if the current working directory has $GOPATH/src as a prefix.
func checkGopath() error {
	gopath := os.Getenv("GOPATH")
	if len(gopath) == 0 {
		return errors.New("please set your $GOPATH")
	}

	// determine the current working directory and coerce it to an absolute
	cwd, err := os.Getwd()
	if err != nil {
		return err
	}
	cwd, err = filepath.Abs(cwd)
	if err != nil {
		return err
	}

	// check $GOPATH/src
	gosrc := filepath.Join(gopath, "src") + string(filepath.Separator)
	if !strings.HasPrefix(cwd, gosrc) {
		return errors.New("you cannot vendor packages outside of your $GOPATH/src")
	}

	return nil
}
<commit_msg>Resolve cwd symlinks before checking if we're in $GOPATH/src<commit_after>// Copyright 2016 govend. All rights reserved.
// Use of this source code is governed by an Apache 2.0
// license that can be found in the LICENSE file.

package deps

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/govend/govend/deps/semver"
)

// Vendorable ensures the current local setup is conducive to vendoring.
//
// If the current version of Go cannot be parsed, then trust it supports
// vendoring, but display a message if verbose is true.
func Vendorable(verbose bool) error {
	err := checkGopath()
	if err != nil {
		return err
	}

	go15, _ := semver.New("1.5.0")
	go16, _ := semver.New("1.6.0")
	go17, _ := semver.New("1.7.0")

	version, err := semver.New(strings.TrimPrefix(runtime.Version(), "go"))
	if err != nil {
		if verbose {
			fmt.Printf("\n%s\n", err)
		}
		return nil
	}

	if version.LessThan(go15) {
		return errors.New("vendoring requires Go versions 1.5+")
	}

	if version.GreaterThanEqual(go15) && version.LessThan(go16) {
		if os.Getenv("GO15VENDOREXPERIMENT") != "1" {
			return errors.New("Go 1.5.x requires 'GO15VENDOREXPERIMENT=1'")
		}
	}

	if version.GreaterThanEqual(go16) && version.LessThan(go17) {
		if os.Getenv("GO15VENDOREXPERIMENT") == "0" {
			return errors.New("Go 1.6.x cannot vendor with 'GO15VENDOREXPERIMENT=0'")
		}
	}

	return nil
}

// checkGopath checks if the current working directory has $GOPATH/src as a prefix.
func checkGopath() error {
	gopath := os.Getenv("GOPATH")
	if len(gopath) == 0 {
		return errors.New("please set your $GOPATH")
	}

	// determine the current working directory and coerce it to an absolute
	cwd, err := os.Getwd()
	if err != nil {
		return err
	}

	cwd, err = filepath.Abs(cwd)
	if err != nil {
		return err
	}

	cwd, err = filepath.EvalSymlinks(cwd)
	if err != nil {
		return err
	}

	// check $GOPATH/src
	gosrc := filepath.Join(gopath, "src") + string(filepath.Separator)
	if !strings.HasPrefix(cwd, gosrc) {
		return errors.New("you cannot vendor packages outside of your $GOPATH/src")
	}

	return nil
}
<|endoftext|>


<commit_before>package brokerage_server_ib

import (
	"context"
	"math"
	"strconv"
	"time"

	"github.com/dimfeld/brokerage_server/types"
	"github.com/dimfeld/ib"
)

func (p *IB) GetTrades(ctx context.Context, startTime time.Time) ([]*types.Trade, error) {
	type tradeData struct {
		output types.Trade
		size   int
		price  float64
	}

	req := &ib.RequestExecutions{}
	if !startTime.IsZero() {
		req.Filter = ib.ExecutionFilter{
			Time: startTime,
		}
	}

	trades := map[int64]*tradeData{}
	commissions := map[string]*ib.CommissionReport{}

	// Commissions reports come in as unmatched for some reason, so subscribe
	// to unmatched data to get them.
	unmChan, closeUnmData := p.subscribeUnmatched()
	go func() {
		for r := range unmChan {
			if data, ok := r.(*ib.CommissionReport); ok {
				commissions[data.ExecutionID] = data
			}
		}
	}()

	_, err := p.syncMatchedRequest(ctx, req, func(r ib.Reply) (replyBehavior, error) {
		switch data := r.(type) {
		case *ib.ExecutionData:
			t := trades[data.Exec.PermID]
			if t == nil {
				t = &tradeData{
					output: types.Trade{
						Symbol:  data.Contract.Symbol,
						Time:    data.Exec.Time,
						Account: data.Exec.AccountCode,
						Broker:  BrokerName,
						OrderId: strconv.FormatInt(data.Exec.PermID, 10),
						RawData: data.Contract,
					},
					size:  0,
					price: 0,
				}
				trades[data.Exec.PermID] = t
			}

			con := &data.Contract
			if con.SecurityType == "BAG" || con.SecurityType == "STK" {
				p.LogDebugNormal("bag trade", "data", data.Exec)
				cq := int(data.Exec.CumQty)
				if cq > t.size {
					t.size = cq
				}

				t.price += data.Exec.Price * float64(data.Exec.Shares)
				break
			}

			e := &types.Execution{
				ExecutionId: data.Exec.ExecID,
				Exchange:    con.Exchange,
				Side:        data.Exec.Side,
				Size:        int(data.Exec.Shares),
				Price:       data.Exec.Price,
			}
			if con.SecurityType == "OPT" {
				if con.Right == "P" || con.Right == "PUT" {
					e.OptionType = "PUT"
				} else {
					e.OptionType = "CALL"
				}
				e.Strike = con.Strike
				e.Expiration = ExpiryIBFormat(con.Expiry).toOCCFormat().String()
				if multiplier, err := strconv.Atoi(con.Multiplier); err == nil {
					e.Multiplier = multiplier
				}
			}

			e.Exchange = con.Exchange
			e.Side = data.Exec.Side
			e.Size = int(data.Exec.Shares)
			e.Price = data.Exec.Price
			e.Time = data.Exec.Time
			e.RawData = data.Exec

			t.output.Executions = append(t.output.Executions, e)

		case *ib.ExecutionDataEnd:
			return REPLY_DONE, nil

		case *ib.ErrorMessage:
			return REPLY_DONE, data.Error()
		}
		return REPLY_CONTINUE, nil
	})

	closeUnmData()

	if err != nil {
		return nil, err
	}

	out := make([]*types.Trade, 0, len(trades))

	for _, v := range trades {
		// Fill in the commissions for each execution.
		for _, e := range v.output.Executions {
			comm := commissions[e.ExecutionId]
			if comm != nil {
				e.Commissions = comm.Commission
				if comm.RealizedPNL < 1e100 { // Filter out meaningless data.
					e.RealizedPnL = comm.RealizedPNL
				} else {
					e.RealizedPnL = math.NaN()
				}
			}
		}
		v.output.Price = v.price / float64(v.size)
		v.output.Size = v.size
		out = append(out, &v.output)
	}

	return out, nil
}
<commit_msg>Save STK executions<commit_after>package brokerage_server_ib

import (
	"context"
	"math"
	"strconv"
	"time"

	"github.com/dimfeld/brokerage_server/types"
	"github.com/dimfeld/ib"
)

func (p *IB) GetTrades(ctx context.Context, startTime time.Time) ([]*types.Trade, error) {
	type tradeData struct {
		output types.Trade
		size   int
		price  float64
	}

	req := &ib.RequestExecutions{}
	if !startTime.IsZero() {
		req.Filter = ib.ExecutionFilter{
			Time: startTime,
		}
	}

	trades := map[int64]*tradeData{}
	commissions := map[string]*ib.CommissionReport{}

	// Commissions reports come in as unmatched for some reason, so subscribe
	// to unmatched data to get them.
	unmChan, closeUnmData := p.subscribeUnmatched()
	go func() {
		for r := range unmChan {
			if data, ok := r.(*ib.CommissionReport); ok {
				commissions[data.ExecutionID] = data
			}
		}
	}()

	_, err := p.syncMatchedRequest(ctx, req, func(r ib.Reply) (replyBehavior, error) {
		switch data := r.(type) {
		case *ib.ExecutionData:
			t := trades[data.Exec.PermID]
			if t == nil {
				t = &tradeData{
					output: types.Trade{
						Symbol:  data.Contract.Symbol,
						Time:    data.Exec.Time,
						Account: data.Exec.AccountCode,
						Broker:  BrokerName,
						OrderId: strconv.FormatInt(data.Exec.PermID, 10),
						RawData: data.Contract,
					},
					size:  0,
					price: 0,
				}
				trades[data.Exec.PermID] = t
			}

			con := &data.Contract
			if con.SecurityType == "BAG" || con.SecurityType == "STK" {
				cq := int(data.Exec.CumQty)
				if cq > t.size {
					t.size = cq
				}

				t.price += data.Exec.Price * float64(data.Exec.Shares)
				if con.SecurityType == "BAG" {
					p.LogDebugNormal("bag trade", "data", data.Exec)
					break
				}
			}

			e := &types.Execution{
				ExecutionId: data.Exec.ExecID,
				Exchange:    con.Exchange,
				Side:        data.Exec.Side,
				Size:        int(data.Exec.Shares),
				Price:       data.Exec.Price,
			}
			if con.SecurityType == "OPT" {
				if con.Right == "P" || con.Right == "PUT" {
					e.OptionType = "PUT"
				} else {
					e.OptionType = "CALL"
				}
				e.Strike = con.Strike
				e.Expiration = ExpiryIBFormat(con.Expiry).toOCCFormat().String()
				if multiplier, err := strconv.Atoi(con.Multiplier); err == nil {
					e.Multiplier = multiplier
				}
			}

			e.Exchange = con.Exchange
			e.Side = data.Exec.Side
			e.Size = int(data.Exec.Shares)
			e.Price = data.Exec.Price
			e.Time = data.Exec.Time
			e.RawData = data.Exec

			t.output.Executions = append(t.output.Executions, e)

		case *ib.ExecutionDataEnd:
			return REPLY_DONE, nil

		case *ib.ErrorMessage:
			return REPLY_DONE, data.Error()
		}
		return REPLY_CONTINUE, nil
	})

	closeUnmData()

	if err != nil {
		return nil, err
	}

	out := make([]*types.Trade, 0, len(trades))

	for _, v := range trades {
		// Fill in the commissions for each execution.
		for _, e := range v.output.Executions {
			comm := commissions[e.ExecutionId]
			if comm != nil {
				e.Commissions = comm.Commission
				if comm.RealizedPNL < 1e100 { // Filter out meaningless data.
					e.RealizedPnL = comm.RealizedPNL
				} else {
					e.RealizedPnL = math.NaN()
				}
			}
		}
		v.output.Price = v.price / float64(v.size)
		v.output.Size = v.size
		out = append(out, &v.output)
	}

	return out, nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"io"
	"log"
	"net"
	"os"
)

func main() {
	if len(os.Args) < 4 {
		println("Too few arguments")
		println("Usage:")
		println("	tunnel [transport] [local host:port] [remote host:port]")
		println("Example:")
		println("	tunnel tls :9999 example.com:443")
		println("Supported transports:")
		println("	tls, plain")
		return
	}

	transport, local, remote := os.Args[1], os.Args[2], os.Args[3]

	var handler func() (net.Conn, error)
	switch transport {
	case "tls":
		handler = func() (net.Conn, error) {
			return tls.Dial("tcp", remote, &tls.Config{
				InsecureSkipVerify: true,
			})
		}
	case "plain":
		handler = func() (net.Conn, error) {
			return net.Dial("tcp", remote)
		}
	}

	l, err := net.Listen("tcp", local)
	if err != nil {
		panic(err)
	}

	for {
		conn, err := l.Accept()
		if err != nil {
			panic(err)
		}

		go func() {
			proxyConn, err := handler()
			if err != nil {
				log.Printf("%s -> %s: failed: %s\n", conn.RemoteAddr(), os.Args[2], err)
				return
			}
			log.Printf("%s -> %s: connected\n", conn.RemoteAddr(), proxyConn.RemoteAddr())

			defer func() {
				log.Printf("%s -> %s: disconnected\n", conn.RemoteAddr(), proxyConn.RemoteAddr())
				proxyConn.Close()
				conn.Close()
			}()

			go io.Copy(proxyConn, conn)
			io.Copy(conn, proxyConn)
		}()
	}
}
<commit_msg>Nicer shutdown<commit_after>package main

import (
	"crypto/tls"
	"io"
	"log"
	"net"
	"os"
)

func main() {
	if len(os.Args) < 4 {
		println("Too few arguments")
		println("Usage:")
		println("	tunnel [transport] [local host:port] [remote host:port]")
		println("Example:")
		println("	tunnel tls :9999 example.com:443")
		println("Supported transports:")
		println("	tls, plain")
		return
	}

	transport, local, remote := os.Args[1], os.Args[2], os.Args[3]

	var handler func() (net.Conn, error)
	switch transport {
	case "tls":
		handler = func() (net.Conn, error) {
			return tls.Dial("tcp", remote, &tls.Config{
				InsecureSkipVerify: true,
			})
		}
	case "plain":
		handler = func() (net.Conn, error) {
			return net.Dial("tcp", remote)
		}
	}

	l, err := net.Listen("tcp", local)
	if err != nil {
		panic(err)
	}

	log.Printf("Transport: %s, binding address: %s, destination: %s", transport, local, remote)

	for {
		conn, err := l.Accept()
		if err != nil {
			panic(err)
		}

		go func() {
			pconn, err := handler()
			if err != nil {
				log.Printf("%s -> %s: failed: %s\n", conn.RemoteAddr(), os.Args[2], err)
				return
			}
			log.Printf("%s -> %s: connected\n", conn.RemoteAddr(), pconn.RemoteAddr())

			closer := make(chan bool, 1)

			go func() {
				<-closer
				conn.Close()
				pconn.Close()
				log.Printf("%s -> %s: disconnected\n", conn.RemoteAddr(), pconn.RemoteAddr())
			}()

			go func() {
				io.Copy(pconn, conn)
				closer <- true
			}()

			go func() {
				io.Copy(conn, pconn)
				closer <- true
			}()
		}()
	}
}
<|endoftext|>


<commit_before>package clusterregistrationtokens

import (
	"fmt"
	"net/url"
	"strings"

	"github.com/rancher/norman/types"
	"github.com/rancher/norman/types/convert"
	util "github.com/rancher/rancher/pkg/cluster"
	v3 "github.com/rancher/rancher/pkg/generated/norman/management.cattle.io/v3"
	"github.com/rancher/rancher/pkg/image"
	"github.com/rancher/rancher/pkg/settings"
	"github.com/rancher/rancher/pkg/systemtemplate"
	"github.com/rancher/rancher/pkg/types/config"
	rketypes "github.com/rancher/rke/types"
)

const (
	commandFormat                 = "kubectl apply -f %s"
	insecureCommandFormat         = "curl --insecure -sfL %s | kubectl apply -f -"
	nodeCommandFormat             = "sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run %s %s --server %s --token %s%s"
	rke2NodeCommandFormat         = "curl -sfL %s | %s sh -s - --server %s --token %s%s"
	rke2InsecureNodeCommandFormat = "curl --insecure -sfL %s | %s sh -s - --server %s --token %s%s"
	loginCommandFormat            = "echo \"%s\" | sudo docker login --username %s --password-stdin %s"
	windowsNodeCommandFormat      = `PowerShell -NoLogo -NonInteractive -Command "& {docker run -v c:\:c:\host %s%s bootstrap --server %s --token %s%s%s | iex}"`
)

type Formatter struct {
	Clusters v3.ClusterLister
}

func NewFormatter(managementContext *config.ScaledContext) *Formatter {
	clusterFormatter := Formatter{
		Clusters: managementContext.Management.Clusters("").Controller().Lister(),
	}
	return &clusterFormatter
}

func (f *Formatter) isRKE2(clusterID string) bool {
	cluster, err := f.Clusters.Get("", clusterID)
	if err != nil {
		return false
	}
	return cluster.Annotations["objectset.rio.cattle.io/owner-gvk"] == "provisioning.cattle.io/v1, Kind=Cluster"
}

func (f *Formatter) Formatter(request *types.APIContext, resource *types.RawResource) {
	ca := systemtemplate.CAChecksum()
	if ca != "" {
		ca = " --ca-checksum " + ca
	}

	token, _ := resource.Values["token"].(string)
	clusterID := convert.ToString(resource.Values["clusterId"])
	if token != "" {
		url := getURL(request, token, clusterID)
		resource.Values["insecureCommand"] = fmt.Sprintf(insecureCommandFormat, url)
		resource.Values["command"] = fmt.Sprintf(commandFormat, url)
		resource.Values["token"] = token
		resource.Values["manifestUrl"] = url
		rootURL := getRootURL(request)

		cluster, _ := f.Clusters.Get("", clusterID)
		if cluster == nil {
			// We used to use a client, not lister which would always return non-nil even on error
			// this is to keep that behavior with listers.
			cluster = &v3.Cluster{}
		}

		agentImage := image.ResolveWithCluster(settings.AgentImage.Get(), cluster)
		if f.isRKE2(clusterID) {
			// for linux
			resource.Values["nodeCommand"] = fmt.Sprintf(rke2NodeCommandFormat,
				rootURL+"/system-agent-install.sh",
				AgentEnvVars(cluster),
				rootURL,
				token,
				ca)
			resource.Values["inseucreNodeCommand"] = fmt.Sprintf(rke2NodeCommandFormat,
				rootURL+"/system-agent-install.sh",
				AgentEnvVars(cluster),
				rootURL,
				token,
				ca)
		} else {
			// for linux
			resource.Values["nodeCommand"] = fmt.Sprintf(nodeCommandFormat,
				AgentEnvVars(cluster),
				agentImage,
				rootURL,
				token,
				ca)
		}
		// for windows
		var agentImageDockerEnv string
		if util.GetPrivateRepoURL(cluster) != "" {
			// patch the AGENT_IMAGE env
			agentImageDockerEnv = fmt.Sprintf("-e AGENT_IMAGE=%s ", agentImage)
		}
		resource.Values["windowsNodeCommand"] = fmt.Sprintf(windowsNodeCommandFormat,
			agentImageDockerEnv,
			agentImage,
			rootURL,
			token,
			ca,
			getWindowsPrefixPathArg(cluster.Spec.RancherKubernetesEngineConfig))
	}
}

func getWindowsPrefixPathArg(rkeConfig *rketypes.RancherKubernetesEngineConfig) string {
	if rkeConfig == nil {
		return ""
	}
	// default to prefix path
	prefixPath := rkeConfig.PrefixPath

	// if windows prefix path set, override
	if rkeConfig.WindowsPrefixPath != "" {
		prefixPath = rkeConfig.WindowsPrefixPath
	}

	if prefixPath != "" {
		return fmt.Sprintf(" --prefix-path %s", prefixPath)
	}

	return ""
}

func agentEnvVarsForShell(cluster *v3.Cluster) string {
	var agentEnvVars []string
	if cluster != nil {
		for _, envVar := range cluster.Spec.AgentEnvVars {
			if envVar.Value != "" {
				agentEnvVars = append(agentEnvVars, fmt.Sprintf("%s=\"%s\"", envVar.Name, envVar.Value))
			}
		}
	}
	return strings.Join(agentEnvVars, " ")
}

func AgentEnvVars(cluster *v3.Cluster) string {
	var agentEnvVars []string
	if cluster != nil {
		for _, envVar := range cluster.Spec.AgentEnvVars {
			if envVar.Value != "" {
				agentEnvVars = append(agentEnvVars, fmt.Sprintf("-e \"%s=%s\"", envVar.Name, envVar.Value))
			}
		}
	}
	return strings.Join(agentEnvVars, " ")
}

func NodeCommand(token string, cluster *v3.Cluster) string {
	ca := systemtemplate.CAChecksum()
	if ca != "" {
		ca = " --ca-checksum " + ca
	}

	return fmt.Sprintf(nodeCommandFormat,
		AgentEnvVars(cluster),
		image.ResolveWithCluster(settings.AgentImage.Get(), cluster),
		getRootURL(nil),
		token,
		ca)
}

func LoginCommand(reg rketypes.PrivateRegistry) string {
	return fmt.Sprintf(
		loginCommandFormat,
		// escape password special characters so it is interpreted correctly when command is executed
		escapeSpecialChars(reg.Password),
		reg.User,
		reg.URL,
	)
}

// escapeSpecialChars escapes ", `, $, \ from a string s
func escapeSpecialChars(s string) string {
	var escaped []rune
	for _, r := range s {
		switch r {
		case '"', '`', '$', '\\': // escape
			escaped = append(escaped, '\\', r)
		default: // no escape
			escaped = append(escaped, r)
		}
	}
	return string(escaped)
}

func getRootURL(request *types.APIContext) string {
	serverURL := settings.ServerURL.Get()
	if serverURL == "" {
		if request != nil {
			serverURL = request.URLBuilder.RelativeToRoot("")
		}
	} else {
		u, err := url.Parse(serverURL)
		if err != nil {
			if request != nil {
				serverURL = request.URLBuilder.RelativeToRoot("")
			}
		} else {
			u.Path = ""
			serverURL = u.String()
		}
	}

	return serverURL
}

func getURL(request *types.APIContext, token, clusterID string) string {
	path := "/v3/import/" + token + "_" + clusterID + ".yaml"
	serverURL := settings.ServerURL.Get()
	if serverURL == "" {
		serverURL = request.URLBuilder.RelativeToRoot(path)
	} else {
		u, err := url.Parse(serverURL)
		if err != nil {
			serverURL = request.URLBuilder.RelativeToRoot(path)
		} else {
			u.Path = path
			serverURL = u.String()
		}
	}

	return serverURL
}
<commit_msg>drop the silent flag from the curl command<commit_after>package clusterregistrationtokens

import (
	"fmt"
	"net/url"
	"strings"

	"github.com/rancher/norman/types"
	"github.com/rancher/norman/types/convert"
	util "github.com/rancher/rancher/pkg/cluster"
	v3 "github.com/rancher/rancher/pkg/generated/norman/management.cattle.io/v3"
	"github.com/rancher/rancher/pkg/image"
	"github.com/rancher/rancher/pkg/settings"
	"github.com/rancher/rancher/pkg/systemtemplate"
	"github.com/rancher/rancher/pkg/types/config"
	rketypes "github.com/rancher/rke/types"
)

const (
	commandFormat                 = "kubectl apply -f %s"
	insecureCommandFormat         = "curl --insecure -sfL %s | kubectl apply -f -"
	nodeCommandFormat             = "sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run %s %s --server %s --token %s%s"
	rke2NodeCommandFormat         = "curl -fL %s | %s sh -s - --server %s --token %s%s"
	rke2InsecureNodeCommandFormat = "curl --insecure -fL %s | %s sh -s - --server %s --token %s%s"
	loginCommandFormat            = "echo \"%s\" | sudo docker login --username %s --password-stdin %s"
	windowsNodeCommandFormat      = `PowerShell -NoLogo -NonInteractive -Command "& {docker run -v c:\:c:\host %s%s bootstrap --server %s --token %s%s%s | iex}"`
)

type Formatter struct {
	Clusters v3.ClusterLister
}

func NewFormatter(managementContext *config.ScaledContext) *Formatter {
	clusterFormatter := Formatter{
		Clusters: managementContext.Management.Clusters("").Controller().Lister(),
	}
	return &clusterFormatter
}

func (f *Formatter) isRKE2(clusterID string) bool {
	cluster, err := f.Clusters.Get("", clusterID)
	if err != nil {
		return false
	}
	return cluster.Annotations["objectset.rio.cattle.io/owner-gvk"] == "provisioning.cattle.io/v1, Kind=Cluster"
}

func (f *Formatter) Formatter(request *types.APIContext, resource *types.RawResource) {
	ca := systemtemplate.CAChecksum()
	if ca != "" {
		ca = " --ca-checksum " + ca
	}

	token, _ := resource.Values["token"].(string)
	clusterID := convert.ToString(resource.Values["clusterId"])
	if token != "" {
		url := getURL(request, token, clusterID)
		resource.Values["insecureCommand"] = fmt.Sprintf(insecureCommandFormat, url)
		resource.Values["command"] = fmt.Sprintf(commandFormat, url)
		resource.Values["token"] = token
		resource.Values["manifestUrl"] = url
		rootURL := getRootURL(request)

		cluster, _ := f.Clusters.Get("", clusterID)
		if cluster == nil {
			// We used to use a client, not lister which would always return non-nil even on error
			// this is to keep that behavior with listers.
			cluster = &v3.Cluster{}
		}

		agentImage := image.ResolveWithCluster(settings.AgentImage.Get(), cluster)
		if f.isRKE2(clusterID) {
			// for linux
			resource.Values["nodeCommand"] = fmt.Sprintf(rke2NodeCommandFormat,
				rootURL+"/system-agent-install.sh",
				AgentEnvVars(cluster),
				rootURL,
				token,
				ca)
			resource.Values["insecureNodeCommand"] = fmt.Sprintf(rke2InsecureNodeCommandFormat,
				rootURL+"/system-agent-install.sh",
				AgentEnvVars(cluster),
				rootURL,
				token,
				ca)
		} else {
			// for linux
			resource.Values["nodeCommand"] = fmt.Sprintf(nodeCommandFormat,
				AgentEnvVars(cluster),
				agentImage,
				rootURL,
				token,
				ca)
		}
		// for windows
		var agentImageDockerEnv string
		if util.GetPrivateRepoURL(cluster) != "" {
			// patch the AGENT_IMAGE env
			agentImageDockerEnv = fmt.Sprintf("-e AGENT_IMAGE=%s ", agentImage)
		}
		resource.Values["windowsNodeCommand"] = fmt.Sprintf(windowsNodeCommandFormat,
			agentImageDockerEnv,
			agentImage,
			rootURL,
			token,
			ca,
			getWindowsPrefixPathArg(cluster.Spec.RancherKubernetesEngineConfig))
	}
}

func getWindowsPrefixPathArg(rkeConfig *rketypes.RancherKubernetesEngineConfig) string {
	if rkeConfig == nil {
		return ""
	}
	// default to prefix path
	prefixPath := rkeConfig.PrefixPath

	// if windows prefix path set, override
	if rkeConfig.WindowsPrefixPath != "" {
		prefixPath = rkeConfig.WindowsPrefixPath
	}

	if prefixPath != "" {
		return fmt.Sprintf(" --prefix-path %s", prefixPath)
	}

	return ""
}

func agentEnvVarsForShell(cluster *v3.Cluster) string {
	var agentEnvVars []string
	if cluster != nil {
		for _, envVar := range cluster.Spec.AgentEnvVars {
			if envVar.Value != "" {
				agentEnvVars = append(agentEnvVars, fmt.Sprintf("%s=\"%s\"", envVar.Name, envVar.Value))
			}
		}
	}
	return strings.Join(agentEnvVars, " ")
}

func AgentEnvVars(cluster *v3.Cluster) string {
	var agentEnvVars []string
	if cluster != nil {
		for _, envVar := range cluster.Spec.AgentEnvVars {
			if envVar.Value != "" {
				agentEnvVars = append(agentEnvVars, fmt.Sprintf("-e \"%s=%s\"", envVar.Name, envVar.Value))
			}
		}
	}
	return strings.Join(agentEnvVars, " ")
}

func NodeCommand(token string, cluster *v3.Cluster) string {
	ca := systemtemplate.CAChecksum()
	if ca != "" {
		ca = " --ca-checksum " + ca
	}

	return fmt.Sprintf(nodeCommandFormat,
		AgentEnvVars(cluster),
		image.ResolveWithCluster(settings.AgentImage.Get(), cluster),
		getRootURL(nil),
		token,
		ca)
}

func LoginCommand(reg rketypes.PrivateRegistry) string {
	return fmt.Sprintf(
		loginCommandFormat,
		// escape password special characters so it is interpreted correctly when command is executed
		escapeSpecialChars(reg.Password),
		reg.User,
		reg.URL,
	)
}

// escapeSpecialChars escapes ", `, $, \ from a string s
func escapeSpecialChars(s string) string {
	var escaped []rune
	for _, r := range s {
		switch r {
		case '"', '`', '$', '\\': // escape
			escaped = append(escaped, '\\', r)
		default: // no escape
			escaped = append(escaped, r)
		}
	}
	return string(escaped)
}

func getRootURL(request *types.APIContext) string {
	serverURL := settings.ServerURL.Get()
	if serverURL == "" {
		if request != nil {
			serverURL = request.URLBuilder.RelativeToRoot("")
		}
	} else {
		u, err := url.Parse(serverURL)
		if err != nil {
			if request != nil {
				serverURL = request.URLBuilder.RelativeToRoot("")
			}
		} else {
			u.Path = ""
			serverURL = u.String()
		}
	}

	return serverURL
}

func getURL(request *types.APIContext, token, clusterID string) string {
	path := "/v3/import/" + token + "_" + clusterID + ".yaml"
	serverURL := settings.ServerURL.Get()
	if serverURL == "" {
		serverURL = request.URLBuilder.RelativeToRoot(path)
	} else {
		u, err := url.Parse(serverURL)
		if err != nil {
			serverURL = request.URLBuilder.RelativeToRoot(path)
		} else {
			u.Path = path
			serverURL = u.String()
		}
	}

	return serverURL
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"sync"

	"github.com/mitchellh/go-homedir"
	"github.com/nishanths/license/pkg/license"
)

func update() {
	err := doUpdate()
	if err != nil {
		errLogger.Println("failed to update licenses:", err)
		os.Exit(1)
	}
	os.Exit(0)
}

func prepareTempDir() (string, error) {
	tempRoot, err := ioutil.TempDir("", "license-")
	if err != nil {
		return "", err
	}

	data := path.Join(tempRoot, "data")
	tmpl := path.Join(data, "tmpl")

	if err := os.MkdirAll(tmpl, 0700); err != nil {
		os.RemoveAll(tempRoot)
		return "", err
	}

	return tempRoot, nil
}

func doUpdate() error {
	// Exit early if we cannot find home directory.

	home, err := homedir.Dir()
	if err != nil {
		return err
	}

	// Work in a temporary directory, then
	// move updated licenses to the home directory.

	tempRoot, err := prepareTempDir()
	if err != nil {
		return nil
	}
	defer os.RemoveAll(tempRoot)

	c := license.NewClient()
	c.ClientID = os.Getenv("GITHUB_CLIENT_ID")
	c.ClientSecret = os.Getenv("GITHUB_CLIENT_SECRET")

	// Write licenses.json.

	lics, err := c.List()
	if err != nil {
		return err
	}

	f, err := os.Create(filepath.Join(tempRoot, "data", "licenses.json"))
	if err != nil {
		return err
	}
	if err := json.NewEncoder(f).Encode(lics); err != nil {
		return err
	}

	wg := sync.WaitGroup{}
	errs := make([]error, len(lics))

	for i, l := range lics {
		i, l := i, l
		wg.Add(1)

		go func() {
			defer wg.Done()

			// Fetch full licenses.

			l, err := c.Info(l.Key)
			if err != nil {
				errs[i] = err
				return
			}

			// Write template file.

			content := textTemplate(l.Body)
			tmplFile, err := os.Create(filepath.Join(
				tempRoot, "data", "tmpl", strings.ToLower(l.Key)+".tmpl",
			))
			if err != nil {
				errs[i] = err
				return
			}
			if _, err := tmplFile.WriteString(content); err != nil {
				errs[i] = err
				return
			}
		}()
	}

	wg.Wait()
	for _, err := range errs {
		if err != nil {
			return err
		}
	}

	licensePath := path.Join(home, ".license")
	if err := os.RemoveAll(licensePath); err != nil {
		return err
	}
	return os.Rename(tempRoot, licensePath)
}

var (
	// placeholders is a map from placeholders in the JSON
	// to placeholders we use in templates.
	// The keys should be kept in sync with the regex below.
	placeholders = map[string]string{
		"year":     "Year",
		"fullname": "Name",
	}
	placeholdersRx = regexp.MustCompile("\\[(year|fullname)\\]")
)

func textTemplate(s string) string {
	return placeholdersRx.ReplaceAllStringFunc(s, func(m string) string {
		if s := placeholdersRx.FindStringSubmatch(m); len(s) > 0 {
			k := s[1]
			if v, ok := placeholders[k]; ok {
				return "{{." + v + "}}"
			}
		}
		return m
	})
}
<commit_msg>Handle HTTP errors specially<commit_after>package main

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"sync"

	"github.com/mitchellh/go-homedir"
	"github.com/nishanths/license/pkg/license"
)

func update() {
	err := doUpdate()
	if err != nil {
		errLogger.Println("failed to update licenses:", err)
		os.Exit(1)
	}
	os.Exit(0)
}

func prepareTempDir() (string, error) {
	tempRoot, err := ioutil.TempDir("", "license-")
	if err != nil {
		return "", err
	}

	data := path.Join(tempRoot, "data")
	tmpl := path.Join(data, "tmpl")

	if err := os.MkdirAll(tmpl, 0700); err != nil {
		os.RemoveAll(tempRoot)
		return "", err
	}

	return tempRoot, nil
}

func doUpdate() error {
	// Exit early if we cannot find home directory.

	home, err := homedir.Dir()
	if err != nil {
		return err
	}

	// Work in a temporary directory, then
	// move updated licenses to the home directory.

	tempRoot, err := prepareTempDir()
	if err != nil {
		return nil
	}
	defer os.RemoveAll(tempRoot)

	c := license.NewClient()
	c.ClientID = os.Getenv("GITHUB_CLIENT_ID")
	c.ClientSecret = os.Getenv("GITHUB_CLIENT_SECRET")

	// Write licenses.json.

	lics, err := c.List()
	if err != nil {
		return convertAPIError(err)
	}

	f, err := os.Create(filepath.Join(tempRoot, "data", "licenses.json"))
	if err != nil {
		return err
	}
	if err := json.NewEncoder(f).Encode(lics); err != nil {
		return err
	}

	wg := sync.WaitGroup{}
	errs := make([]error, len(lics))

	for i, l := range lics {
		i, l := i, l
		wg.Add(1)

		go func() {
			defer wg.Done()

			// Fetch full licenses.

			l, err := c.Info(l.Key)
			if err != nil {
				errs[i] = convertAPIError(err)
				return
			}

			// Write template file.

			content := textTemplate(l.Body)
			tmplFile, err := os.Create(filepath.Join(
				tempRoot, "data", "tmpl", strings.ToLower(l.Key)+".tmpl",
			))
			if err != nil {
				errs[i] = err
				return
			}
			if _, err := tmplFile.WriteString(content); err != nil {
				errs[i] = err
				return
			}
		}()
	}

	wg.Wait()
	for _, err := range errs {
		if err != nil {
			return err
		}
	}

	licensePath := path.Join(home, ".license")
	if err := os.RemoveAll(licensePath); err != nil {
		return err
	}
	return os.Rename(tempRoot, licensePath)
}

// convertAPIError handles HTTP errors as a special case
// because simply logging the error may print the client ID
// and client secret as part of the error message URL.
func convertAPIError(err error) error {
	switch err.(type) {
	case nil:
		return nil
	case license.StatusError:
		return err
	default:
		return errors.New("error: failed to fetch licenses")
	}
}

var (
	// placeholders is a map from placeholders in GitHub's JSON
	// to placeholders we use in templates.
	// The keys should be kept in sync with the regex below.
	placeholders = map[string]string{
		"year":     "Year",
		"fullname": "Name",
	}
	placeholdersRx = regexp.MustCompile("\\[(year|fullname)\\]")
)

func textTemplate(s string) string {
	return placeholdersRx.ReplaceAllStringFunc(s, func(m string) string {
		if s := placeholdersRx.FindStringSubmatch(m); len(s) > 0 {
			k := s[1]
			if v, ok := placeholders[k]; ok {
				return "{{." + v + "}}"
			}
		}
		return m
	})
}
<|endoftext|>