<commit_before>package common

import (
	"fmt"
	"io"
	"math/rand"
	"net"
	"regexp"
)

const (
	chars        = "abcdefghiklmnopqrstuvwxyz"
	subdomainLen = 4
)

// client request for a string, if already taken, get a new one. else
// use the one asked by client.
func newRandString() string {
	var str [subdomainLen]byte
	// rand.Seed(time.Now().Unix())
	for i := 0; i < subdomainLen; i++ {
		rnum := rand.Intn(len(chars))
		str[i] = chars[rnum]
	}
	return string(str[:])
}

// can be a Stringer interface.
type Proxy struct {
	id    string
	Proxy *ProxyClient
	Admin io.ReadWriteCloser
}

func (p *Proxy) Id() string {
	return p.id
}

func (p *Proxy) Host(addr string) string {
	if p.id != "" {
		return net.JoinHostPort(p.id+"."+addr, p.Proxy.Port())
	}
	return net.JoinHostPort(addr, p.Proxy.Port())
}
func (p *Proxy) Port() string {
	return p.Proxy.Port()
}

type TCPRouter struct {
	pool    *PortPool
	proxies map[string]*Proxy
}

func NewTCPRouter(cpools, cpoole int) *TCPRouter {
	return &TCPRouter{
		NewPortPool(cpools, cpoole),
		make(map[string]*Proxy),
	}
}

func IdForHost(host string) (string, bool) {
	h, _, err := net.SplitHostPort(host)
	if err != nil {
		return "", false
	}

	reg, err := regexp.Compile(`^([A-Za-z]*)`)
	if err != nil {
		return "", false
	}

	if reg.Match([]byte(host)) {
		log("Router: id for host", reg.FindString(h), host)
		return reg.FindString(host), true
	}
	return "", false
}
func HostForId(id string) string {
	return id
}

func (r *TCPRouter) setupClientCommChan(id string) *ProxyClient {
	port, ok := r.pool.GetAvailable()
	if !ok {
		fatal("Coudn't get a port for client communication")
	}

	proxy, err := NewProxyClient(port)
	if err != nil {
		fatal("Coulnd't setup client communication channel", err)
	}

	return proxy
}

func (r *TCPRouter) Register(ac net.Conn, suggestedId string) (proxy *Proxy) {
	// check if its suggestedId is already registered.
	proxyClient := r.setupClientCommChan(suggestedId)

	id := suggestedId
	for _, ok := r.proxies[id]; ok; _, ok = r.proxies[id] {
		id = newRandString()
	}

	log("Router: registering with (%s)", id)
	r.proxies[id] = &Proxy{Proxy: proxyClient, Admin: ac, id: id}

	return r.proxies[id]
}

func (r *TCPRouter) Deregister(id string) {
	delete(r.proxies, id)
}

func (r *TCPRouter) String() string {
	return fmt.Sprintf("Router: %v", r.proxies)
}

// given a connection, figures out the subdomain and gives respective
// proxy.
func (r *TCPRouter) GetProxy(host string) (*Proxy, bool) {
	id, ok := IdForHost(host)
	if !ok {
		log("Router: Couldn't find the subdomain for the request", host)
		return nil, false
	}

	log("Router: for id: ", id, r.String())
	if proxy, ok := r.proxies[id]; ok {
		log("Router: found proxy")
		return proxy, true
	}
	return nil, false
}
<commit_msg>fix bug with backend and frontend urls<commit_after>package common

import (
	"fmt"
	"io"
	"math/rand"
	"net"
	"regexp"
)

const (
	chars        = "abcdefghiklmnopqrstuvwxyz"
	subdomainLen = 4
)

// client request for a string, if already taken, get a new one. else
// use the one asked by client.
func newRandString() string {
	var str [subdomainLen]byte
	// rand.Seed(time.Now().Unix())
	for i := 0; i < subdomainLen; i++ {
		rnum := rand.Intn(len(chars))
		str[i] = chars[rnum]
	}
	return string(str[:])
}

// can be a Stringer interface.
type Proxy struct {
	id    string
	Proxy *ProxyClient
	Admin io.ReadWriteCloser
}

func (p *Proxy) Id() string {
	return p.id
}

func (p *Proxy) BackendHost(addr string) string {
	return net.JoinHostPort(addr, p.Proxy.Port())
}
func (p *Proxy) FrontHost(addr, port string) string {
	if p.id != "" {
		return net.JoinHostPort(p.id+"."+addr, port)
	}
	return net.JoinHostPort(addr, port)
}
func (p *Proxy) Port() string {
	return p.Proxy.Port()
}

type TCPRouter struct {
	pool    *PortPool
	proxies map[string]*Proxy
}

func NewTCPRouter(cpools, cpoole int) *TCPRouter {
	return &TCPRouter{
		NewPortPool(cpools, cpoole),
		make(map[string]*Proxy),
	}
}

func IdForHost(host string) (string, bool) {
	h, _, err := net.SplitHostPort(host)
	if err != nil {
		return "", false
	}

	reg, err := regexp.Compile(`^([A-Za-z]*)`)
	if err != nil {
		return "", false
	}

	if reg.Match([]byte(host)) {
		log("Router: id for host", reg.FindString(h), host)
		return reg.FindString(host), true
	}
	return "", false
}
func HostForId(id string) string {
	return id
}

func (r *TCPRouter) setupClientCommChan(id string) *ProxyClient {
	port, ok := r.pool.GetAvailable()
	if !ok {
		fatal("Coudn't get a port for client communication")
	}

	proxy, err := NewProxyClient(port)
	if err != nil {
		fatal("Coulnd't setup client communication channel", err)
	}

	return proxy
}

func (r *TCPRouter) Register(ac net.Conn, suggestedId string) (proxy *Proxy) {
	// check if its suggestedId is already registered.
	proxyClient := r.setupClientCommChan(suggestedId)

	id := suggestedId
	for _, ok := r.proxies[id]; ok; _, ok = r.proxies[id] {
		id = newRandString()
	}

	log("Router: registering with (%s)", id)
	r.proxies[id] = &Proxy{Proxy: proxyClient, Admin: ac, id: id}

	return r.proxies[id]
}

func (r *TCPRouter) Deregister(id string) {
	delete(r.proxies, id)
}

func (r *TCPRouter) String() string {
	return fmt.Sprintf("Router: %v", r.proxies)
}

// given a connection, figures out the subdomain and gives respective
// proxy.
func (r *TCPRouter) GetProxy(host string) (*Proxy, bool) {
	id, ok := IdForHost(host)
	if !ok {
		log("Router: Couldn't find the subdomain for the request", host)
		return nil, false
	}

	log("Router: for id: ", id, r.String())
	if proxy, ok := r.proxies[id]; ok {
		log("Router: found proxy")
		return proxy, true
	}
	return nil, false
}
<|endoftext|>


<commit_before>package common

const (
	MaxEditctBits = 11
	IndexMask     = ((1 << MaxEditctBits) - 1)
)

const weaponPrefix = "weapon_"

type (
	RoundMVPReason   byte
	Hitgroup         byte
	RoundEndReason   byte
	Team             byte
	EquipmentElement int
	EquipmentClass   int
)

const (
	MVPReason_MostEliminations RoundMVPReason = iota + 1
	MVPReason_BombDefused
	MVPReason_BombPlanted
)

const (
	HG_Generic  Hitgroup = 0
	HG_Head     Hitgroup = 1
	HG_Chest    Hitgroup = 2
	HG_Stomach  Hitgroup = 3
	HG_LeftArm  Hitgroup = 4
	HG_RightArm Hitgroup = 5
	HG_LeftLeg  Hitgroup = 6
	HG_RightLeg Hitgroup = 7
	HG_Gear     Hitgroup = 10
)

const (
	RER_TargetBombed RoundEndReason = iota + 1
	RER_VIPEscaped
	RER_VIPKilled
	RER_TerroristsEscaped
	RER_CTStoppedEscape
	RER_TerroristsStopped
	RER_BombDefused
	RER_CTWin
	RER_TerrprostWin
	RER_Draw
	RER_HostagesRescued
	RER_TargetSaved
	RER_HostagesNotRescued
	RER_TerroristsNotEscaped
	RER_VIPNotEscaped
	RER_GameStart
	RER_TerroristsSurrender
	RER_CTSurrender
)

const (
	Team_Unassigned Team = iota
	Team_Spectators
	Team_Terrorists
	Team_CounterTerrorists
)

const (
	EE_Unknown EquipmentElement = 0

	// Pistols

	EE_P2000        EquipmentElement = 1
	EE_Glock        EquipmentElement = 2
	EE_P250         EquipmentElement = 3
	EE_Deagle       EquipmentElement = 4
	EE_FiveSeven    EquipmentElement = 5
	EE_DualBarettas EquipmentElement = 6
	EE_Tec9         EquipmentElement = 7
	EE_CZ           EquipmentElement = 8
	EE_USP          EquipmentElement = 9
	EE_Revolver     EquipmentElement = 10

	// SMGs

	EE_MP7   EquipmentElement = 101
	EE_MP9   EquipmentElement = 102
	EE_Bizon EquipmentElement = 103
	EE_Mac10 EquipmentElement = 104
	EE_UMP   EquipmentElement = 105
	EE_P90   EquipmentElement = 106

	// Heavy

	EE_SawedOff EquipmentElement = 201
	EE_Nova     EquipmentElement = 202
	EE_Swag7    EquipmentElement = 203
	EE_XM1014   EquipmentElement = 204
	EE_M249     EquipmentElement = 205
	EE_Negev    EquipmentElement = 206

	// Rifles

	EE_Gallil EquipmentElement = 301
	EE_Famas  EquipmentElement = 302
	EE_AK47   EquipmentElement = 303
	EE_M4A4   EquipmentElement = 304
	EE_M4A1   EquipmentElement = 305
	EE_Scout  EquipmentElement = 306
	EE_SG556  EquipmentElement = 307
	EE_AUG    EquipmentElement = 308
	EE_AWP    EquipmentElement = 309
	EE_Scar20 EquipmentElement = 310
	EE_G3SG1  EquipmentElement = 311

	// Equipment

	EE_Zeus      EquipmentElement = 401
	EE_Kevlar    EquipmentElement = 402
	EE_Helmet    EquipmentElement = 403
	EE_Bomb      EquipmentElement = 404
	EE_Knife     EquipmentElement = 405
	EE_DefuseKit EquipmentElement = 406
	EE_World     EquipmentElement = 407

	// Grenades

	EE_Decoy      EquipmentElement = 501
	EE_Molotov    EquipmentElement = 502
	EE_Incendiary EquipmentElement = 503
	EE_Flash      EquipmentElement = 504
	EE_Smoke      EquipmentElement = 505
	EE_HE         EquipmentElement = 506
)

const (
	EC_Unknown EquipmentClass = iota
	EC_Pistols
	EC_SMG
	EC_Heavy
	EC_Rifle
	EC_Equipment
	EC_Grenade
)
<commit_msg>Typo and consistent plural vs singular<commit_after>package common

const (
	MaxEditctBits = 11
	IndexMask     = ((1 << MaxEditctBits) - 1)
)

const weaponPrefix = "weapon_"

type (
	RoundMVPReason   byte
	Hitgroup         byte
	RoundEndReason   byte
	Team             byte
	EquipmentElement int
	EquipmentClass   int
)

const (
	MVPReason_MostEliminations RoundMVPReason = iota + 1
	MVPReason_BombDefused
	MVPReason_BombPlanted
)

const (
	HG_Generic  Hitgroup = 0
	HG_Head     Hitgroup = 1
	HG_Chest    Hitgroup = 2
	HG_Stomach  Hitgroup = 3
	HG_LeftArm  Hitgroup = 4
	HG_RightArm Hitgroup = 5
	HG_LeftLeg  Hitgroup = 6
	HG_RightLeg Hitgroup = 7
	HG_Gear     Hitgroup = 10
)

const (
	RER_TargetBombed RoundEndReason = iota + 1
	RER_VIPEscaped
	RER_VIPKilled
	RER_TerroristsEscaped
	RER_CTStoppedEscape
	RER_TerroristsStopped
	RER_BombDefused
	RER_CTWin
	RER_TerroristsWin
	RER_Draw
	RER_HostagesRescued
	RER_TargetSaved
	RER_HostagesNotRescued
	RER_TerroristsNotEscaped
	RER_VIPNotEscaped
	RER_GameStart
	RER_TerroristsSurrender
	RER_CTSurrender
)

const (
	Team_Unassigned Team = iota
	Team_Spectators
	Team_Terrorists
	Team_CounterTerrorists
)

const (
	EE_Unknown EquipmentElement = 0

	// Pistols

	EE_P2000        EquipmentElement = 1
	EE_Glock        EquipmentElement = 2
	EE_P250         EquipmentElement = 3
	EE_Deagle       EquipmentElement = 4
	EE_FiveSeven    EquipmentElement = 5
	EE_DualBarettas EquipmentElement = 6
	EE_Tec9         EquipmentElement = 7
	EE_CZ           EquipmentElement = 8
	EE_USP          EquipmentElement = 9
	EE_Revolver     EquipmentElement = 10

	// SMGs

	EE_MP7   EquipmentElement = 101
	EE_MP9   EquipmentElement = 102
	EE_Bizon EquipmentElement = 103
	EE_Mac10 EquipmentElement = 104
	EE_UMP   EquipmentElement = 105
	EE_P90   EquipmentElement = 106

	// Heavy

	EE_SawedOff EquipmentElement = 201
	EE_Nova     EquipmentElement = 202
	EE_Swag7    EquipmentElement = 203
	EE_XM1014   EquipmentElement = 204
	EE_M249     EquipmentElement = 205
	EE_Negev    EquipmentElement = 206

	// Rifles

	EE_Gallil EquipmentElement = 301
	EE_Famas  EquipmentElement = 302
	EE_AK47   EquipmentElement = 303
	EE_M4A4   EquipmentElement = 304
	EE_M4A1   EquipmentElement = 305
	EE_Scout  EquipmentElement = 306
	EE_SG556  EquipmentElement = 307
	EE_AUG    EquipmentElement = 308
	EE_AWP    EquipmentElement = 309
	EE_Scar20 EquipmentElement = 310
	EE_G3SG1  EquipmentElement = 311

	// Equipment

	EE_Zeus      EquipmentElement = 401
	EE_Kevlar    EquipmentElement = 402
	EE_Helmet    EquipmentElement = 403
	EE_Bomb      EquipmentElement = 404
	EE_Knife     EquipmentElement = 405
	EE_DefuseKit EquipmentElement = 406
	EE_World     EquipmentElement = 407

	// Grenades

	EE_Decoy      EquipmentElement = 501
	EE_Molotov    EquipmentElement = 502
	EE_Incendiary EquipmentElement = 503
	EE_Flash      EquipmentElement = 504
	EE_Smoke      EquipmentElement = 505
	EE_HE         EquipmentElement = 506
)

const (
	EC_Unknown EquipmentClass = iota
	EC_Pistols
	EC_SMG
	EC_Heavy
	EC_Rifle
	EC_Equipment
	EC_Grenade
)
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

type SimpleChaincode struct {
}

func main() {
	err := shim.Start(new(SimpleChaincode))

	if err != nil {
		//do nothing
	}
}

func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	// added by chenlin@20170308
	if len(args) != 0 {
		return nil, errors.New("incorrect args")
	}
	// adminCert, err := stub.GetCallerMetadata()
	// if err != nil {
	// 	return nil, errors.New("failed getting metadata")
	// }
	// if len(adminCert) == 0 {
	// 	return nil, errors.New("invalid admin certificate. Empty")
	// }
	stub.PutState("admin", []byte("hahahahaha"))
	return nil, nil
}

func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	switch function {
	case "putState":
		if len(args) != 2 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		value := []byte(args[1])
		err := stub.PutState(key, value)
		if err != nil {
			return nil, err
		}
		return value, err
	case "delState":
		if len(args) != 1 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		err := stub.DelState(key)
		if err != nil {
			return nil, err
		}
		result := key + "has been deleted"
		return []byte(result), err

	case "createTable":
		err := stub.CreateTable("AssetsOwnership", []*shim.ColumnDefinition{
			&shim.ColumnDefinition{Name: "Asset", Type: shim.ColumnDefinition_STRING, Key: true},
			&shim.ColumnDefinition{Name: "Owner", Type: shim.ColumnDefinition_BYTES, Key: false},
		})
		if err != nil {
			return nil, errors.New("Failed creating AssetsOnwership table.")
		}
		return nil, nil

	case "insertTable":
		if len(args) != 2 {
			return nil, errors.New("incorrect args")
		}
		asset := args[0]
		owner := []byte(args[1])
		row := shim.Row{
			Columns: []*shim.Column{
				&shim.Column{Value: &shim.Column_String_{String_: asset}},
				&shim.Column{Value: &shim.Column_Bytes{Bytes: owner}}}}
		ok, err := stub.InsertRow("AssetsOwnership", row)
		if !ok && err == nil {
			return nil, errors.New("asset was already assigned")
		}
		return nil, nil

	}
	return nil, nil
}

func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	switch function {
	case "getTxID":
		txID := stub.GetTxID()
		result := []byte(txID)
		return result, nil
	case "getTxTimestamp":
		time, err := stub.GetTxTimestamp()
		result := []byte(time.String()) //时间转换为字符串，time.String()
		return result, err
	case "getStringArgs":
		strList := stub.GetStringArgs()
		var result string
		for index := 0; index < len(strList); index++ {
			result += "***" + strList[index] + "***"
		}
		return []byte(result), nil

	case "getState":
		if len(args) != 1 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		result, err := stub.GetState(key)

		if err != nil {
			return nil, errors.New("Failed in function getState")
		}
		return result, err

	case "getCallerCert":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetCallerCertificate()
		if err != nil {
			return nil, errors.New("Failed in function getCallerCert")
		}
		return result, err

	case "getCallerMetadata":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetCallerMetadata()
		if err != nil {
			return nil, errors.New("Failed in function getCallerMetadata")
		}
		fmt.Println(result)
		return result, err
		// return []byte("getCallerMetadata"), err
	case "getBinding":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetBinding()

		if err != nil {
			return nil, errors.New("Failed in function getBinding")
		}
		fmt.Println(result)
		return result, err
		// return []byte("getBinding"), err
	case "getPayload":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetPayload()

		if err != nil {
			return nil, errors.New("Failed in function getPayload")
		}
		fmt.Println(result)
		return result, err
	// return []byte("getPayload"), err
	case "getRow":
		if len(args) != 1 {
			return nil, errors.New("incorrect args")
		}
		asset := args[0]
		var columns []shim.Column
		col1 := shim.Column{Value: &shim.Column_String_{String_: asset}}
		columns = append(columns, col1)

		row, err := stub.GetRow("AssetsOwnership", columns)
		if err != nil {
			return nil, errors.New("failed in function getRow")
		}

		val0 := row.Columns[0].GetString_()
		val1 := row.Columns[1].GetString_()
		result := "****" + val0 + "****" + val1
		return []byte(result), err
		//end of "getRow"
	}

	return nil, nil
}
<commit_msg>add code to init()<commit_after>package main

import (
	"errors"
	"fmt"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

type SimpleChaincode struct {
}

func main() {
	err := shim.Start(new(SimpleChaincode))

	if err != nil {
		//do nothing
	}
}

func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	// added by chenlin@20170308
	if len(args) != 0 {
		return nil, errors.New("incorrect args")
	}
	// adminCert, err := stub.GetCallerMetadata()
	// if err != nil {
	// 	return nil, errors.New("failed getting metadata")
	// }
	// if len(adminCert) == 0 {
	// 	return nil, errors.New("invalid admin certificate. Empty")
	// }
	stub.PutState("admin", []byte("hahahahaha"))
	err := stub.CreateTable("AssetsOwnership", []*shim.ColumnDefinition{
		&shim.ColumnDefinition{Name: "Asset", Type: shim.ColumnDefinition_STRING, Key: true},
		&shim.ColumnDefinition{Name: "Owner", Type: shim.ColumnDefinition_BYTES, Key: false},
	})
	if err != nil {
		return nil, errors.New("Failed creating AssetsOnwership table.")
	}
	return nil, nil
}

func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	switch function {
	case "putState":
		if len(args) != 2 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		value := []byte(args[1])
		err := stub.PutState(key, value)
		if err != nil {
			return nil, err
		}
		return value, err
	case "delState":
		if len(args) != 1 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		err := stub.DelState(key)
		if err != nil {
			return nil, err
		}
		result := key + "has been deleted"
		return []byte(result), err

	case "createTable":
		err := stub.CreateTable("AssetsOwnership", []*shim.ColumnDefinition{
			&shim.ColumnDefinition{Name: "Asset", Type: shim.ColumnDefinition_STRING, Key: true},
			&shim.ColumnDefinition{Name: "Owner", Type: shim.ColumnDefinition_BYTES, Key: false},
		})
		if err != nil {
			return nil, errors.New("Failed creating AssetsOnwership table.")
		}
		return nil, nil

	case "insertTable":
		if len(args) != 2 {
			return nil, errors.New("incorrect args")
		}
		asset := args[0]
		owner := []byte(args[1])
		row := shim.Row{
			Columns: []*shim.Column{
				&shim.Column{Value: &shim.Column_String_{String_: asset}},
				&shim.Column{Value: &shim.Column_Bytes{Bytes: owner}}}}
		ok, err := stub.InsertRow("AssetsOwnership", row)
		if !ok && err == nil {
			return nil, errors.New("asset was already assigned")
		}
		return nil, nil

	}
	return nil, nil
}

func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	switch function {
	case "getTxID":
		txID := stub.GetTxID()
		result := []byte(txID)
		return result, nil
	case "getTxTimestamp":
		time, err := stub.GetTxTimestamp()
		result := []byte(time.String()) //时间转换为字符串，time.String()
		return result, err
	case "getStringArgs":
		strList := stub.GetStringArgs()
		var result string
		for index := 0; index < len(strList); index++ {
			result += "***" + strList[index] + "***"
		}
		return []byte(result), nil

	case "getState":
		if len(args) != 1 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		result, err := stub.GetState(key)

		if err != nil {
			return nil, errors.New("Failed in function getState")
		}
		return result, err

	case "getCallerCert":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetCallerCertificate()
		if err != nil {
			return nil, errors.New("Failed in function getCallerCert")
		}
		return result, err

	case "getCallerMetadata":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetCallerMetadata()
		if err != nil {
			return nil, errors.New("Failed in function getCallerMetadata")
		}
		fmt.Println(result)
		return result, err
		// return []byte("getCallerMetadata"), err
	case "getBinding":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetBinding()

		if err != nil {
			return nil, errors.New("Failed in function getBinding")
		}
		fmt.Println(result)
		return result, err
		// return []byte("getBinding"), err
	case "getPayload":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetPayload()

		if err != nil {
			return nil, errors.New("Failed in function getPayload")
		}
		fmt.Println(result)
		return result, err
	// return []byte("getPayload"), err
	case "getRow":
		if len(args) != 1 {
			return nil, errors.New("incorrect args")
		}
		asset := args[0]
		var columns []shim.Column
		col1 := shim.Column{Value: &shim.Column_String_{String_: asset}}
		columns = append(columns, col1)

		row, err := stub.GetRow("AssetsOwnership", columns)
		if err != nil {
			return nil, errors.New("failed in function getRow")
		}

		val0 := row.Columns[0].GetString_()
		val1 := row.Columns[1].GetString_()
		result := "****" + val0 + "****" + val1
		return []byte(result), err
		//end of "getRow"
	}

	return nil, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pubsub

import (
	"fmt"
	"math/rand"
	"reflect"
	"sync"
	"testing"
	"time"

	"golang.org/x/net/context"

	"cloud.google.com/go/internal/testutil"
	"google.golang.org/api/option"
)

const timeout = time.Minute * 10
const ackDeadline = time.Second * 10

const batchSize = 100
const batches = 100

// messageCounter keeps track of how many times a given message has been received.
type messageCounter struct {
	mu     sync.Mutex
	counts map[string]int
	// A value is sent to recv each time Inc is called.
	recv chan struct{}
}

func (mc *messageCounter) Inc(msgID string) {
	mc.mu.Lock()
	mc.counts[msgID] += 1
	mc.mu.Unlock()
	mc.recv <- struct{}{}
}

// process handles a message and records it in mc.
func process(t *testing.T, ctx context.Context, m *Message, mc *messageCounter) {
	mc.Inc(m.ID)
	// Simulate time taken to process m, while continuing to process more messages.
	go func() {
		// Some messages will need to have their ack deadline extended due to this delay.
		delay := rand.Intn(int(ackDeadline * 3))
		time.After(time.Duration(delay))
		m.Ack()
	}()
}

// iteratorLifetime controls how long iterators live for before they are stopped.
type iteratorLifetimes interface {
	// lifetimeChan should be called when an iterator is started. The
	// returned channel will send when the iterator should be stopped.
	lifetimeChan() <-chan time.Time
}

var immortal = &explicitLifetimes{}

// explicitLifetimes implements iteratorLifetime with hard-coded lifetimes, falling back
// to indefinite lifetimes when no explicit lifetimes remain.
type explicitLifetimes struct {
	mu        sync.Mutex
	lifetimes []time.Duration
}

func (el *explicitLifetimes) lifetimeChan() <-chan time.Time {
	el.mu.Lock()
	defer el.mu.Unlock()
	if len(el.lifetimes) == 0 {
		return nil
	}
	lifetime := el.lifetimes[0]
	el.lifetimes = el.lifetimes[1:]
	return time.After(lifetime)
}

// consumer consumes messages according to its configuration.
type consumer struct {
	lifetimes iteratorLifetimes
}

// consume reads messages from a subscription, and keeps track of what it receives in mc.
// After consume returns, the caller should wait on wg to ensure that no more updates to mc will be made.
func (c *consumer) consume(t *testing.T, ctx context.Context, sub *Subscription, mc *messageCounter, wg *sync.WaitGroup) {
	wg.Add(1)
	go func() {
		defer wg.Done()
		for {
			ctx2, cancel := context.WithCancel(ctx)
			defer cancel()
			go func() {
				select {
				case <-c.lifetimes.lifetimeChan():
					cancel()
				case <-ctx2.Done():
				}
			}()
			err := sub.Receive(ctx2, func(ctx3 context.Context, m *Message) {
				process(t, ctx3, m, mc)
			})
			if err != nil {
				t.Fatalf("error from Receive: %v", err)
			}
			select {
			case <-ctx.Done():
				return
			default:
			}
		}
	}()
}

// publish publishes many messages to topic, and returns the published message ids.
func publish(t *testing.T, ctx context.Context, topic *Topic) []string {
	var published []string
	msgs := make([]*Message, batchSize)
	for i := 0; i < batches; i++ {
		for j := 0; j < batchSize; j++ {
			text := fmt.Sprintf("msg %02d-%02d", i, j)
			msgs[j] = &Message{Data: []byte(text)}
		}
		var rs []*PublishResult
		for _, m := range msgs {
			rs = append(rs, topic.Publish(ctx, m))
		}
		for _, r := range rs {
			id, err := r.Get(ctx)
			if err != nil {
				t.Errorf("Publish error: %v", err)
			}
			published = append(published, id)
		}
	}
	return published
}

// diff returns counts of the differences between got and want.
func diff(got, want map[string]int) map[string]int {
	ids := make(map[string]struct{})
	for k := range got {
		ids[k] = struct{}{}
	}
	for k := range want {
		ids[k] = struct{}{}
	}

	gotWantCount := make(map[string]int)
	for k := range ids {
		if got[k] == want[k] {
			continue
		}
		desc := fmt.Sprintf("<got: %v ; want: %v>", got[k], want[k])
		gotWantCount[desc] += 1
	}
	return gotWantCount
}

// TestEndToEnd pumps many messages into a topic and tests that they are all delivered to each subscription for the topic.
// It also tests that messages are not unexpectedly redelivered.
func TestEndToEnd(t *testing.T) {
	if testing.Short() {
		t.Skip("Integration tests skipped in short mode")
	}
	ctx := context.Background()
	ts := testutil.TokenSource(ctx, ScopePubSub, ScopeCloudPlatform)
	if ts == nil {
		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
	}

	now := time.Now()
	topicName := fmt.Sprintf("endtoend-%d", now.Unix())
	subPrefix := fmt.Sprintf("endtoend-%d", now.Unix())

	client, err := NewClient(ctx, testutil.ProjID(), option.WithTokenSource(ts))
	if err != nil {
		t.Fatalf("Creating client error: %v", err)
	}

	var topic *Topic
	if topic, err = client.CreateTopic(ctx, topicName); err != nil {
		t.Fatalf("CreateTopic error: %v", err)
	}
	defer topic.Delete(ctx)

	// Three subscriptions to the same topic.
	var subA, subB, subC *Subscription
	if subA, err = client.CreateSubscription(ctx, subPrefix+"-a", topic, ackDeadline, nil); err != nil {
		t.Fatalf("CreateSub error: %v", err)
	}
	defer subA.Delete(ctx)

	if subB, err = client.CreateSubscription(ctx, subPrefix+"-b", topic, ackDeadline, nil); err != nil {
		t.Fatalf("CreateSub error: %v", err)
	}
	defer subB.Delete(ctx)

	if subC, err = client.CreateSubscription(ctx, subPrefix+"-c", topic, ackDeadline, nil); err != nil {
		t.Fatalf("CreateSub error: %v", err)
	}
	defer subC.Delete(ctx)

	expectedCounts := make(map[string]int)
	for _, id := range publish(t, ctx, topic) {
		expectedCounts[id] = 1
	}

	// recv provides an indication that messages are still arriving.
	recv := make(chan struct{})

	// Keep track of the number of times each message (by message id) was
	// seen from each subscription.
	mcA := &messageCounter{counts: make(map[string]int), recv: recv}
	mcB := &messageCounter{counts: make(map[string]int), recv: recv}
	mcC := &messageCounter{counts: make(map[string]int), recv: recv}

	// We have three subscriptions to our topic.
	// Each subscription will get a copy of each pulished message.
	//
	// subA has just one iterator, while subB has two. The subB iterators
	// will each process roughly half of the messages for subB. All of
	// these iterators live until all messages have been consumed.  subC is
	// processed by a series of short-lived iterators.

	var wg sync.WaitGroup

	cctx, cancel := context.WithCancel(ctx)
	defer cancel()

	con := &consumer{
		lifetimes: immortal,
	}
	con.consume(t, cctx, subA, mcA, &wg)

	con = &consumer{
		lifetimes: immortal,
	}
	con.consume(t, cctx, subB, mcB, &wg)

	con = &consumer{
		lifetimes: &explicitLifetimes{
			lifetimes: []time.Duration{ackDeadline, ackDeadline, ackDeadline / 2, ackDeadline / 2},
		},
	}
	con.consume(t, cctx, subC, mcC, &wg)

	timeoutC := time.After(timeout)
	// Every time this ticker ticks, we will check if we have received any
	// messages since the last time it ticked.  We check less frequently
	// than the ack deadline, so that we can detect if messages are
	// redelivered after having their ack deadline extended.
	checkQuiescence := time.NewTicker(ackDeadline * 3)
	defer checkQuiescence.Stop()

	var received bool
	for {
		select {
		case <-recv:
			received = true
		case <-checkQuiescence.C:
			if received {
				received = false
			} else {
				cancel()
				return
			}
		case <-timeoutC:
			t.Errorf("timed out")
			cancel()
			return
		}
	}

	wg.Wait()
	for i, mc := range []*messageCounter{mcA, mcB, mcC} {
		if got, want := mc.counts, expectedCounts; !reflect.DeepEqual(got, want) {
			t.Errorf("%d: message counts: %v\n", i, diff(got, want))
		}
	}
}
<commit_msg>pubsub: fix end-to-end test<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pubsub

import (
	"fmt"
	"math/rand"
	"reflect"
	"sync"
	"testing"
	"time"

	"golang.org/x/net/context"

	"cloud.google.com/go/internal/testutil"
	"google.golang.org/api/option"
)

const timeout = time.Minute * 10
const ackDeadline = time.Second * 10

const batchSize = 100
const batches = 100

// messageCounter keeps track of how many times a given message has been received.
type messageCounter struct {
	mu     sync.Mutex
	counts map[string]int
	// A value is sent to recv each time Inc is called.
	recv chan struct{}
}

func (mc *messageCounter) Inc(msgID string) {
	mc.mu.Lock()
	mc.counts[msgID] += 1
	mc.mu.Unlock()
	mc.recv <- struct{}{}
}

// process handles a message and records it in mc.
func process(t *testing.T, ctx context.Context, m *Message, mc *messageCounter) {
	mc.Inc(m.ID)
	// Simulate time taken to process m, while continuing to process more messages.
	go func() {
		// Some messages will need to have their ack deadline extended due to this delay.
		delay := rand.Intn(int(ackDeadline * 3))
		time.After(time.Duration(delay))
		m.Ack()
	}()
}

// iteratorLifetime controls how long iterators live for before they are stopped.
type iteratorLifetimes interface {
	// lifetimeChan should be called when an iterator is started. The
	// returned channel will send when the iterator should be stopped.
	lifetimeChan() <-chan time.Time
}

var immortal = &explicitLifetimes{}

// explicitLifetimes implements iteratorLifetime with hard-coded lifetimes, falling back
// to indefinite lifetimes when no explicit lifetimes remain.
type explicitLifetimes struct {
	mu        sync.Mutex
	lifetimes []time.Duration
}

func (el *explicitLifetimes) lifetimeChan() <-chan time.Time {
	el.mu.Lock()
	defer el.mu.Unlock()
	if len(el.lifetimes) == 0 {
		return nil
	}
	lifetime := el.lifetimes[0]
	el.lifetimes = el.lifetimes[1:]
	return time.After(lifetime)
}

// consumer consumes messages according to its configuration.
type consumer struct {
	lifetimes iteratorLifetimes
}

// consume reads messages from a subscription, and keeps track of what it receives in mc.
// After consume returns, the caller should wait on wg to ensure that no more updates to mc will be made.
func (c *consumer) consume(t *testing.T, ctx context.Context, sub *Subscription, mc *messageCounter, wg *sync.WaitGroup) {
	wg.Add(1)
	go func() {
		defer wg.Done()
		for {
			ctx2, cancel := context.WithCancel(ctx)
			defer cancel()
			go func() {
				select {
				case <-c.lifetimes.lifetimeChan():
					cancel()
				case <-ctx2.Done():
				}
			}()
			err := sub.Receive(ctx2, func(ctx3 context.Context, m *Message) {
				process(t, ctx3, m, mc)
			})
			if err != nil {
				t.Fatalf("error from Receive: %v", err)
			}
			select {
			case <-ctx.Done():
				return
			default:
			}
		}
	}()
}

// publish publishes many messages to topic, and returns the published message ids.
func publish(t *testing.T, ctx context.Context, topic *Topic) []string {
	var published []string
	msgs := make([]*Message, batchSize)
	for i := 0; i < batches; i++ {
		for j := 0; j < batchSize; j++ {
			text := fmt.Sprintf("msg %02d-%02d", i, j)
			msgs[j] = &Message{Data: []byte(text)}
		}
		var rs []*PublishResult
		for _, m := range msgs {
			rs = append(rs, topic.Publish(ctx, m))
		}
		for _, r := range rs {
			id, err := r.Get(ctx)
			if err != nil {
				t.Errorf("Publish error: %v", err)
			}
			published = append(published, id)
		}
	}
	return published
}

// diff returns counts of the differences between got and want.
func diff(got, want map[string]int) map[string]int {
	ids := make(map[string]struct{})
	for k := range got {
		ids[k] = struct{}{}
	}
	for k := range want {
		ids[k] = struct{}{}
	}

	gotWantCount := make(map[string]int)
	for k := range ids {
		if got[k] == want[k] {
			continue
		}
		desc := fmt.Sprintf("<got: %v ; want: %v>", got[k], want[k])
		gotWantCount[desc] += 1
	}
	return gotWantCount
}

// TestEndToEnd pumps many messages into a topic and tests that they are all delivered to each subscription for the topic.
// It also tests that messages are not unexpectedly redelivered.
func TestEndToEnd(t *testing.T) {
	if testing.Short() {
		t.Skip("Integration tests skipped in short mode")
	}
	ctx := context.Background()
	ts := testutil.TokenSource(ctx, ScopePubSub, ScopeCloudPlatform)
	if ts == nil {
		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
	}

	now := time.Now()
	topicName := fmt.Sprintf("endtoend-%d", now.Unix())
	subPrefix := fmt.Sprintf("endtoend-%d", now.Unix())

	client, err := NewClient(ctx, testutil.ProjID(), option.WithTokenSource(ts))
	if err != nil {
		t.Fatalf("Creating client error: %v", err)
	}

	var topic *Topic
	if topic, err = client.CreateTopic(ctx, topicName); err != nil {
		t.Fatalf("CreateTopic error: %v", err)
	}
	defer topic.Delete(ctx)

	// Three subscriptions to the same topic.
	var subA, subB, subC *Subscription
	if subA, err = client.CreateSubscription(ctx, subPrefix+"-a", topic, ackDeadline, nil); err != nil {
		t.Fatalf("CreateSub error: %v", err)
	}
	defer subA.Delete(ctx)

	if subB, err = client.CreateSubscription(ctx, subPrefix+"-b", topic, ackDeadline, nil); err != nil {
		t.Fatalf("CreateSub error: %v", err)
	}
	defer subB.Delete(ctx)

	if subC, err = client.CreateSubscription(ctx, subPrefix+"-c", topic, ackDeadline, nil); err != nil {
		t.Fatalf("CreateSub error: %v", err)
	}
	defer subC.Delete(ctx)

	expectedCounts := make(map[string]int)
	for _, id := range publish(t, ctx, topic) {
		expectedCounts[id] = 1
	}

	// recv provides an indication that messages are still arriving.
	recv := make(chan struct{})

	// Keep track of the number of times each message (by message id) was
	// seen from each subscription.
	mcA := &messageCounter{counts: make(map[string]int), recv: recv}
	mcB := &messageCounter{counts: make(map[string]int), recv: recv}
	mcC := &messageCounter{counts: make(map[string]int), recv: recv}

	// We have three subscriptions to our topic.
	// Each subscription will get a copy of each pulished message.
	//
	// subA has just one iterator, while subB has two. The subB iterators
	// will each process roughly half of the messages for subB. All of
	// these iterators live until all messages have been consumed.  subC is
	// processed by a series of short-lived iterators.

	var wg sync.WaitGroup

	cctx, cancel := context.WithCancel(ctx)
	defer cancel()

	con := &consumer{
		lifetimes: immortal,
	}
	con.consume(t, cctx, subA, mcA, &wg)

	con = &consumer{
		lifetimes: immortal,
	}
	con.consume(t, cctx, subB, mcB, &wg)

	con = &consumer{
		lifetimes: &explicitLifetimes{
			lifetimes: []time.Duration{ackDeadline, ackDeadline, ackDeadline / 2, ackDeadline / 2},
		},
	}
	con.consume(t, cctx, subC, mcC, &wg)

	timeoutC := time.After(timeout)
	// Every time this ticker ticks, we will check if we have received any
	// messages since the last time it ticked.  We check less frequently
	// than the ack deadline, so that we can detect if messages are
	// redelivered after having their ack deadline extended.
	checkQuiescence := time.NewTicker(ackDeadline * 3)
	defer checkQuiescence.Stop()

	var received bool
loop:
	for {
		select {
		case <-recv:
			received = true
		case <-checkQuiescence.C:
			if received {
				received = false
			} else {
				cancel()
				break loop
			}
		case <-timeoutC:
			t.Errorf("timed out")
			cancel()
			return
		}
	}

	wg.Wait()
	for i, mc := range []*messageCounter{mcA, mcB, mcC} {
		if got, want := mc.counts, expectedCounts; !reflect.DeepEqual(got, want) {
			t.Errorf("%d: message counts: %v\n", i, diff(got, want))
		}
	}
}
<|endoftext|>


<commit_before>package cmds

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"text/tabwriter"

	"github.com/golang/protobuf/jsonpb"
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pps/example"
	"github.com/pachyderm/pachyderm/src/server/pps/pretty"
	"github.com/spf13/cobra"
	"go.pedge.io/pkg/cobra"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

func Cmds(address string) ([]*cobra.Command, error) {
	marshaller := &jsonpb.Marshaler{Indent: "  "}

	job := &cobra.Command{
		Use:   "job",
		Short: "Docs for jobs.",
		Long: `Jobs are the basic unit of computation in Pachyderm.

Jobs run a containerized workload over a set of finished input commits.
Creating a job will also create a new repo and a commit in that repo which
contains the output of the job. Unless the job is created with another job as a
parent. If the job is created with a parent it will use the same repo as its
parent job and the commit it creates will use the parent job's commit as a
parent.
If the job fails the commit it creates will not be finished.
The increase the throughput of a job increase the Shard paremeter.
`,
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			return nil
		}),
	}

	exampleCreateJobRequest, err := marshaller.MarshalToString(example.CreateJobRequest())
	if err != nil {
		return nil, err
	}

	exampleRunPipelineSpec, err := marshaller.MarshalToString(example.RunPipelineSpec())
	if err != nil {
		return nil, err
	}

	var jobPath string
	createJob := &cobra.Command{
		Use:   "create-job -f job.json",
		Short: "Create a new job. Returns the id of the created job.",
		Long:  fmt.Sprintf("Create a new job from a spec, the spec looks like this\n%s", exampleCreateJobRequest),
		Run: func(cmd *cobra.Command, args []string) {
			apiClient, err := getAPIClient(address)
			if err != nil {
				errorAndExit("Error connecting to pps: %s", err.Error())
			}
			var jobReader io.Reader
			if jobPath == "-" {
				jobReader = os.Stdin
				fmt.Print("Reading from stdin.\n")
			} else {
				jobFile, err := os.Open(jobPath)
				if err != nil {
					errorAndExit("Error opening %s: %s", jobPath, err.Error())
				}
				defer func() {
					if err := jobFile.Close(); err != nil {
						errorAndExit("Error closing%s: %s", jobPath, err.Error())
					}
				}()
				jobReader = jobFile
			}
			var request ppsclient.CreateJobRequest
			if err := jsonpb.Unmarshal(jobReader, &request); err != nil {
				errorAndExit("Error reading from stdin: %s", err.Error())
			}
			job, err := apiClient.CreateJob(
				context.Background(),
				&request,
			)
			if err != nil {
				errorAndExit("Error from CreateJob: %s", err.Error())
			}
			fmt.Println(job.ID)
		},
	}
	createJob.Flags().StringVarP(&jobPath, "file", "f", "-", "The file containing the job, - reads from stdin.")

	inspectJob := &cobra.Command{
		Use:   "inspect-job job-id",
		Short: "Return info about a job.",
		Long:  "Return info about a job.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			jobInfo, err := apiClient.InspectJob(
				context.Background(),
				&ppsclient.InspectJobRequest{
					Job: &ppsclient.Job{
						ID: args[0],
					},
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			if jobInfo == nil {
				errorAndExit("Job %s not found.", args[0])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintJobHeader(writer)
			pretty.PrintJobInfo(writer, jobInfo)
			return writer.Flush()
		}),
	}

	var pipelineName string
	listJob := &cobra.Command{
		Use:   "list-job -p pipeline-name",
		Short: "Return info about all jobs.",
		Long:  "Return info about all jobs.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			var pipeline *ppsclient.Pipeline
			if pipelineName != "" {
				pipeline = &ppsclient.Pipeline{
					Name: pipelineName,
				}
			}
			jobInfos, err := apiClient.ListJob(
				context.Background(),
				&ppsclient.ListJobRequest{
					Pipeline: pipeline,
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintJobHeader(writer)
			for _, jobInfo := range jobInfos.JobInfo {
				pretty.PrintJobInfo(writer, jobInfo)
			}
			return writer.Flush()
		}),
	}
	listJob.Flags().StringVarP(&pipelineName, "pipeline", "p", "", "Limit to jobs made by pipeline.")

	getLogs := &cobra.Command{
		Use:   "get-logs job-id",
		Short: "Return logs from a job.",
		Long:  "Return logs from a job.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			return ppsclient.GetLogs(apiClient, args[0], os.Stdout)
		}),
	}

	pipeline := &cobra.Command{
		Use:   "pipeline",
		Short: "Docs for pipelines.",
		Long: `Pipelines are a powerful abstraction for automating jobs.

Pipelines take a set of repos as inputs, rather than the set of commits that
jobs take. Pipelines then subscribe to commits on those repos and launches a job
to process each incoming commit.
Creating a pipeline will also create a repo of the same name.
All jobs created by a pipeline will create commits in the pipeline's repo.
`,
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			return nil
		}),
	}

	var pipelinePath string
	exampleCreatePipelineRequest, err := marshaller.MarshalToString(example.CreatePipelineRequest())
	if err != nil {
		return nil, err
	}
	createPipeline := &cobra.Command{
		Use:   "create-pipeline -f pipeline.json",
		Short: "Create a new pipeline.",
		Long:  fmt.Sprintf("Create a new pipeline from a spec, the spec looks like this\n%s", exampleCreatePipelineRequest),
		Run: func(cmd *cobra.Command, args []string) {
			apiClient, err := getAPIClient(address)
			if err != nil {
				errorAndExit("Error connecting to pps: %s", err.Error())
			}
			var pipelineReader io.Reader
			if pipelinePath == "-" {
				pipelineReader = os.Stdin
				fmt.Print("Reading from stdin.\n")
			} else {
				pipelineFile, err := os.Open(pipelinePath)
				if err != nil {
					errorAndExit("Error opening %s: %s", pipelinePath, err.Error())
				}
				defer func() {
					if err := pipelineFile.Close(); err != nil {
						errorAndExit("Error closing%s: %s", pipelinePath, err.Error())
					}
				}()
				pipelineReader = pipelineFile
			}
			var request ppsclient.CreatePipelineRequest
			decoder := json.NewDecoder(pipelineReader)
			for {
				message := json.RawMessage{}
				if err := decoder.Decode(&message); err != nil {
					if err == io.EOF {
						break
					} else {
						errorAndExit("Error reading from stdin: %s", err.Error())
					}
				}
				if err := jsonpb.UnmarshalString(string(message), &request); err != nil {
					errorAndExit("Error reading from stdin: %s", err.Error())
				}
				if _, err := apiClient.CreatePipeline(
					context.Background(),
					&request,
				); err != nil {
					errorAndExit("Error from CreatePipeline: %s", err.Error())
				}
			}
		},
	}
	createPipeline.Flags().StringVarP(&pipelinePath, "file", "f", "-", "The file containing the pipeline, - reads from stdin.")

	inspectPipeline := &cobra.Command{
		Use:   "inspect-pipeline pipeline-name",
		Short: "Return info about a pipeline.",
		Long:  "Return info about a pipeline.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			pipelineInfo, err := apiClient.InspectPipeline(
				context.Background(),
				&ppsclient.InspectPipelineRequest{
					Pipeline: &ppsclient.Pipeline{
						Name: args[0],
					},
				},
			)
			if err != nil {
				errorAndExit("Error from InspectPipeline: %s", err.Error())
			}
			if pipelineInfo == nil {
				errorAndExit("Pipeline %s not found.", args[0])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintPipelineHeader(writer)
			pretty.PrintPipelineInfo(writer, pipelineInfo)
			return writer.Flush()
		}),
	}

	listPipeline := &cobra.Command{
		Use:   "list-pipeline",
		Short: "Return info about all pipelines.",
		Long:  "Return info about all pipelines.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			pipelineInfos, err := apiClient.ListPipeline(
				context.Background(),
				&ppsclient.ListPipelineRequest{},
			)
			if err != nil {
				errorAndExit("Error from ListPipeline: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintPipelineHeader(writer)
			for _, pipelineInfo := range pipelineInfos.PipelineInfo {
				pretty.PrintPipelineInfo(writer, pipelineInfo)
			}
			return writer.Flush()
		}),
	}

	deletePipeline := &cobra.Command{
		Use:   "delete-pipeline pipeline-name",
		Short: "Delete a pipeline.",
		Long:  "Delete a pipeline.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			if _, err := apiClient.DeletePipeline(
				context.Background(),
				&ppsclient.DeletePipelineRequest{
					Pipeline: &ppsclient.Pipeline{
						Name: args[0],
					},
				},
			); err != nil {
				errorAndExit("Error from DeletePipeline: %s", err.Error())
			}
			return nil
		}),
	}

	var specPath string
	runPipeline := &cobra.Command{
		Use:   "run-pipeline pipeline-name [-f job.json]",
		Short: "Run a pipeline once.",
		Long:  fmt.Sprintf("Run a pipeline once, optionally overriding some pipeline options by providing a spec.  The spec looks like this:\n%s", exampleRunPipelineSpec),
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}

			request := &ppsclient.CreateJobRequest{
				Pipeline: &ppsclient.Pipeline{
					Name: args[0],
				},
			}

			var specReader io.Reader
			if specPath == "-" {
				specReader = os.Stdin
				fmt.Print("Reading from stdin.\n")
			} else if specPath != "" {
				specFile, err := os.Open(specPath)
				if err != nil {
					errorAndExit("Error opening %s: %s", specPath, err.Error())
				}

				defer func() {
					if err := specFile.Close(); err != nil {
						errorAndExit("Error closing%s: %s", specPath, err.Error())
					}
				}()

				specReader = specFile
				if err := jsonpb.Unmarshal(specReader, request); err != nil {
					errorAndExit("Error reading from stdin: %s", err.Error())
				}
			}

			job, err := apiClient.CreateJob(
				context.Background(),
				request,
			)
			if err != nil {
				errorAndExit("Error from RunPipeline: %s", err.Error())
			}
			fmt.Println(job.ID)
			return nil
		}),
	}
	runPipeline.Flags().StringVarP(&specPath, "file", "f", "", "The file containing the run-pipeline spec, - reads from stdin.")

	var result []*cobra.Command
	result = append(result, job)
	result = append(result, createJob)
	result = append(result, inspectJob)
	result = append(result, getLogs)
	result = append(result, listJob)
	result = append(result, pipeline)
	result = append(result, createPipeline)
	result = append(result, inspectPipeline)
	result = append(result, listPipeline)
	result = append(result, deletePipeline)
	result = append(result, runPipeline)
	return result, nil
}

func errorAndExit(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "%s\n", fmt.Sprintf(format, args...))
	os.Exit(1)
}

func getAPIClient(address string) (ppsclient.APIClient, error) {
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	if err != nil {
		return nil, err
	}
	return ppsclient.NewAPIClient(clientConn), nil
}
<commit_msg>Do the same for pps<commit_after>package cmds

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"text/tabwriter"

	"github.com/golang/protobuf/jsonpb"
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmd"
	"github.com/pachyderm/pachyderm/src/server/pps/example"
	"github.com/pachyderm/pachyderm/src/server/pps/pretty"
	"github.com/spf13/cobra"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

func Cmds(address string) ([]*cobra.Command, error) {
	marshaller := &jsonpb.Marshaler{Indent: "  "}

	job := &cobra.Command{
		Use:   "job",
		Short: "Docs for jobs.",
		Long: `Jobs are the basic unit of computation in Pachyderm.

Jobs run a containerized workload over a set of finished input commits.
Creating a job will also create a new repo and a commit in that repo which
contains the output of the job. Unless the job is created with another job as a
parent. If the job is created with a parent it will use the same repo as its
parent job and the commit it creates will use the parent job's commit as a
parent.
If the job fails the commit it creates will not be finished.
The increase the throughput of a job increase the Shard paremeter.
`,
		Run: cmd.RunFixedArgs(0, func(args []string) error {
			return nil
		}),
	}

	exampleCreateJobRequest, err := marshaller.MarshalToString(example.CreateJobRequest())
	if err != nil {
		return nil, err
	}

	exampleRunPipelineSpec, err := marshaller.MarshalToString(example.RunPipelineSpec())
	if err != nil {
		return nil, err
	}

	var jobPath string
	createJob := &cobra.Command{
		Use:   "create-job -f job.json",
		Short: "Create a new job. Returns the id of the created job.",
		Long:  fmt.Sprintf("Create a new job from a spec, the spec looks like this\n%s", exampleCreateJobRequest),
		Run: func(cmd *cobra.Command, args []string) {
			apiClient, err := getAPIClient(address)
			if err != nil {
				errorAndExit("Error connecting to pps: %s", err.Error())
			}
			var jobReader io.Reader
			if jobPath == "-" {
				jobReader = os.Stdin
				fmt.Print("Reading from stdin.\n")
			} else {
				jobFile, err := os.Open(jobPath)
				if err != nil {
					errorAndExit("Error opening %s: %s", jobPath, err.Error())
				}
				defer func() {
					if err := jobFile.Close(); err != nil {
						errorAndExit("Error closing%s: %s", jobPath, err.Error())
					}
				}()
				jobReader = jobFile
			}
			var request ppsclient.CreateJobRequest
			if err := jsonpb.Unmarshal(jobReader, &request); err != nil {
				errorAndExit("Error reading from stdin: %s", err.Error())
			}
			job, err := apiClient.CreateJob(
				context.Background(),
				&request,
			)
			if err != nil {
				errorAndExit("Error from CreateJob: %s", err.Error())
			}
			fmt.Println(job.ID)
		},
	}
	createJob.Flags().StringVarP(&jobPath, "file", "f", "-", "The file containing the job, - reads from stdin.")

	inspectJob := &cobra.Command{
		Use:   "inspect-job job-id",
		Short: "Return info about a job.",
		Long:  "Return info about a job.",
		Run: cmd.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			jobInfo, err := apiClient.InspectJob(
				context.Background(),
				&ppsclient.InspectJobRequest{
					Job: &ppsclient.Job{
						ID: args[0],
					},
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			if jobInfo == nil {
				errorAndExit("Job %s not found.", args[0])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintJobHeader(writer)
			pretty.PrintJobInfo(writer, jobInfo)
			return writer.Flush()
		}),
	}

	var pipelineName string
	listJob := &cobra.Command{
		Use:   "list-job -p pipeline-name",
		Short: "Return info about all jobs.",
		Long:  "Return info about all jobs.",
		Run: cmd.RunFixedArgs(0, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			var pipeline *ppsclient.Pipeline
			if pipelineName != "" {
				pipeline = &ppsclient.Pipeline{
					Name: pipelineName,
				}
			}
			jobInfos, err := apiClient.ListJob(
				context.Background(),
				&ppsclient.ListJobRequest{
					Pipeline: pipeline,
				},
			)
			if err != nil {
				errorAndExit("Error from InspectJob: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintJobHeader(writer)
			for _, jobInfo := range jobInfos.JobInfo {
				pretty.PrintJobInfo(writer, jobInfo)
			}
			return writer.Flush()
		}),
	}
	listJob.Flags().StringVarP(&pipelineName, "pipeline", "p", "", "Limit to jobs made by pipeline.")

	getLogs := &cobra.Command{
		Use:   "get-logs job-id",
		Short: "Return logs from a job.",
		Long:  "Return logs from a job.",
		Run: cmd.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			return ppsclient.GetLogs(apiClient, args[0], os.Stdout)
		}),
	}

	pipeline := &cobra.Command{
		Use:   "pipeline",
		Short: "Docs for pipelines.",
		Long: `Pipelines are a powerful abstraction for automating jobs.

Pipelines take a set of repos as inputs, rather than the set of commits that
jobs take. Pipelines then subscribe to commits on those repos and launches a job
to process each incoming commit.
Creating a pipeline will also create a repo of the same name.
All jobs created by a pipeline will create commits in the pipeline's repo.
`,
		Run: cmd.RunFixedArgs(0, func(args []string) error {
			return nil
		}),
	}

	var pipelinePath string
	exampleCreatePipelineRequest, err := marshaller.MarshalToString(example.CreatePipelineRequest())
	if err != nil {
		return nil, err
	}
	createPipeline := &cobra.Command{
		Use:   "create-pipeline -f pipeline.json",
		Short: "Create a new pipeline.",
		Long:  fmt.Sprintf("Create a new pipeline from a spec, the spec looks like this\n%s", exampleCreatePipelineRequest),
		Run: func(cmd *cobra.Command, args []string) {
			apiClient, err := getAPIClient(address)
			if err != nil {
				errorAndExit("Error connecting to pps: %s", err.Error())
			}
			var pipelineReader io.Reader
			if pipelinePath == "-" {
				pipelineReader = os.Stdin
				fmt.Print("Reading from stdin.\n")
			} else {
				pipelineFile, err := os.Open(pipelinePath)
				if err != nil {
					errorAndExit("Error opening %s: %s", pipelinePath, err.Error())
				}
				defer func() {
					if err := pipelineFile.Close(); err != nil {
						errorAndExit("Error closing%s: %s", pipelinePath, err.Error())
					}
				}()
				pipelineReader = pipelineFile
			}
			var request ppsclient.CreatePipelineRequest
			decoder := json.NewDecoder(pipelineReader)
			for {
				message := json.RawMessage{}
				if err := decoder.Decode(&message); err != nil {
					if err == io.EOF {
						break
					} else {
						errorAndExit("Error reading from stdin: %s", err.Error())
					}
				}
				if err := jsonpb.UnmarshalString(string(message), &request); err != nil {
					errorAndExit("Error reading from stdin: %s", err.Error())
				}
				if _, err := apiClient.CreatePipeline(
					context.Background(),
					&request,
				); err != nil {
					errorAndExit("Error from CreatePipeline: %s", err.Error())
				}
			}
		},
	}
	createPipeline.Flags().StringVarP(&pipelinePath, "file", "f", "-", "The file containing the pipeline, - reads from stdin.")

	inspectPipeline := &cobra.Command{
		Use:   "inspect-pipeline pipeline-name",
		Short: "Return info about a pipeline.",
		Long:  "Return info about a pipeline.",
		Run: cmd.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			pipelineInfo, err := apiClient.InspectPipeline(
				context.Background(),
				&ppsclient.InspectPipelineRequest{
					Pipeline: &ppsclient.Pipeline{
						Name: args[0],
					},
				},
			)
			if err != nil {
				errorAndExit("Error from InspectPipeline: %s", err.Error())
			}
			if pipelineInfo == nil {
				errorAndExit("Pipeline %s not found.", args[0])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintPipelineHeader(writer)
			pretty.PrintPipelineInfo(writer, pipelineInfo)
			return writer.Flush()
		}),
	}

	listPipeline := &cobra.Command{
		Use:   "list-pipeline",
		Short: "Return info about all pipelines.",
		Long:  "Return info about all pipelines.",
		Run: cmd.RunFixedArgs(0, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			pipelineInfos, err := apiClient.ListPipeline(
				context.Background(),
				&ppsclient.ListPipelineRequest{},
			)
			if err != nil {
				errorAndExit("Error from ListPipeline: %s", err.Error())
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintPipelineHeader(writer)
			for _, pipelineInfo := range pipelineInfos.PipelineInfo {
				pretty.PrintPipelineInfo(writer, pipelineInfo)
			}
			return writer.Flush()
		}),
	}

	deletePipeline := &cobra.Command{
		Use:   "delete-pipeline pipeline-name",
		Short: "Delete a pipeline.",
		Long:  "Delete a pipeline.",
		Run: cmd.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			if _, err := apiClient.DeletePipeline(
				context.Background(),
				&ppsclient.DeletePipelineRequest{
					Pipeline: &ppsclient.Pipeline{
						Name: args[0],
					},
				},
			); err != nil {
				errorAndExit("Error from DeletePipeline: %s", err.Error())
			}
			return nil
		}),
	}

	var specPath string
	runPipeline := &cobra.Command{
		Use:   "run-pipeline pipeline-name [-f job.json]",
		Short: "Run a pipeline once.",
		Long:  fmt.Sprintf("Run a pipeline once, optionally overriding some pipeline options by providing a spec.  The spec looks like this:\n%s", exampleRunPipelineSpec),
		Run: cmd.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}

			request := &ppsclient.CreateJobRequest{
				Pipeline: &ppsclient.Pipeline{
					Name: args[0],
				},
			}

			var specReader io.Reader
			if specPath == "-" {
				specReader = os.Stdin
				fmt.Print("Reading from stdin.\n")
			} else if specPath != "" {
				specFile, err := os.Open(specPath)
				if err != nil {
					errorAndExit("Error opening %s: %s", specPath, err.Error())
				}

				defer func() {
					if err := specFile.Close(); err != nil {
						errorAndExit("Error closing%s: %s", specPath, err.Error())
					}
				}()

				specReader = specFile
				if err := jsonpb.Unmarshal(specReader, request); err != nil {
					errorAndExit("Error reading from stdin: %s", err.Error())
				}
			}

			job, err := apiClient.CreateJob(
				context.Background(),
				request,
			)
			if err != nil {
				errorAndExit("Error from RunPipeline: %s", err.Error())
			}
			fmt.Println(job.ID)
			return nil
		}),
	}
	runPipeline.Flags().StringVarP(&specPath, "file", "f", "", "The file containing the run-pipeline spec, - reads from stdin.")

	var result []*cobra.Command
	result = append(result, job)
	result = append(result, createJob)
	result = append(result, inspectJob)
	result = append(result, getLogs)
	result = append(result, listJob)
	result = append(result, pipeline)
	result = append(result, createPipeline)
	result = append(result, inspectPipeline)
	result = append(result, listPipeline)
	result = append(result, deletePipeline)
	result = append(result, runPipeline)
	return result, nil
}

func errorAndExit(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "%s\n", fmt.Sprintf(format, args...))
	os.Exit(1)
}

func getAPIClient(address string) (ppsclient.APIClient, error) {
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	if err != nil {
		return nil, err
	}
	return ppsclient.NewAPIClient(clientConn), nil
}
<|endoftext|>


<commit_before>package worker

import (
	"context"
	"fmt"
	"path"

	"github.com/gogo/protobuf/types"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/debug"
	"github.com/pachyderm/pachyderm/src/client/pps"

	etcd "github.com/coreos/etcd/clientv3"
	"google.golang.org/grpc"
)

const (
	// WorkerEtcdPrefix is the prefix in etcd that we use to store worker information.
	WorkerEtcdPrefix = "workers"
)

// Status returns the statuses of workers referenced by pipelineRcName.
// pipelineRcName is the name of the pipeline's RC and can be gotten with
// ppsutil.PipelineRcName. You can also pass "" for pipelineRcName to get all
// clients for all workers.
func Status(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client, etcdPrefix string, workerGrpcPort uint16) ([]*pps.WorkerStatus, error) {
	workerClients, err := Clients(ctx, pipelineRcName, etcdClient, etcdPrefix, workerGrpcPort)
	if err != nil {
		return nil, err
	}
	var result []*pps.WorkerStatus
	for _, workerClient := range workerClients {
		status, err := workerClient.Status(ctx, &types.Empty{})
		if err != nil {
			return nil, err
		}
		result = append(result, status)
	}
	return result, nil
}

// Cancel cancels a set of datums running on workers.
// pipelineRcName is the name of the pipeline's RC and can be gotten with
// ppsutil.PipelineRcName.
func Cancel(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client,
	etcdPrefix string, workerGrpcPort uint16, jobID string, dataFilter []string) error {
	workerClients, err := Clients(ctx, pipelineRcName, etcdClient, etcdPrefix, workerGrpcPort)
	if err != nil {
		return err
	}
	success := false
	for _, workerClient := range workerClients {
		resp, err := workerClient.Cancel(ctx, &CancelRequest{
			JobID:       jobID,
			DataFilters: dataFilter,
		})
		if err != nil {
			return err
		}
		if resp.Success {
			success = true
		}
	}
	if !success {
		return fmt.Errorf("datum matching filter %+v could not be found for jobID %s", dataFilter, jobID)
	}
	return nil
}

// Conns returns a slice of connections to worker servers.
// pipelineRcName is the name of the pipeline's RC and can be gotten with
// ppsutil.PipelineRcName. You can also pass "" for pipelineRcName to get all
// clients for all workers.
func Conns(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client, etcdPrefix string, workerGrpcPort uint16) ([]*grpc.ClientConn, error) {
	resp, err := etcdClient.Get(ctx, path.Join(etcdPrefix, WorkerEtcdPrefix, pipelineRcName), etcd.WithPrefix())
	if err != nil {
		return nil, err
	}
	var result []*grpc.ClientConn
	for _, kv := range resp.Kvs {
		conn, err := grpc.Dial(fmt.Sprintf("%s:%d", path.Base(string(kv.Key)), workerGrpcPort),
			append(client.DefaultDialOptions(), grpc.WithInsecure())...)
		if err != nil {
			return nil, err
		}
		result = append(result, conn)
	}
	return result, nil
}

// Client combines the WorkerAPI and the DebugAPI into a single client.
type Client struct {
	WorkerClient
	debug.DebugClient
}

func newClient(conn *grpc.ClientConn) Client {
	return Client{
		NewWorkerClient(conn),
		debug.NewDebugClient(conn),
	}
}

// Clients returns a slice of worker clients for a pipeline.
// pipelineRcName is the name of the pipeline's RC and can be gotten with
// ppsutil.PipelineRcName. You can also pass "" for pipelineRcName to get all
// clients for all workers.
func Clients(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client, etcdPrefix string, workerGrpcPort uint16) ([]Client, error) {
	conns, err := Conns(ctx, pipelineRcName, etcdClient, etcdPrefix, workerGrpcPort)
	if err != nil {
		return nil, err
	}
	var result []Client
	for _, conn := range conns {
		result = append(result, newClient(conn))
	}
	return result, nil
}
<commit_msg>NewClient function to get a worker client from an address<commit_after>package worker

import (
	"context"
	"fmt"
	"path"

	"github.com/gogo/protobuf/types"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/debug"
	"github.com/pachyderm/pachyderm/src/client/pps"

	etcd "github.com/coreos/etcd/clientv3"
	"google.golang.org/grpc"
)

const (
	// WorkerEtcdPrefix is the prefix in etcd that we use to store worker information.
	WorkerEtcdPrefix = "workers"
)

// Status returns the statuses of workers referenced by pipelineRcName.
// pipelineRcName is the name of the pipeline's RC and can be gotten with
// ppsutil.PipelineRcName. You can also pass "" for pipelineRcName to get all
// clients for all workers.
func Status(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client, etcdPrefix string, workerGrpcPort uint16) ([]*pps.WorkerStatus, error) {
	workerClients, err := Clients(ctx, pipelineRcName, etcdClient, etcdPrefix, workerGrpcPort)
	if err != nil {
		return nil, err
	}
	var result []*pps.WorkerStatus
	for _, workerClient := range workerClients {
		status, err := workerClient.Status(ctx, &types.Empty{})
		if err != nil {
			return nil, err
		}
		result = append(result, status)
	}
	return result, nil
}

// Cancel cancels a set of datums running on workers.
// pipelineRcName is the name of the pipeline's RC and can be gotten with
// ppsutil.PipelineRcName.
func Cancel(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client,
	etcdPrefix string, workerGrpcPort uint16, jobID string, dataFilter []string) error {
	workerClients, err := Clients(ctx, pipelineRcName, etcdClient, etcdPrefix, workerGrpcPort)
	if err != nil {
		return err
	}
	success := false
	for _, workerClient := range workerClients {
		resp, err := workerClient.Cancel(ctx, &CancelRequest{
			JobID:       jobID,
			DataFilters: dataFilter,
		})
		if err != nil {
			return err
		}
		if resp.Success {
			success = true
		}
	}
	if !success {
		return fmt.Errorf("datum matching filter %+v could not be found for jobID %s", dataFilter, jobID)
	}
	return nil
}

// Conns returns a slice of connections to worker servers.
// pipelineRcName is the name of the pipeline's RC and can be gotten with
// ppsutil.PipelineRcName. You can also pass "" for pipelineRcName to get all
// clients for all workers.
func Conns(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client, etcdPrefix string, workerGrpcPort uint16) ([]*grpc.ClientConn, error) {
	resp, err := etcdClient.Get(ctx, path.Join(etcdPrefix, WorkerEtcdPrefix, pipelineRcName), etcd.WithPrefix())
	if err != nil {
		return nil, err
	}
	var result []*grpc.ClientConn
	for _, kv := range resp.Kvs {
		conn, err := grpc.Dial(fmt.Sprintf("%s:%d", path.Base(string(kv.Key)), workerGrpcPort),
			append(client.DefaultDialOptions(), grpc.WithInsecure())...)
		if err != nil {
			return nil, err
		}
		result = append(result, conn)
	}
	return result, nil
}

// Client combines the WorkerAPI and the DebugAPI into a single client.
type Client struct {
	WorkerClient
	debug.DebugClient
}

func newClient(conn *grpc.ClientConn) Client {
	return Client{
		NewWorkerClient(conn),
		debug.NewDebugClient(conn),
	}
}

// Clients returns a slice of worker clients for a pipeline.
// pipelineRcName is the name of the pipeline's RC and can be gotten with
// ppsutil.PipelineRcName. You can also pass "" for pipelineRcName to get all
// clients for all workers.
func Clients(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client, etcdPrefix string, workerGrpcPort uint16) ([]Client, error) {
	conns, err := Conns(ctx, pipelineRcName, etcdClient, etcdPrefix, workerGrpcPort)
	if err != nil {
		return nil, err
	}
	var result []Client
	for _, conn := range conns {
		result = append(result, newClient(conn))
	}
	return result, nil
}

// NewClient returns a worker client for the worker at the IP address passed in.
func NewClient(address string) (Client, error) {
	conn, err := grpc.Dial(fmt.Sprintf("%s:%d", address, client.PPSWorkerPort),
		append(client.DefaultDialOptions(), grpc.WithInsecure())...)
	if err != nil {
		return Client{}, err
	}
	return newClient(conn), nil
}
<|endoftext|>


<commit_before>package main

import "net"
import "fmt"
import "log"

//import "bytes"
import "strconv"
import "errors"
import "math/rand"
import "io/ioutil"
import "encoding/json"

const (
	MAX_BUFF_LEN   = 1024
	SECONDARY_PORT = 7346
	letters        = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
	sessLen        = 24  // 24 digits session length
	heartbeat      = 600 // set heartbeat time, default 600
)

// Define the message format
type message struct {
	src  string
	dest string
	err  error
}

// Mainloop tfor tcp connection
// Open server thread client --> server
func runServer(addr string, port int) error {
	lstn, err := net.Listen("tcp", fmt.Sprintf("%s:%d", addr, port))
	if err != nil {
		return err
	}
	defer lstn.Close()

	m := make(chan message)
	go runClient(addr, m)

	for {
		conn, err := lstn.Accept()
		log.Println("Incoming connection from: ", conn.RemoteAddr())
		if err != nil {
			return err
		}
		go comm(conn, m)
		defer conn.Close()
	}

}

// Open client thread server --> client
func runClient(addr string, m chan message) {
    lstn, err := net.Listen("tcp", fmt.Sprintf("%s:%d", addr, SECONDARY_PORT))

    if err != nil {
        log.Fatal(err)
        return
    }

    defer lstn.Close()

    for {
        conn, err := lstn.Accept()
        log.Println("Incoming connection from:", conn.RemoteAddr())
        if err != nil {
            log.Fatal(err)
            return
        }
        go checkStatus(conn, m)
        defer conn.Close()
    }

	return
}

// Listen the heartbeat of client and communicate
func checkStatus(conn net.Conn, m chan message) {
    return
}

// communicate with the client, main logic
func comm(conn net.Conn, msgQueue chan message) {
	// First send connection message
	// init the connection
	for {
		recvbyte, err := ioutil.ReadAll(conn)
		if err != nil {
			log.Fatal(err)
			return
		}

		defer conn.Close()

		var data map[string]interface{}
		err = json.Unmarshal(recvbyte, data)
		if err != nil {
			log.Fatal(err)
			return
		}
		// Process the data according to the protocal
		switch data["status"].(string) {
		case "q":
			// do query hadling, login the client
			msg := data["info"].(map[string]string)

			switch msg["req"] {
			case "login":
				if login(msg, conn) != nil {
					// returning messages
					m := make(map[string]interface{})
					m["status"] = "r"
					m["status_code"] = 20
					mdata := make(map[string]string)
					mdata["error"] = "Invalid username or password"
					m["info"] = mdata
					reply, err := json.Marshal(m)
					_, err = conn.Write(reply)
					if err != nil {
						log.Fatal(err)
					}
					return
				}
			// Query online users, should firstly verify sessions
			// Following all operations should carry a session id
			case "users":
				if checkLogin(msg) {
					ulist := getUsers()
					m := make(map[string]interface{})
					m["status"] = "r"
					m["status_code"] = 30
					mdata := make(map[string]interface{})
					mdata["ulist"] = ulist
					m["info"] = mdata
					reply, err := json.Marshal(m)
					_, err = conn.Write(reply)
					if err != nil {
						log.Fatal(err)
					}
				} else {
					m := make(map[string]interface{})
					m["status"] = "e"
					m["status_code"] = 40
					mdata := make(map[string]string)
					mdata["error"] = "Invalid username or sessionid"
					m["info"] = mdata
					reply, err := json.Marshal(m)
					_, err = conn.Write(reply)
					if err != nil {
						log.Fatal(err)
					}
					return
				}
			case "logout":
				if checkLogin(msg) {
					if err := logout(msg, conn); err != nil {
						log.Fatal(err)
					}
					return

				} else {

				}

			default:
				return

			}

		case "m":
			// sending messages
			// firstly we will check whether arrivable
		default:
			// return error, and exit
			return
		}
	}
}

func comm2(conn net.Conn, m chan message) {

}

func cleaner(m chan message) {
	// Periodically check un arrivable message and sweep them out in the queue
	for {

	}
}

//@deprecated, each function should log error itself
//func logger(m chan message) {
// Handle the error during server run
//    log.Println("Init server logger")
//}
// Function for generate session key

func randSeq(n int) string {
	b := make([]byte, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

func login(msg map[string]string, conn net.Conn) error {
	user := msg["user"]
	passwd := msg["passwd"]

	// Call the database function to validater user passwd
	if checkLogin(msg) {
		log.Fatal("User is already online")
		return errors.New("User is already online...")
	}

	if validate(user, passwd) {
		// Sending the secondary port of server
		m := make(map[string]interface{})
		m["status"] = "r"
		m["status_code"] = 30
		infomap := make(map[string]string)
		m["info"] = infomap
		//Listen to secondary, for ending informations
		infomap["port"] = strconv.Itoa(SECONDARY_PORT)
		infomap["session"] = randSeq(sessLen)
		// Marshal the map
		reply, err := json.Marshal(m)

		// Register and activate the user session
		if err = activate(user, infomap["session"]); err != nil {
			log.Fatal(err)
			return err
		}
		if err != nil {
			log.Fatal(err)
			return err
		}
		_, err = conn.Write(reply)
		if err != nil {
			log.Fatal(err)
			return err
		}

		return nil
	} else {
		ret := errors.New("Cannot validate user!")
		log.Fatal(ret)
		return ret
	}
}

func logout(msg map[string]string, conn net.Conn) error {
	uname, hasUserName := msg["uname"]
	sessionId, hasSessionId := msg["sessionId"]

	if hasSessionId && hasUserName {
		return killOnline(uname, sessionId)
	} else {
		return errors.New("No such user!")
	}
}

func checkLogin(msg map[string]string) bool {

	uname, hasUserName := msg["uname"]
	sessionId, hasSessionId := msg["sessionId"]

	if hasSessionId && hasUserName {
		return checkAlive(uname, sessionId)
	} else {
		return false
	}
}
<commit_msg>[devel] Adding heartbeat mechanisms<commit_after>package main

import "net"
import "fmt"
import "log"

//import "bytes"
import "strconv"
import "errors"
import "math/rand"
import "io/ioutil"
import "encoding/json"
import "time"
import "sync"

const (
	MAX_BUFF_LEN   = 1024
	SECONDARY_PORT = 7346
	letters        = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
	sessLen        = 24  // 24 digits session length
	heartbeat      = 600 // set heartbeat time, default 600
)

// Define the message format
type message struct {
	src  string
	dest string
    state string
    timeStamp time.Time
	err  error
}

// Mainloop tfor tcp connection
// Open server thread client --> server
func runServer(addr string, port int) error {
	lstn, err := net.Listen("tcp", fmt.Sprintf("%s:%d", addr, port))
	if err != nil {
		return err
	}
	defer lstn.Close()

	m := make(chan message)
	go runClient(addr, m)

	for {
		conn, err := lstn.Accept()
		log.Println("Incoming connection from: ", conn.RemoteAddr())
		if err != nil {
			return err
		}
		go comm(conn, m)
		defer conn.Close()
	}

}

// Open client thread server --> client
func runClient(addr string, m chan message) {
    lstn, err := net.Listen("tcp", fmt.Sprintf("%s:%d", addr, SECONDARY_PORT))

    if err != nil {
        log.Fatal(err)
        return
    }

    defer lstn.Close()

    for {
        conn, err := lstn.Accept()
        log.Println("Incoming connection from:", conn.RemoteAddr())
        if err != nil {
            log.Fatal(err)
            return
        }
        go checkStatus(conn, m)
        defer conn.Close()
    }
}

// Listen the heartbeat of client and communicate
func checkStatus(conn net.Conn, m chan message) {
    // Receive the connection message from the client
    recvbyte, err := ioutil.ReadAll(conn)
	if err != nil {
		log.Fatal(err)
		return
	}

	defer conn.Close()

	var data map[string]interface{}
	err = json.Unmarshal(recvbyte, data)
	if err != nil {
		log.Fatal(err)
		return
	}

    infomap := data["info"].(map[string]string)
    session, hasSession := infomap["session"]
    user, hasUser := infomap["user"]

    if !hasSession {
        log.Fatal("No session id, exit.")
        return
    }

    if !hasUser {
        log.Fatal("No username, exit.")
        return
    }

    if !checkAlive(user, session) {
        log.Fatal("Session not alive, exit.")
        return
    }

    mt := &sync.Mutex{}
    cond := sync.NewCond(mt)

    // Openning loop, for the message and heartbeat
    for {
        mt.Lock()
        //go handleMsg(m)
        cond.Wait()
        mt.Unlock()
    }

}

// communicate with the client, main logic
func comm(conn net.Conn, msgQueue chan message) {
	// First send connection message
	// init the connection
	for {
		recvbyte, err := ioutil.ReadAll(conn)
		if err != nil {
			log.Fatal(err)
			return
		}

        defer conn.Close()

		var data map[string]interface{}
		err = json.Unmarshal(recvbyte, data)
		if err != nil {
			log.Fatal(err)
			return
		}
		// Process the data according to the protocal
		switch data["status"].(string) {
		case "q":
			// do query hadling, login the client
			msg := data["info"].(map[string]string)

			switch msg["req"] {
			case "login":
				if login(msg, conn) != nil {
					// returning messages
					m := make(map[string]interface{})
					m["status"] = "r"
					m["status_code"] = 20
					mdata := make(map[string]string)
					mdata["error"] = "Invalid username or password"
					m["info"] = mdata
					reply, err := json.Marshal(m)
					_, err = conn.Write(reply)
					if err != nil {
						log.Fatal(err)
					}
					return
				}
			// Query online users, should firstly verify sessions
			// Following all operations should carry a session id
			case "users":
				if checkLogin(msg) {
					ulist := getUsers()
					m := make(map[string]interface{})
					m["status"] = "r"
					m["status_code"] = 30
					mdata := make(map[string]interface{})
					mdata["ulist"] = ulist
					m["info"] = mdata
					reply, err := json.Marshal(m)
					_, err = conn.Write(reply)
					if err != nil {
						log.Fatal(err)
					}
				} else {
					m := make(map[string]interface{})
					m["status"] = "e"
					m["status_code"] = 40
					mdata := make(map[string]string)
					mdata["error"] = "Invalid username or sessionid"
					m["info"] = mdata
					reply, err := json.Marshal(m)
					_, err = conn.Write(reply)
					if err != nil {
						log.Fatal(err)
					}
					return
				}
			case "logout":
				if checkLogin(msg) {
					if err := logout(msg, conn); err != nil {
						log.Fatal(err)
					}
					return

				} else {

				}

			default:
				return

			}

		case "m":
			// sending messages
			// firstly we will check whether arrivable
		default:
			// return error, and exit
			return
		}
	}
}

func cleaner(m chan message) {
	// Periodically check un arrivable message and sweep them out in the queue
	for {

	}
}

//@deprecated, each function should log error itself
//func logger(m chan message) {
// Handle the error during server run
//    log.Println("Init server logger")
//}
// Function for generate session key

func randSeq(n int) string {
	b := make([]byte, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

func login(msg map[string]string, conn net.Conn) error {
	user := msg["user"]
	passwd := msg["passwd"]

	// Call the database function to validater user passwd
	if checkLogin(msg) {
		log.Fatal("User is already online")
		return errors.New("User is already online...")
	}

	if validate(user, passwd) {
		// Sending the secondary port of server
		m := make(map[string]interface{})
		m["status"] = "r"
		m["status_code"] = 30
		infomap := make(map[string]string)
		m["info"] = infomap
		//Listen to secondary, for ending informations
		infomap["port"] = strconv.Itoa(SECONDARY_PORT)
		infomap["session"] = randSeq(sessLen)
		// Marshal the map
		reply, err := json.Marshal(m)

		// Register and activate the user session
		if err = activate(user, infomap["session"]); err != nil {
			log.Fatal(err)
			return err
		}
		if err != nil {
			log.Fatal(err)
			return err
		}
		_, err = conn.Write(reply)
		if err != nil {
			log.Fatal(err)
			return err
		}

		return nil
	} else {
		ret := errors.New("Cannot validate user!")
		log.Fatal(ret)
		return ret
	}
}

func logout(msg map[string]string, conn net.Conn) error {
	uname, hasUserName := msg["uname"]
	sessionId, hasSessionId := msg["sessionId"]

	if hasSessionId && hasUserName {
		return killOnline(uname, sessionId)
	} else {
		return errors.New("No such user!")
	}
}

func checkLogin(msg map[string]string) bool {

	uname, hasUserName := msg["uname"]
	sessionId, hasSessionId := msg["sessionId"]

	if hasSessionId && hasUserName {
		return checkAlive(uname, sessionId)
	} else {
		return false
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package networking

import (
	"context"
	"reflect"
	"testing"

	"knative.dev/pkg/apis"
)

func TestProtocolTypeValidate(t *testing.T) {
	cases := []struct {
		name   string
		proto  ProtocolType
		expect *apis.FieldError
	}{{
		name:   "no protocol",
		proto:  "",
		expect: apis.ErrMissingField(apis.CurrentField),
	}, {
		name:   "invalid protocol",
		proto:  "invalidProtocol",
		expect: apis.ErrInvalidValue("invalidProtocol", apis.CurrentField),
	}, {
		name:   "valid h2c protocol",
		proto:  ProtocolH2C,
		expect: nil,
	}, {
		name:   "valid http1 protocol",
		proto:  ProtocolHTTP1,
		expect: nil,
	}}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			if got, want := c.proto.Validate(context.TODO()), c.expect; !reflect.DeepEqual(got, want) {
				t.Errorf("got = %v, want: %v", got, want)
			}
		})
	}
}
<commit_msg>Remove context.TODO() from tests (#5943)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package networking

import (
	"context"
	"reflect"
	"testing"

	"knative.dev/pkg/apis"
)

func TestProtocolTypeValidate(t *testing.T) {
	cases := []struct {
		name   string
		proto  ProtocolType
		expect *apis.FieldError
	}{{
		name:   "no protocol",
		proto:  "",
		expect: apis.ErrMissingField(apis.CurrentField),
	}, {
		name:   "invalid protocol",
		proto:  "invalidProtocol",
		expect: apis.ErrInvalidValue("invalidProtocol", apis.CurrentField),
	}, {
		name:   "valid h2c protocol",
		proto:  ProtocolH2C,
		expect: nil,
	}, {
		name:   "valid http1 protocol",
		proto:  ProtocolHTTP1,
		expect: nil,
	}}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			if got, want := c.proto.Validate(context.Background()), c.expect; !reflect.DeepEqual(got, want) {
				t.Errorf("got = %v, want: %v", got, want)
			}
		})
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Camlistore Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package drive

import (
	"flag"
	"log"
	"testing"

	"camlistore.org/pkg/blobserver"
	"camlistore.org/pkg/blobserver/storagetest"
	"camlistore.org/pkg/constants/google"
	"camlistore.org/pkg/jsonconfig"
	"camlistore.org/pkg/oauthutil"

	"camlistore.org/third_party/golang.org/x/oauth2"
)

var (
	// TODO(mpl): use a config file generated with the help of googinit, like for googlestorage tests.
	parentId     = flag.String("parentDir", "", "id of the directory on google drive to use for testing. If empty or \"root\", testing is skipped.")
	clientID     = flag.String("client_id", "", "OAuth2 client_id for testing")
	clientSecret = flag.String("client_secret", "", "OAuth2 client secret for testing")
	tokenCache   = flag.String("token_cache", ".tokencache", "Token cache file.")
	authCode     = flag.String("auth_code", "", "Use when instructed to do so, when the --token_cache is empty.")
)

func TestStorage(t *testing.T) {
	if *parentId == "" || *parentId == "root" {
		t.Skip("Skipping test, refusing to use goodle drive's root directory.")
	}
	if *clientID == "" || *clientSecret == "" {
		t.Fatal("--client_id and --client_secret required. Obtain from https://console.developers.google.com/ > Project > APIs & Auth > Credentials. Should be a 'native' or 'Installed application'")
	}

	config := &oauth2.Config{
		Scopes:       []string{Scope},
		Endpoint:     google.Endpoint,
		ClientID:     *clientID,
		ClientSecret: *clientSecret,
		RedirectURL:  oauthutil.TitleBarRedirectURL,
	}
	token, err := oauth2.ReuseTokenSource(nil,
		&oauthutil.TokenSource{
			Config:    config,
			CacheFile: *tokenCache,
			AuthCode: func() string {
				if *authCode == "" {
					t.Skipf("Re-run using --auth_code= with the value obtained from %s",
						config.AuthCodeURL("", oauth2.AccessTypeOffline, oauth2.ApprovalForce))
					return ""
				}
				return *authCode
			},
		}).Token()
	if err != nil {
		t.Fatalf("could not acquire token: %v", err)
	}

	storagetest.TestOpt(t, storagetest.Opts{
		New: func(t *testing.T) (sto blobserver.Storage, cleanup func()) {
			sto, err := newFromConfig(nil, jsonconfig.Obj{
				"parent_id": *parentId,
				"auth": map[string]interface{}{
					"client_id":     *clientID,
					"client_secret": *clientSecret,
					"refresh_token": token.RefreshToken,
				},
			})
			if err != nil {
				t.Fatal(err)
			}
			if !testing.Short() {
				log.Printf("Warning: this test does many serial operations. Without the go test -short flag, this test will be very slow.")
			}
			clearDirectory := func() {
				log.Printf("WARNING: no cleanup in %v directory was done.", *parentId)
			}
			return sto, clearDirectory
		},
		SkipEnum: true,
	})
}
<commit_msg>blobserver/drive: allow config file for tests<commit_after>/*
Copyright 2014 The Camlistore Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package drive

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"testing"

	"camlistore.org/pkg/blobserver"
	"camlistore.org/pkg/blobserver/storagetest"
	"camlistore.org/pkg/constants/google"
	"camlistore.org/pkg/jsonconfig"
	"camlistore.org/pkg/oauthutil"

	"camlistore.org/third_party/golang.org/x/oauth2"
)

var (
	configFile   = flag.String("config", "", "Path to a configuration JSON file. If given, all other configuration flags are ignored. Use \"camtool googinit --type=drive\" to generate the auth parameters.")
	parentId     = flag.String("parentDir", "", "id of the directory on google drive to use for testing. If empty or \"root\", and --config blank too, testing is skipped.")
	clientID     = flag.String("client_id", "", "OAuth2 client_id for testing")
	clientSecret = flag.String("client_secret", "", "OAuth2 client secret for testing")
	tokenCache   = flag.String("token_cache", ".tokencache", "Token cache file.")
	authCode     = flag.String("auth_code", "", "Use when instructed to do so, when the --token_cache is empty.")
)

type Config struct {
	Auth      AuthConfig `json:"auth"`
	ParentDir string     `json:"parentDir"`
}

type AuthConfig struct {
	ClientID     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
	RefreshToken string `json:"refresh_token"`
}

func TestStorage(t *testing.T) {
	if (*parentId == "" || *parentId == "root") && *configFile == "" {
		t.Skip("Skipping test, refusing to use goodle drive's root directory. (you need to specify --parentDir or --config).")
	}
	var refreshToken string
	if *configFile != "" {
		data, err := ioutil.ReadFile(*configFile)
		if err != nil {
			t.Fatalf("Error reading config file %v: %v", *configFile, err)
		}
		var conf Config
		if err := json.Unmarshal(data, &conf); err != nil {
			t.Fatalf("Error decoding config file %v: %v", *configFile, err)
		}
		*clientID = conf.Auth.ClientID
		*clientSecret = conf.Auth.ClientSecret
		refreshToken = conf.Auth.RefreshToken
		*parentId = conf.ParentDir
	}
	if *parentId == "" || *parentId == "root" {
		t.Fatal("ParentDir must be specified, and not \"root\"")
	}
	if *clientID == "" || *clientSecret == "" {
		t.Fatal("--client_id and --client_secret required. Obtain from https://console.developers.google.com/ > Project > APIs & Auth > Credentials. Should be a 'native' or 'Installed application'")
	}
	if *configFile == "" {
		config := &oauth2.Config{
			Scopes:       []string{Scope},
			Endpoint:     google.Endpoint,
			ClientID:     *clientID,
			ClientSecret: *clientSecret,
			RedirectURL:  oauthutil.TitleBarRedirectURL,
		}
		token, err := oauth2.ReuseTokenSource(nil,
			&oauthutil.TokenSource{
				Config:    config,
				CacheFile: *tokenCache,
				AuthCode: func() string {
					if *authCode == "" {
						t.Skipf("Re-run using --auth_code= with the value obtained from %s",
							config.AuthCodeURL("", oauth2.AccessTypeOffline, oauth2.ApprovalForce))
						return ""
					}
					return *authCode
				},
			}).Token()
		if err != nil {
			t.Fatalf("could not acquire token: %v", err)
		}
		refreshToken = token.RefreshToken
	}

	storagetest.TestOpt(t, storagetest.Opts{
		New: func(t *testing.T) (sto blobserver.Storage, cleanup func()) {
			sto, err := newFromConfig(nil, jsonconfig.Obj{
				"parent_id": *parentId,
				"auth": map[string]interface{}{
					"client_id":     *clientID,
					"client_secret": *clientSecret,
					"refresh_token": refreshToken,
				},
			})
			if err != nil {
				t.Fatal(err)
			}
			if !testing.Short() {
				log.Printf("Warning: this test does many serial operations. Without the go test -short flag, this test will be very slow.")
			}
			clearDirectory := func() {
				log.Printf("WARNING: no cleanup in %v directory was done.", *parentId)
			}
			return sto, clearDirectory
		},
		SkipEnum: true,
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Nuclio Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v3iostream

import (
	"github.com/nuclio/nuclio-sdk-go"
	"github.com/v3io/v3io-go/pkg/dataplane"
)

type Event struct {
	nuclio.AbstractEvent
	record *v3io.StreamRecord
}

func (e *Event) GetBody() []byte {
	return e.record.Data
}

func (e *Event) GetSize() int {
	return len(e.GetBody())
}

func (e *Event) GetShardID() int {
	return *e.record.ShardID
}

func (e *Event) GetPath() string {
	return "Unsupported"
}

func (e *Event) GetOffset() int {
	return int(e.record.SequenceNumber)
}
<commit_msg>Update V3IO trigger event path (#2285)<commit_after>/*
Copyright 2018 The Nuclio Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v3iostream

import (
	"github.com/nuclio/nuclio-sdk-go"
	"github.com/v3io/v3io-go/pkg/dataplane"
)

type Event struct {
	nuclio.AbstractEvent
	record *v3io.StreamRecord
}

func (e *Event) GetBody() []byte {
	return e.record.Data
}

func (e *Event) GetSize() int {
	return len(e.GetBody())
}

func (e *Event) GetShardID() int {
	return *e.record.ShardID
}

func (e *Event) GetOffset() int {
	return int(e.record.SequenceNumber)
}
<|endoftext|>