<commit_before>package main

import (
	"errors"
	"fmt"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

type SimpleChaincode struct {
}

func main() {
	err := shim.Start(new(SimpleChaincode))

	if err != nil {
		//do nothing
	}
}

func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	// added by chenlin@20170308
	if len(args) != 0 {
		return nil, errors.New("incorrect args")
	}
	adminCert, err := stub.GetCallerMetadata()
	if err != nil {
		return nil, errors.New("failed getting metadata")
	}
	if len(adminCert) == 0 {
		return nil, errors.New("invalid admin certificate. Empty")
	}
	stub.PutState("admin", adminCert)
	return nil, nil
}

func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	switch function {
	case "putState":
		if len(args) != 2 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		value := []byte(args[1])
		err := stub.PutState(key, value)
		if err != nil {
			return nil, err
		}
		return value, err
	case "delState":
		if len(args) != 1 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		err := stub.DelState(key)
		if err != nil {
			return nil, err
		}
		result := key + "has been deleted"
		return []byte(result), err

	case "createTable":
		err := stub.CreateTable("AssetsOwnership", []*shim.ColumnDefinition{
			&shim.ColumnDefinition{Name: "Asset", Type: shim.ColumnDefinition_STRING, Key: true},
			&shim.ColumnDefinition{Name: "Owner", Type: shim.ColumnDefinition_BYTES, Key: false},
		})
		if err != nil {
			return nil, errors.New("Failed creating AssetsOnwership table.")
		}
		return nil, nil
	}

	return nil, nil
}

func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	switch function {
	case "getTxID":
		txID := stub.GetTxID()
		result := []byte(txID)
		return result, nil
	case "getTxTimestamp":
		time, err := stub.GetTxTimestamp()
		result := []byte(time.String()) //时间转换为字符串，time.String()
		return result, err
	case "getStringArgs":
		strList := stub.GetStringArgs()
		var result string
		for index := 0; index < len(strList); index++ {
			result += "***" + strList[index] + "***"
		}
		return []byte(result), nil

	case "getState":
		if len(args) != 1 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		result, err := stub.GetState(key)

		if err != nil {
			return nil, errors.New("Failed in function getState")
		}
		return result, err

	case "getCallerCert":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetCallerCertificate()
		if err != nil {
			return nil, errors.New("Failed in function getCallerCert")
		}
		return result, err

	case "getCallerMetadata":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetCallerMetadata()
		if err != nil {
			return nil, errors.New("Failed in function getCallerMetadata")
		}
		fmt.Println(result)
		return result, err
		// return []byte("getCallerMetadata"), err
	case "getBinding":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetBinding()

		if err != nil {
			return nil, errors.New("Failed in function getBinding")
		}
		fmt.Println(result)
		return result, err
		// return []byte("getBinding"), err
	case "getPayload":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetPayload()

		if err != nil {
			return nil, errors.New("Failed in function getPayload")
		}
		fmt.Println(result)
		return result, err
		// return []byte("getPayload"), err
	}

	return nil, nil
}
<commit_msg>init() try<commit_after>package main

import (
	"errors"
	"fmt"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

type SimpleChaincode struct {
}

func main() {
	err := shim.Start(new(SimpleChaincode))

	if err != nil {
		//do nothing
	}
}

func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	// added by chenlin@20170308
	if len(args) != 0 {
		return nil, errors.New("incorrect args")
	}
	// adminCert, err := stub.GetCallerMetadata()
	// if err != nil {
	// 	return nil, errors.New("failed getting metadata")
	// }
	// if len(adminCert) == 0 {
	// 	return nil, errors.New("invalid admin certificate. Empty")
	// }
	stub.PutState("admin", []byte("hahahahaha"))
	return nil, nil
}

func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	switch function {
	case "putState":
		if len(args) != 2 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		value := []byte(args[1])
		err := stub.PutState(key, value)
		if err != nil {
			return nil, err
		}
		return value, err
	case "delState":
		if len(args) != 1 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		err := stub.DelState(key)
		if err != nil {
			return nil, err
		}
		result := key + "has been deleted"
		return []byte(result), err

	case "createTable":
		err := stub.CreateTable("AssetsOwnership", []*shim.ColumnDefinition{
			&shim.ColumnDefinition{Name: "Asset", Type: shim.ColumnDefinition_STRING, Key: true},
			&shim.ColumnDefinition{Name: "Owner", Type: shim.ColumnDefinition_BYTES, Key: false},
		})
		if err != nil {
			return nil, errors.New("Failed creating AssetsOnwership table.")
		}
		return nil, nil
	}

	return nil, nil
}

func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	switch function {
	case "getTxID":
		txID := stub.GetTxID()
		result := []byte(txID)
		return result, nil
	case "getTxTimestamp":
		time, err := stub.GetTxTimestamp()
		result := []byte(time.String()) //时间转换为字符串，time.String()
		return result, err
	case "getStringArgs":
		strList := stub.GetStringArgs()
		var result string
		for index := 0; index < len(strList); index++ {
			result += "***" + strList[index] + "***"
		}
		return []byte(result), nil

	case "getState":
		if len(args) != 1 {
			return nil, errors.New("incorrect args")
		}
		key := args[0]
		result, err := stub.GetState(key)

		if err != nil {
			return nil, errors.New("Failed in function getState")
		}
		return result, err

	case "getCallerCert":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetCallerCertificate()
		if err != nil {
			return nil, errors.New("Failed in function getCallerCert")
		}
		return result, err

	case "getCallerMetadata":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetCallerMetadata()
		if err != nil {
			return nil, errors.New("Failed in function getCallerMetadata")
		}
		fmt.Println(result)
		return result, err
		// return []byte("getCallerMetadata"), err
	case "getBinding":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetBinding()

		if err != nil {
			return nil, errors.New("Failed in function getBinding")
		}
		fmt.Println(result)
		return result, err
		// return []byte("getBinding"), err
	case "getPayload":
		if len(args) != 0 {
			return nil, errors.New("incorrect args")
		}
		result, err := stub.GetPayload()

		if err != nil {
			return nil, errors.New("Failed in function getPayload")
		}
		fmt.Println(result)
		return result, err
		// return []byte("getPayload"), err
	}

	return nil, nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"strconv"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

type SimpleChaincode struct {
}

var projectName string
var projectRate int
var projectPeriod int
var projectGoal int
var projectTimes int
var projectBenifary string
var projectState string
var currentPrice float64
var projectSummary int
var userList []string

var shareList map[string]int

// var availableList map[string]int

func main() {
	fmt.Println("started logging in main()")
	err := shim.Start(new(SimpleChaincode))
	if err != nil {
		fmt.Println("failed in function main()")
	}
}

//Init comment
func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Init()")

	userList = append(userList, "hello1")
	userList = append(userList, "welcome")

	shareList = make(map[string]int)
	shareList["aaa"] = 111
	shareList["bbb"] = 222

	//started to initialize the projectState
	//step1: check whether args == 5
	if len(args) != 5 {
		return nil, errors.New("failed in args")
	}

	//step 2, intialize the project properties
	projectName = args[0]
	stub.PutState("projectName", []byte(projectName))

	projectRate, _ = strconv.Atoi(args[1])
	if projectRate <= 0 {
		return nil, errors.New("errors in args[1], it cannot be negative")
	}
	stub.PutState("projectRate", []byte(args[1]))

	projectPeriod, _ = strconv.Atoi(args[2])
	if projectPeriod <= 0 {
		return nil, errors.New("errors in args[2], it cannot be negative")
	}
	stub.PutState("projectPeriod", []byte(args[2]))

	projectGoal, _ = strconv.Atoi(args[3])
	if projectGoal <= 0 {
		return nil, errors.New("errors in args[3], it cannot be negative")
	}
	stub.PutState("projectGoal", []byte(args[3]))

	projectTimes = 1
	stub.PutState("projectTimes", []byte(strconv.Itoa(projectTimes)))

	projectBenifary = args[4]
	stub.PutState("projectBenifary", []byte(args[4]))

	projectState = "draft"
	stub.PutState("projectState", []byte(projectState))

	currentPrice = 1.0
	projectSummary = 0

	return nil, nil
} //end of Init()

// Invoke comment
func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Invoke()")
	switch function {
	case "goPublic":
		fmt.Println("started logging in goPublic")
		if len(args) != 0 {
			return nil, errors.New("failed in args")
		}
		projectState = "public"
		return nil, nil //end of goPublic
	case "pay":
		fmt.Println("started logging in pay()")

		if len(args) != 2 {
			return nil, errors.New("failed in args")
		}
		user := args[0]
		amount := args[1]
		fmt.Printf("user is %v and amount is %v\n", user, amount)
		amountInt, _ := strconv.Atoi(amount)
		if amountInt <= 0 {
			return nil, errors.New("amount is negative")
		}

		shareList[user] += amountInt

		projectSummary += amountInt
		return nil, nil
	case "checkGoalReached":
		fmt.Println("started logging in checkGoalReached()")
		// TODO: write code for checkGoalReached
		return nil, nil
	case "checkDaoqi":
		fmt.Println("started logging in checkDaoqi()")
		// TODO: write code for checkDaoqi
		return nil, nil
	case "calculatePrice":
		fmt.Println("started logging in calculatePrice")
		return nil, nil
	case "calculateResult":
		fmt.Println("started logging in calculateResult")
		return nil, nil
	}
	return nil, nil
}

// Query comment
func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Query()")
	switch function {
	case "getProjectName":
		result, err := stub.GetState("projectName")
		if err != nil {
			return nil, errors.New("error in getProjectName")
		}
		return result, nil
	case "getProjectGoal":
		result, err := stub.GetState("projectGoal")
		if err != nil {
			return nil, errors.New("error in getProjectGoal")
		}
		return result, nil
	case "getProjectSummary":
		result := strconv.Itoa(projectSummary)
		return []byte(result), nil
		// case "getUserList":
		// 	fmt.Println("started logging in getUserList")
		// 	var result string
		//
		// 	for _, str := range userList {
		// 		result += str + "****"
		// 	}
		// 	return []byte(result), nil
	case "getShareList":
		fmt.Println("started logging in getUserList")

		var result string
		for user, amount := range shareList {
			result += "****" + user + "/" + strconv.Itoa(amount)
		}
		return []byte(result), nil
	case "getAvailableList":
		fmt.Println("started logging in getAvailableList")
		return nil, nil
	}
	return nil, nil
}
<commit_msg>demo7.go<commit_after>package main

import (
	"errors"
	"fmt"
	"strconv"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

type SimpleChaincode struct {
}

var projectName string
var projectRate int
var projectPeriod int
var projectGoal int
var projectTimes int
var projectBenifary string
var projectState string
var currentPrice float64
var projectSummary int

var shareList map[string]int

// var availableList map[string]int

func main() {
	fmt.Println("started logging in main()")
	err := shim.Start(new(SimpleChaincode))
	if err != nil {
		fmt.Println("failed in function main()")
	}
}

//Init comment
func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Init()")
	shareList = make(map[string]int)

	//started to initialize the projectState
	//step1: check whether args == 5
	if len(args) != 5 {
		return nil, errors.New("failed in args")
	}

	//step 2, intialize the project properties
	projectName = args[0]
	stub.PutState("projectName", []byte(projectName))

	projectRate, _ = strconv.Atoi(args[1])
	if projectRate <= 0 {
		return nil, errors.New("errors in args[1], it cannot be negative")
	}
	stub.PutState("projectRate", []byte(args[1]))

	projectPeriod, _ = strconv.Atoi(args[2])
	if projectPeriod <= 0 {
		return nil, errors.New("errors in args[2], it cannot be negative")
	}
	stub.PutState("projectPeriod", []byte(args[2]))

	projectGoal, _ = strconv.Atoi(args[3])
	if projectGoal <= 0 {
		return nil, errors.New("errors in args[3], it cannot be negative")
	}
	stub.PutState("projectGoal", []byte(args[3]))

	projectTimes = 1
	stub.PutState("projectTimes", []byte(strconv.Itoa(projectTimes)))

	projectBenifary = args[4]
	stub.PutState("projectBenifary", []byte(args[4]))

	projectState = "draft"
	stub.PutState("projectState", []byte(projectState))

	currentPrice = 1.0
	projectSummary = 0

	return nil, nil
} //end of Init()

// Invoke comment
func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Invoke()")
	switch function {
	case "goPublic":
		fmt.Println("started logging in goPublic")
		if len(args) != 0 {
			return nil, errors.New("failed in args")
		}
		projectState = "public"
		return nil, nil //end of goPublic
	case "pay":
		fmt.Println("started logging in pay()")

		if len(args) != 2 {
			return nil, errors.New("failed in args")
		}
		user := args[0]
		amount := args[1]
		fmt.Printf("user is %v and amount is %v\n", user, amount)
		amountInt, _ := strconv.Atoi(amount)
		if amountInt <= 0 {
			return nil, errors.New("amount is negative")
		}

		shareList[user] += amountInt

		projectSummary += amountInt
		return nil, nil
	case "checkGoalReached":
		fmt.Println("started logging in checkGoalReached()")
		// TODO: write code for checkGoalReached
		return nil, nil
	case "checkDaoqi":
		fmt.Println("started logging in checkDaoqi()")
		// TODO: write code for checkDaoqi
		return nil, nil
	case "calculatePrice":
		fmt.Println("started logging in calculatePrice")
		return nil, nil
	case "calculateResult":
		fmt.Println("started logging in calculateResult")
		return nil, nil
	}
	return nil, nil
}

// Query comment
func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Query()")
	switch function {
	case "getProjectName":
		result, err := stub.GetState("projectName")
		if err != nil {
			return nil, errors.New("error in getProjectName")
		}
		return result, nil
	case "getProjectGoal":
		result, err := stub.GetState("projectGoal")
		if err != nil {
			return nil, errors.New("error in getProjectGoal")
		}
		return result, nil
	case "getProjectSummary":
		result := strconv.Itoa(projectSummary)
		return []byte(result), nil

	case "getShareList":
		fmt.Println("started logging in getUserList")

		var result string
		for user, amount := range shareList {
			result += "****" + user + "/" + strconv.Itoa(amount)
		}
		return []byte(result), nil
	case "getAvailableList":
		fmt.Println("started logging in getAvailableList")
		return nil, nil
	}
	return nil, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	"github.com/crask/kafka/consumergroup"
	"github.com/golang/glog"
)

type Worker struct {
	Callback    *WorkerCallback
	Topics      []string
	Zookeeper   []string
	ZkPath      string
	Consumer    *consumergroup.ConsumerGroup
	Serializer  string
	ContentType string
	Tracker     OffsetMap
	Transport   http.RoundTripper
}

type (
	OffsetMap map[string]*TrackerData
)

type TrackerData struct {
	LastRecordOpTime int64
	CurrRecordOpTime int64
	LogId            string
	Offset           int64
}

type RMSG struct {
	Topic        string `json:"Topic"`
	PartitionKey string `json:"PartitionKey"`
	TimeStamp    int64  `json:"TimeStamp"`
	Data         string `json:"Data"`
	LogId        string `json:"LogId"`
	ContentType  string `json:"ContentType"`
}

type WorkerCallback struct {
	Url          string
	RetryTimes   int
	Timeout      time.Duration
	BypassFailed bool
	FailedSleep  time.Duration
}

func NewWorker() *Worker {
	return &Worker{}
}

func (this *Worker) Init(config *CallbackItemConfig, transport http.RoundTripper) error {
	this.Callback = &WorkerCallback{
		Url:          config.Url,
		RetryTimes:   config.RetryTimes,
		Timeout:      config.Timeout,
		BypassFailed: config.BypassFailed,
		FailedSleep:  config.FailedSleep,
	}
	this.Topics = config.Topics
	this.Zookeeper = config.Zookeepers
	this.ZkPath = config.ZkPath
	this.Consumer = nil
	this.Serializer = config.Serializer
	this.ContentType = config.ContentType
	this.Tracker = make(OffsetMap)
	this.Transport = transport
	this.LogCollectRatio = config.LogCollectRatio

	cgConfig := consumergroup.NewConfig()
	cgConfig.Offsets.ProcessingTimeout = 10 * time.Second
	cgConfig.Offsets.CommitInterval = time.Duration(commitInterval) * time.Second
	cgConfig.Offsets.Initial = sarama.OffsetNewest

	// Random Sleeping to avoid burst commit onto ZK.
	r := rand.Intn(commitInterval)
	time.Sleep(time.Duration(r))

	if len(this.ZkPath) > 0 {
		cgConfig.Zookeeper.Chroot = this.ZkPath
	}

	cgName := getGroupName(this.Callback.Url)
	consumer, err := consumergroup.JoinConsumerGroup(cgName, this.Topics, this.Zookeeper, cgConfig)
	if err != nil {
		glog.Errorf("Failed to join consumer group for url[%v], %v", this.Callback.Url, err.Error())
		return err
	} else {
		glog.V(1).Infof("Join consumer group for url[%s] with UUID[%s]", this.Callback.Url, cgName)
	}

	glog.Infoln(consumer)

	this.Consumer = consumer
	return nil
}

func (this *Worker) Work() {
	var tsrpc, terpc time.Time

	consumer := this.Consumer

	go func() {
		for err := range consumer.Errors() {
			glog.Errorln("Error working consumers", err)
		}
	}()

	eventCount := 0
	offsets := make(map[string]map[int32]int64)

	for message := range consumer.Messages() {
		if offsets[message.Topic] == nil {
			offsets[message.Topic] = make(map[int32]int64)
		}

		eventCount += 1
		if offsets[message.Topic][message.Partition] != 0 && offsets[message.Topic][message.Partition] != message.Offset-1 {
			glog.Errorf("Unexpected offset on %s:%d. Expected %d, found %d, diff %d.\n", message.Topic, message.Partition, offsets[message.Topic][message.Partition]+1, message.Offset, message.Offset-offsets[message.Topic][message.Partition]+1)
		}

		msg := CreateMsg(message)
		glog.V(2).Infof("received message,[topic:%s][partition:%d][offset:%d]", msg.Topic, msg.Partition, msg.Offset)

		deliverySuccessed := false
		retry_times := 0
		tsrpc = time.Now()
		for {
			for !deliverySuccessed && retry_times < this.Callback.RetryTimes {
				deliverySuccessed, _ = this.delivery(msg, retry_times)
				if !deliverySuccessed {
					retry_times++
				} else {
					break
				}
			}

			if deliverySuccessed {
				break
			}

			if this.Callback.BypassFailed {
				glog.Errorf("tried to delivery message [url:%s][topic:%s][partition:%d][offset:%d] for %d times and all failed. BypassFailed is :%t ,will not retry", this.Callback.Url, msg.Topic, msg.Partition, msg.Offset, retry_times, this.Callback.BypassFailed)
				break
			} else {
				glog.Errorf("tried to delivery message [url:%s][topic:%s][partition:%d][offset:%d] for %d times and all failed. BypassFailed is :%t ,sleep %s to retry", this.Callback.Url, msg.Topic, msg.Partition, msg.Offset, retry_times, this.Callback.BypassFailed, this.Callback.FailedSleep)
				time.Sleep(this.Callback.FailedSleep)
				retry_times = 0
			}
		}
		terpc = time.Now()

		offsets[message.Topic][message.Partition] = message.Offset
		consumer.CommitUpto(message)
		glog.Infof("commited message,[topic:%s][partition:%d][offset:%d][url:%s][cost:%vms]", msg.Topic, msg.Partition, msg.Offset, this.Callback.Url, fmt.Sprintf("%.2f", terpc.Sub(tsrpc).Seconds()*1000))

	}

}

func (this *Worker) delivery(msg *Msg, retry_times int) (success bool, err error) {
	var tsrpc, terpc time.Time

	client := &http.Client{Transport: this.Transport}
	client.Timeout = this.Callback.Timeout

	var rmsg RMSG

	switch this.Serializer {
	case "":
		fallthrough
	case "raw":
		rmsg.Data = string(msg.Value)
	case "json":
		fallthrough
	default:
		json.Unmarshal(msg.Value, &rmsg)
	}

	if this.ContentType != "" {
		rmsg.ContentType = this.ContentType
	} else if rmsg.ContentType == "" {
		rmsg.ContentType = "application/x-www-form-urlencoded"
	}

	req, _ := http.NewRequest("POST", this.Callback.Url, ioutil.NopCloser(strings.NewReader(rmsg.Data)))
	req.Header.Set("Content-Type", rmsg.ContentType)
	req.Header.Set("User-Agent", "Taiji pusher consumer(go)/v"+VERSION)
	req.Header.Set("X-Retry-Times", fmt.Sprintf("%d", retry_times))
	req.Header.Set("X-Kmq-Topic", msg.Topic)
	req.Header.Set("X-Kmq-Partition", fmt.Sprintf("%d", msg.Partition))
	req.Header.Set("X-Kmq-Partition-Key", rmsg.PartitionKey)
	req.Header.Set("X-Kmq-Offset", fmt.Sprintf("%d", msg.Offset))
	req.Header.Set("X-Kmq-Logid", fmt.Sprintf("%s", rmsg.LogId))
	req.Header.Set("X-Kmq-Timestamp", fmt.Sprintf("%d", rmsg.TimeStamp))
	req.Header.Set("Meilishuo", "uid:0;ip:0.0.0.0;v:0;master:0")
	tsrpc = time.Now()
	resp, err := client.Do(req)
	terpc = time.Now()

	var op_time int64
	if this.Serializer == "json" {
		op_time = time.Now().UnixNano()/1000000 - rmsg.TimeStamp
	} else {
		op_time = terpc.Sub(tsrpc).Nanoseconds() / 1000000
	}

	suc := true
	if nil == err {
		defer resp.Body.Close()
		suc = (resp.StatusCode == 200)
		if suc == false {
			rbody, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				rbody = []byte{}
			}
			glog.Errorf("delivery failed,[retry_times:%d][topic:%s][partition:%d][offset:%d][msg:%s][url:%s][http_code:%d][cost:%vms][response_body:%s][current:%s]", retry_times, msg.Topic, msg.Partition, msg.Offset, rmsg.Data, this.Callback.Url, resp.StatusCode, fmt.Sprintf("%.2f", terpc.Sub(tsrpc).Seconds()*1000), rbody, terpc)
		} else {
			if this.Serializer == "json" {
				this.CommitNewTracker(&rmsg, msg)
			}

			// consume all data in response body to enable connection reusing
			discardBody(resp.Body)
		}
	} else {
		glog.Errorf("delivery failed,[retry_times:%d][topic:%s][partition:%d][offset:%d][msg:%s][url:%s][error:%s][cost:%vms][total_cost:%vms][current:%s]", retry_times, msg.Topic, msg.Partition, msg.Offset, rmsg.Data, this.Callback.Url, err.Error(), fmt.Sprintf("%.2f", terpc.Sub(tsrpc).Seconds()*1000), fmt.Sprintf("%.2f", terpc.Sub(tsrpc).Seconds()*1000), op_time, terpc)
		suc = false
	}
	if switcher := this.LogSamplingCollect(); switcher != false {
		glog.Infof("log sampling,[url:%s][retry_times:%d][topic:%s][partition:%d][offset:%d][cost:%v][total_cost:%v][content-type:%s][current_time:%s]", this.Callback.Url, retry_times, msg.Topic, msg.Partition, msg.Offset, fmt.Sprintf("%.2f", terpc.Sub(tsrpc).Seconds()*1000), op_time, rmsg.ContentType, terpc)
	}

	return suc, err
}

func (this *Worker) Closed() bool {
	return this.Consumer.Closed()
}

func (this *Worker) Close() {
	if err := this.Consumer.Close(); err != nil {
		glog.Errorln("Error closing consumers", err)
	}
}

func (this *Worker) GetWorkerTracker() OffsetMap {
	return this.Tracker
}

func (this *Worker) CommitNewTracker(rmsg *RMSG, msg *Msg) (err error) {
	value, ok := this.Tracker[fmt.Sprintf("%d", msg.Partition)]
	if !ok {
		this.Tracker[fmt.Sprintf("%d", msg.Partition)] = &TrackerData{
			LastRecordOpTime: 0,
			CurrRecordOpTime: rmsg.TimeStamp,
			LogId:            rmsg.LogId,
			Offset:           msg.Offset,
		}
		return nil
	} else if ok && rmsg.TimeStamp >= value.CurrRecordOpTime {
		this.Tracker[fmt.Sprintf("%d", msg.Partition)] = &TrackerData{
			LastRecordOpTime: value.CurrRecordOpTime,
			CurrRecordOpTime: rmsg.TimeStamp,
			LogId:            rmsg.LogId,
			Offset:           msg.Offset,
		}
		return nil
	}
	return nil
}

func (this *Worker) LogSamplingCollect() bool {
	r := rand.Intn(this.LogCollectRatio)
	if r == 1 {
		return true
	}
	return false
}
<commit_msg>add new config for worker<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	"github.com/crask/kafka/consumergroup"
	"github.com/golang/glog"
)

type Worker struct {
	Callback        *WorkerCallback
	Topics          []string
	Zookeeper       []string
	ZkPath          string
	Consumer        *consumergroup.ConsumerGroup
	Serializer      string
	ContentType     string
	Tracker         OffsetMap
	Transport       http.RoundTripper
	LogCollectRatio int
}

type (
	OffsetMap map[string]*TrackerData
)

type TrackerData struct {
	LastRecordOpTime int64
	CurrRecordOpTime int64
	LogId            string
	Offset           int64
}

type RMSG struct {
	Topic        string `json:"Topic"`
	PartitionKey string `json:"PartitionKey"`
	TimeStamp    int64  `json:"TimeStamp"`
	Data         string `json:"Data"`
	LogId        string `json:"LogId"`
	ContentType  string `json:"ContentType"`
}

type WorkerCallback struct {
	Url          string
	RetryTimes   int
	Timeout      time.Duration
	BypassFailed bool
	FailedSleep  time.Duration
}

func NewWorker() *Worker {
	return &Worker{}
}

func (this *Worker) Init(config *CallbackItemConfig, transport http.RoundTripper) error {
	this.Callback = &WorkerCallback{
		Url:          config.Url,
		RetryTimes:   config.RetryTimes,
		Timeout:      config.Timeout,
		BypassFailed: config.BypassFailed,
		FailedSleep:  config.FailedSleep,
	}
	this.Topics = config.Topics
	this.Zookeeper = config.Zookeepers
	this.ZkPath = config.ZkPath
	this.Consumer = nil
	this.Serializer = config.Serializer
	this.ContentType = config.ContentType
	this.Tracker = make(OffsetMap)
	this.Transport = transport
	this.LogCollectRatio = config.LogCollectRatio

	cgConfig := consumergroup.NewConfig()
	cgConfig.Offsets.ProcessingTimeout = 10 * time.Second
	cgConfig.Offsets.CommitInterval = time.Duration(commitInterval) * time.Second
	cgConfig.Offsets.Initial = sarama.OffsetNewest

	// Random Sleeping to avoid burst commit onto ZK.
	r := rand.Intn(commitInterval)
	time.Sleep(time.Duration(r))

	if len(this.ZkPath) > 0 {
		cgConfig.Zookeeper.Chroot = this.ZkPath
	}

	cgName := getGroupName(this.Callback.Url)
	consumer, err := consumergroup.JoinConsumerGroup(cgName, this.Topics, this.Zookeeper, cgConfig)
	if err != nil {
		glog.Errorf("Failed to join consumer group for url[%v], %v", this.Callback.Url, err.Error())
		return err
	} else {
		glog.V(1).Infof("Join consumer group for url[%s] with UUID[%s]", this.Callback.Url, cgName)
	}

	glog.Infoln(consumer)

	this.Consumer = consumer
	return nil
}

func (this *Worker) Work() {
	var tsrpc, terpc time.Time

	consumer := this.Consumer

	go func() {
		for err := range consumer.Errors() {
			glog.Errorln("Error working consumers", err)
		}
	}()

	eventCount := 0
	offsets := make(map[string]map[int32]int64)

	for message := range consumer.Messages() {
		if offsets[message.Topic] == nil {
			offsets[message.Topic] = make(map[int32]int64)
		}

		eventCount += 1
		if offsets[message.Topic][message.Partition] != 0 && offsets[message.Topic][message.Partition] != message.Offset-1 {
			glog.Errorf("Unexpected offset on %s:%d. Expected %d, found %d, diff %d.\n", message.Topic, message.Partition, offsets[message.Topic][message.Partition]+1, message.Offset, message.Offset-offsets[message.Topic][message.Partition]+1)
		}

		msg := CreateMsg(message)
		glog.V(2).Infof("received message,[topic:%s][partition:%d][offset:%d]", msg.Topic, msg.Partition, msg.Offset)

		deliverySuccessed := false
		retry_times := 0
		tsrpc = time.Now()
		for {
			for !deliverySuccessed && retry_times < this.Callback.RetryTimes {
				deliverySuccessed, _ = this.delivery(msg, retry_times)
				if !deliverySuccessed {
					retry_times++
				} else {
					break
				}
			}

			if deliverySuccessed {
				break
			}

			if this.Callback.BypassFailed {
				glog.Errorf("tried to delivery message [url:%s][topic:%s][partition:%d][offset:%d] for %d times and all failed. BypassFailed is :%t ,will not retry", this.Callback.Url, msg.Topic, msg.Partition, msg.Offset, retry_times, this.Callback.BypassFailed)
				break
			} else {
				glog.Errorf("tried to delivery message [url:%s][topic:%s][partition:%d][offset:%d] for %d times and all failed. BypassFailed is :%t ,sleep %s to retry", this.Callback.Url, msg.Topic, msg.Partition, msg.Offset, retry_times, this.Callback.BypassFailed, this.Callback.FailedSleep)
				time.Sleep(this.Callback.FailedSleep)
				retry_times = 0
			}
		}
		terpc = time.Now()

		offsets[message.Topic][message.Partition] = message.Offset
		consumer.CommitUpto(message)
		glog.Infof("commited message,[topic:%s][partition:%d][offset:%d][url:%s][cost:%vms]", msg.Topic, msg.Partition, msg.Offset, this.Callback.Url, fmt.Sprintf("%.2f", terpc.Sub(tsrpc).Seconds()*1000))

	}

}

func (this *Worker) delivery(msg *Msg, retry_times int) (success bool, err error) {
	var tsrpc, terpc time.Time

	client := &http.Client{Transport: this.Transport}
	client.Timeout = this.Callback.Timeout

	var rmsg RMSG

	switch this.Serializer {
	case "":
		fallthrough
	case "raw":
		rmsg.Data = string(msg.Value)
	case "json":
		fallthrough
	default:
		json.Unmarshal(msg.Value, &rmsg)
	}

	if this.ContentType != "" {
		rmsg.ContentType = this.ContentType
	} else if rmsg.ContentType == "" {
		rmsg.ContentType = "application/x-www-form-urlencoded"
	}

	req, _ := http.NewRequest("POST", this.Callback.Url, ioutil.NopCloser(strings.NewReader(rmsg.Data)))
	req.Header.Set("Content-Type", rmsg.ContentType)
	req.Header.Set("User-Agent", "Taiji pusher consumer(go)/v"+VERSION)
	req.Header.Set("X-Retry-Times", fmt.Sprintf("%d", retry_times))
	req.Header.Set("X-Kmq-Topic", msg.Topic)
	req.Header.Set("X-Kmq-Partition", fmt.Sprintf("%d", msg.Partition))
	req.Header.Set("X-Kmq-Partition-Key", rmsg.PartitionKey)
	req.Header.Set("X-Kmq-Offset", fmt.Sprintf("%d", msg.Offset))
	req.Header.Set("X-Kmq-Logid", fmt.Sprintf("%s", rmsg.LogId))
	req.Header.Set("X-Kmq-Timestamp", fmt.Sprintf("%d", rmsg.TimeStamp))
	req.Header.Set("Meilishuo", "uid:0;ip:0.0.0.0;v:0;master:0")
	tsrpc = time.Now()
	resp, err := client.Do(req)
	terpc = time.Now()

	var op_time int64
	if this.Serializer == "json" {
		op_time = time.Now().UnixNano()/1000000 - rmsg.TimeStamp
	} else {
		op_time = terpc.Sub(tsrpc).Nanoseconds() / 1000000
	}

	suc := true
	if nil == err {
		defer resp.Body.Close()
		suc = (resp.StatusCode == 200)
		if suc == false {
			rbody, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				rbody = []byte{}
			}
			glog.Errorf("delivery failed,[retry_times:%d][topic:%s][partition:%d][offset:%d][msg:%s][url:%s][http_code:%d][cost:%vms][response_body:%s][current:%s]", retry_times, msg.Topic, msg.Partition, msg.Offset, rmsg.Data, this.Callback.Url, resp.StatusCode, fmt.Sprintf("%.2f", terpc.Sub(tsrpc).Seconds()*1000), rbody, terpc)
		} else {
			if this.Serializer == "json" {
				this.CommitNewTracker(&rmsg, msg)
			}

			// consume all data in response body to enable connection reusing
			discardBody(resp.Body)
		}
	} else {
		glog.Errorf("delivery failed,[retry_times:%d][topic:%s][partition:%d][offset:%d][msg:%s][url:%s][error:%s][cost:%vms][total_cost:%vms][current:%s]", retry_times, msg.Topic, msg.Partition, msg.Offset, rmsg.Data, this.Callback.Url, err.Error(), fmt.Sprintf("%.2f", terpc.Sub(tsrpc).Seconds()*1000), fmt.Sprintf("%.2f", terpc.Sub(tsrpc).Seconds()*1000), op_time, terpc)
		suc = false
	}
	if switcher := this.LogSamplingCollect(); switcher != false {
		glog.Infof("log sampling,[url:%s][retry_times:%d][topic:%s][partition:%d][offset:%d][cost:%v][total_cost:%v][content-type:%s][current_time:%s]", this.Callback.Url, retry_times, msg.Topic, msg.Partition, msg.Offset, fmt.Sprintf("%.2f", terpc.Sub(tsrpc).Seconds()*1000), op_time, rmsg.ContentType, terpc)
	}

	return suc, err
}

func (this *Worker) Closed() bool {
	return this.Consumer.Closed()
}

func (this *Worker) Close() {
	if err := this.Consumer.Close(); err != nil {
		glog.Errorln("Error closing consumers", err)
	}
}

func (this *Worker) GetWorkerTracker() OffsetMap {
	return this.Tracker
}

func (this *Worker) CommitNewTracker(rmsg *RMSG, msg *Msg) (err error) {
	value, ok := this.Tracker[fmt.Sprintf("%d", msg.Partition)]
	if !ok {
		this.Tracker[fmt.Sprintf("%d", msg.Partition)] = &TrackerData{
			LastRecordOpTime: 0,
			CurrRecordOpTime: rmsg.TimeStamp,
			LogId:            rmsg.LogId,
			Offset:           msg.Offset,
		}
		return nil
	} else if ok && rmsg.TimeStamp >= value.CurrRecordOpTime {
		this.Tracker[fmt.Sprintf("%d", msg.Partition)] = &TrackerData{
			LastRecordOpTime: value.CurrRecordOpTime,
			CurrRecordOpTime: rmsg.TimeStamp,
			LogId:            rmsg.LogId,
			Offset:           msg.Offset,
		}
		return nil
	}
	return nil
}

func (this *Worker) LogSamplingCollect() bool {
	r := rand.Intn(this.LogCollectRatio)
	if r == 1 {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package zerolog

import (
	"io"
	"sync"
)

// LevelWriter defines as interface a writer may implement in order
// to receive level information with payload.
type LevelWriter interface {
	io.Writer
	WriteLevel(level Level, p []byte) (n int, err error)
}

type levelWriterAdapter struct {
	io.Writer
}

func (lw levelWriterAdapter) WriteLevel(l Level, p []byte) (n int, err error) {
	return lw.Write(p)
}

type syncWriter struct {
	mu sync.Mutex
	lw LevelWriter
}

// SyncWriter wraps w so that each call to Write is synchronized with a mutex.
// This syncer can be the call to writer's Write method is not thread safe.
// Note that os.File Write operation is using write() syscall which is supposed
// to be thread-safe on POSIX systems. So there is no need to use this with
// os.File on such systems as zerolog guaranties to issue a single Write call
// per log event.
func SyncWriter(w io.Writer) io.Writer {
	if lw, ok := w.(LevelWriter); ok {
		return &syncWriter{lw: lw}
	}
	return &syncWriter{lw: levelWriterAdapter{w}}
}

// Write implements the io.Writer interface.
func (s *syncWriter) Write(p []byte) (n int, err error) {
	s.mu.Lock()
	n, err = s.lw.Write(p)
	s.mu.Unlock()
	return
}

// WriteLevel implements the LevelWriter interface.
func (s *syncWriter) WriteLevel(l Level, p []byte) (n int, err error) {
	s.mu.Lock()
	n, err = s.lw.WriteLevel(l, p)
	s.mu.Unlock()
	return
}

type multiLevelWriter struct {
	writers []LevelWriter
}

func (t multiLevelWriter) Write(p []byte) (n int, err error) {
	for _, w := range t.writers {
		n, err = w.Write(p)
		if err != nil {
			return
		}
		if n != len(p) {
			err = io.ErrShortWrite
			return
		}
	}
	return len(p), nil
}

func (t multiLevelWriter) WriteLevel(l Level, p []byte) (n int, err error) {
	for _, w := range t.writers {
		n, err = w.WriteLevel(l, p)
		if err != nil {
			return
		}
		if n != len(p) {
			err = io.ErrShortWrite
			return
		}
	}
	return len(p), nil
}

// MultiLevelWriter creates a writer that duplicates its writes to all the
// provided writers, similar to the Unix tee(1) command. If some writers
// implement LevelWriter, their WriteLevel method will be used instead of Write.
func MultiLevelWriter(writers ...io.Writer) LevelWriter {
	lwriters := make([]LevelWriter, 0, len(writers))
	for _, w := range writers {
		if lw, ok := w.(LevelWriter); ok {
			lwriters = append(lwriters, lw)
		} else {
			lwriters = append(lwriters, levelWriterAdapter{w})
		}
	}
	return multiLevelWriter{lwriters}
}
<commit_msg>Use defer when expected<commit_after>package zerolog

import (
	"io"
	"sync"
)

// LevelWriter defines as interface a writer may implement in order
// to receive level information with payload.
type LevelWriter interface {
	io.Writer
	WriteLevel(level Level, p []byte) (n int, err error)
}

type levelWriterAdapter struct {
	io.Writer
}

func (lw levelWriterAdapter) WriteLevel(l Level, p []byte) (n int, err error) {
	return lw.Write(p)
}

type syncWriter struct {
	mu sync.Mutex
	lw LevelWriter
}

// SyncWriter wraps w so that each call to Write is synchronized with a mutex.
// This syncer can be the call to writer's Write method is not thread safe.
// Note that os.File Write operation is using write() syscall which is supposed
// to be thread-safe on POSIX systems. So there is no need to use this with
// os.File on such systems as zerolog guaranties to issue a single Write call
// per log event.
func SyncWriter(w io.Writer) io.Writer {
	if lw, ok := w.(LevelWriter); ok {
		return &syncWriter{lw: lw}
	}
	return &syncWriter{lw: levelWriterAdapter{w}}
}

// Write implements the io.Writer interface.
func (s *syncWriter) Write(p []byte) (n int, err error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.lw.Write(p)
}

// WriteLevel implements the LevelWriter interface.
func (s *syncWriter) WriteLevel(l Level, p []byte) (n int, err error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.lw.WriteLevel(l, p)
}

type multiLevelWriter struct {
	writers []LevelWriter
}

func (t multiLevelWriter) Write(p []byte) (n int, err error) {
	for _, w := range t.writers {
		n, err = w.Write(p)
		if err != nil {
			return
		}
		if n != len(p) {
			err = io.ErrShortWrite
			return
		}
	}
	return len(p), nil
}

func (t multiLevelWriter) WriteLevel(l Level, p []byte) (n int, err error) {
	for _, w := range t.writers {
		n, err = w.WriteLevel(l, p)
		if err != nil {
			return
		}
		if n != len(p) {
			err = io.ErrShortWrite
			return
		}
	}
	return len(p), nil
}

// MultiLevelWriter creates a writer that duplicates its writes to all the
// provided writers, similar to the Unix tee(1) command. If some writers
// implement LevelWriter, their WriteLevel method will be used instead of Write.
func MultiLevelWriter(writers ...io.Writer) LevelWriter {
	lwriters := make([]LevelWriter, 0, len(writers))
	for _, w := range writers {
		if lw, ok := w.(LevelWriter); ok {
			lwriters = append(lwriters, lw)
		} else {
			lwriters = append(lwriters, levelWriterAdapter{w})
		}
	}
	return multiLevelWriter{lwriters}
}
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

//go:generate msgp -io=false

package core

// DataUpAppReq represents the actual payloads sent to application on uplink
type DataUpAppReq struct {
	Payload  []byte        `msg:"payload" json:"payload"`
	Metadata []AppMetadata `msg:"metadata" json:"metadata"`
}

// OTAAAppReq are used to notify application of an accepted OTAA
type OTAAAppReq struct {
	Metadata []AppMetadata `msg:"metadata" json:"metadata"`
}

// AppMetadata represents gathered metadata that are sent to gateways
type AppMetadata struct {
	Frequency  float32 `msg:"frequency" json:"frequency"`
	DataRate   string  `msg:"datarate" json:"datarate"`
	CodingRate string  `msg:"coderate" json:"coderate"`
	Timestamp  uint32  `msg:"timestamp" json:"timestamp"`
	Time       string  `msg:"time" json:"time"`
	Rssi       int32   `msg:"rssi" json:"rssi"`
	Lsnr       float32 `msg:"lsnr" json:"lsnr"`
	RFChain    uint32  `msg:"rfchain" json:"rfchain"`
	CRCStatus  int32   `msg:"crc" json:"crc"`
	Modulation string  `msg:"modulation" json:"modulation"`
	Altitude   int32   `msg:"altitude" json:"altitude"`
	Longitude  float32 `msg:"longitude" json:"longitude"`
	Latitude   float32 `msg:"latitude" json:"latitude"`
}

// DataDownAppReq represents downlink messages sent by applications
type DataDownAppReq struct {
	Payload []byte `msg:"payload" json:"payload"`
	TTL     string `msg:"ttl" json:"ttl"`
}

// ABPSubAppReq defines the shape of the request made by an application to the handler
type ABPSubAppReq struct {
	DevAddr string `msg:"devaddr" json:"devaddr"`
	NwkSKey string `msg:"nwkskey" json:"nwkskey"`
	AppSKey string `msg:"appskey" json:"appskey"`
}

// AuthBrokerClient gathers both BrokerClient & BrokerManagerClient interfaces
type AuthBrokerClient interface {
	BrokerClient
	BrokerManagerClient
}

// AuthHandlerClient gathers both HandlerClient & HandlerManagerClient interfaces
type AuthHandlerClient interface {
	HandlerClient
	HandlerManagerClient
}
<commit_msg>Renamed coderate to codingrate<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

//go:generate msgp -io=false

package core

// DataUpAppReq represents the actual payloads sent to application on uplink
type DataUpAppReq struct {
	Payload  []byte        `msg:"payload" json:"payload"`
	Metadata []AppMetadata `msg:"metadata" json:"metadata"`
}

// OTAAAppReq are used to notify application of an accepted OTAA
type OTAAAppReq struct {
	Metadata []AppMetadata `msg:"metadata" json:"metadata"`
}

// AppMetadata represents gathered metadata that are sent to gateways
type AppMetadata struct {
	Frequency  float32 `msg:"frequency" json:"frequency"`
	DataRate   string  `msg:"datarate" json:"datarate"`
	CodingRate string  `msg:"codingrate" json:"codingrate"`
	Timestamp  uint32  `msg:"timestamp" json:"timestamp"`
	Time       string  `msg:"time" json:"time"`
	Rssi       int32   `msg:"rssi" json:"rssi"`
	Lsnr       float32 `msg:"lsnr" json:"lsnr"`
	RFChain    uint32  `msg:"rfchain" json:"rfchain"`
	CRCStatus  int32   `msg:"crc" json:"crc"`
	Modulation string  `msg:"modulation" json:"modulation"`
	Altitude   int32   `msg:"altitude" json:"altitude"`
	Longitude  float32 `msg:"longitude" json:"longitude"`
	Latitude   float32 `msg:"latitude" json:"latitude"`
}

// DataDownAppReq represents downlink messages sent by applications
type DataDownAppReq struct {
	Payload []byte `msg:"payload" json:"payload"`
	TTL     string `msg:"ttl" json:"ttl"`
}

// ABPSubAppReq defines the shape of the request made by an application to the handler
type ABPSubAppReq struct {
	DevAddr string `msg:"devaddr" json:"devaddr"`
	NwkSKey string `msg:"nwkskey" json:"nwkskey"`
	AppSKey string `msg:"appskey" json:"appskey"`
}

// AuthBrokerClient gathers both BrokerClient & BrokerManagerClient interfaces
type AuthBrokerClient interface {
	BrokerClient
	BrokerManagerClient
}

// AuthHandlerClient gathers both HandlerClient & HandlerManagerClient interfaces
type AuthHandlerClient interface {
	HandlerClient
	HandlerManagerClient
}
<|endoftext|>


<commit_before>package stackengine

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"path"
	"strings"
)

// Client is an empty wrapper around the StackEngine client
type Client struct {
	client    *http.Client
	token     string
	base      string
	transport *http.Transport
}

// NewStackEngineClient returns a client object with connection information.
func NewStackEngineClient(nodes []string, scheme, cert, key, caCert string, authToken string) (*Client, error) {
	var (
		err  error
		host string
	)

	if len(nodes) > 0 {
		host = nodes[0]
	} else {
		host = "127.0.0.1:8443"
	}

	base := scheme + "://" + host

	tlsConfig := &tls.Config{}
	if cert != "" && key != "" {
		clientCert, err := tls.LoadX509KeyPair(cert, key)
		if err != nil {
			return nil, err
		}
		tlsConfig.Certificates = []tls.Certificate{clientCert}
		tlsConfig.BuildNameToCertificate()
	}
	if caCert != "" {
		ca, err := ioutil.ReadFile(caCert)
		if err != nil {
			return nil, err
		}
		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(ca)
		tlsConfig.RootCAs = caCertPool
	}
	tlsConfig.InsecureSkipVerify = true
	transport := &http.Transport{
		TLSClientConfig: tlsConfig,
	}

	client := &http.Client{Transport: transport}

	return &Client{client, authToken, base, transport}, err
}

// KVPair is used to represent a single K/V entry
type KVPair struct {
	Key         string
	CreateIndex uint64
	ModifyIndex uint64
	LockIndex   uint64
	Flags       uint64
	Value       []byte
	Session     string
}

// GetValues queries redis for keys prefixed by prefix.
func (c *Client) GetValues(keys []string) (map[string]string, error) {
	vars := make(map[string]string)
	var pairs []KVPair

	for _, key := range keys {
		key := strings.TrimPrefix(key, "/")

		uri := c.base + "/v1/kv/" + key + "?recurse"
		req, err := http.NewRequest("GET", uri, nil)

		bearer := "Bearer " + c.token

		fmt.Println("URI: ", uri, bearer)

		req.Header.Add("Authorization", bearer)
		parseFormErr := req.ParseForm()
		if parseFormErr != nil {
			fmt.Println(parseFormErr)
		}

		// Fetch Request
		resp, err := c.client.Do(req)

		if err != nil {
			fmt.Println("Failure : ", err)
			return nil, err
		}

		// Read Response Body
		respBody, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			fmt.Println("Error reading http response: ", err)
		}

		// Display Results
		fmt.Println("response Status : ", resp.Status)
		fmt.Println("response Headers : ", resp.Header)
		fmt.Println("response Body : ", string(respBody))

		err = json.Unmarshal(respBody, &pairs)

		if err != nil {
			return vars, err
		}
		for _, p := range pairs {
			vars[path.Join("/", p.Key)] = string(p.Value)
		}
	}

	fmt.Println(vars)

	return vars, nil
}

type watchResponse struct {
	waitIndex uint64
	err       error
}

func (c *Client) WatchPrefix(prefix string, waitIndex uint64, stopChan chan bool) (uint64, error) {
	respChan := make(chan watchResponse)
	go func() {
		//opts := api.QueryOptions{
		//	WaitIndex: waitIndex,
		//}
		//_, meta, err := c.client.List(prefix, &opts)
		//respChan <- watchResponse{meta.LastIndex, err}
	}()
	for {
		select {
		case <-stopChan:
			return waitIndex, nil
		case r := <-respChan:
			return r.waitIndex, r.err
		}
	}
}
<commit_msg>Remove debug prints<commit_after>package stackengine

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"path"
	"strings"
)

// Client is an empty wrapper around the StackEngine client
type Client struct {
	client    *http.Client
	token     string
	base      string
	transport *http.Transport
}

// NewStackEngineClient returns a client object with connection information.
func NewStackEngineClient(nodes []string, scheme, cert, key, caCert string, authToken string) (*Client, error) {
	var (
		err  error
		host string
	)

	if len(nodes) > 0 {
		host = nodes[0]
	} else {
		host = "127.0.0.1:8443"
	}

	base := scheme + "://" + host

	tlsConfig := &tls.Config{}
	if cert != "" && key != "" {
		clientCert, err := tls.LoadX509KeyPair(cert, key)
		if err != nil {
			return nil, err
		}
		tlsConfig.Certificates = []tls.Certificate{clientCert}
		tlsConfig.BuildNameToCertificate()
	}
	if caCert != "" {
		ca, err := ioutil.ReadFile(caCert)
		if err != nil {
			return nil, err
		}
		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(ca)
		tlsConfig.RootCAs = caCertPool
	}
	tlsConfig.InsecureSkipVerify = true
	transport := &http.Transport{
		TLSClientConfig: tlsConfig,
	}

	client := &http.Client{Transport: transport}

	return &Client{client, authToken, base, transport}, err
}

// KVPair is used to represent a single K/V entry
type KVPair struct {
	Key         string
	CreateIndex uint64
	ModifyIndex uint64
	LockIndex   uint64
	Flags       uint64
	Value       []byte
	Session     string
}

// GetValues queries redis for keys prefixed by prefix.
func (c *Client) GetValues(keys []string) (map[string]string, error) {
	vars := make(map[string]string)
	var pairs []KVPair

	for _, key := range keys {
		key := strings.TrimPrefix(key, "/")

		uri := c.base + "/v1/kv/" + key + "?recurse"
		req, err := http.NewRequest("GET", uri, nil)

		bearer := "Bearer " + c.token

		fmt.Println("URI: ", uri, bearer)

		req.Header.Add("Authorization", bearer)
		parseFormErr := req.ParseForm()
		if parseFormErr != nil {
			fmt.Println(parseFormErr)
		}

		// Fetch Request
		resp, err := c.client.Do(req)

		if err != nil {
			fmt.Println("Failure : ", err)
			return nil, err
		}

		// Read Response Body
		respBody, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			fmt.Println("Error reading http response: ", err)
		}

		err = json.Unmarshal(respBody, &pairs)

		if err != nil {
			return vars, err
		}
		for _, p := range pairs {
			vars[path.Join("/", p.Key)] = string(p.Value)
		}
	}

	fmt.Println(vars)

	return vars, nil
}

type watchResponse struct {
	waitIndex uint64
	err       error
}

func (c *Client) WatchPrefix(prefix string, waitIndex uint64, stopChan chan bool) (uint64, error) {
	<-stopChan
	return 0, nil
}
<|endoftext|>


<commit_before>package core

import (
	"database/sql"
	"encoding/json"
	"strconv"
	"time"

	"github.com/af83/edwig/logger"
	"github.com/af83/edwig/model"
)

type ReferentialId string
type ReferentialSlug string

const (
	REFERENTIAL_SETTING_MODEL_RELOAD_AT = "model.reload_at"
)

type Referential struct {
	model.ClockConsumer

	id   ReferentialId
	slug ReferentialSlug

	Settings map[string]string `json:"Settings,omitempty"`

	collectManager CollectManagerInterface
	manager        Referentials
	model          *model.MemoryModel
	modelGuardian  *ModelGuardian
	partners       Partners
	startedAt      time.Time
	nextReloadAt   time.Time
}

type Referentials interface {
	model.Startable

	New(slug ReferentialSlug) *Referential
	Find(id ReferentialId) *Referential
	FindBySlug(slug ReferentialSlug) *Referential
	FindAll() []*Referential
	Save(stopArea *Referential) bool
	Delete(stopArea *Referential) bool
	Load() error
}

var referentials = NewMemoryReferentials()

type APIReferential struct {
	id       ReferentialId
	Slug     ReferentialSlug   `json:"Slug,omitempty"`
	Errors   Errors            `json:"Errors,omitempty"`
	Settings map[string]string `json:"Settings,omitempty"`

	manager Referentials
}

func (referential *APIReferential) Id() ReferentialId {
	return referential.id
}

func (referential *APIReferential) Validate() bool {
	referential.Errors = NewErrors()

	if referential.Slug == "" {
		referential.Errors.Add("Slug", ERROR_BLANK)
	}

	// Check Slug uniqueness
	for _, existingReferential := range referential.manager.FindAll() {
		if existingReferential.id != referential.Id() {
			if referential.Slug == existingReferential.slug {
				referential.Errors.Add("Slug", ERROR_UNIQUE)
			}
		}
	}

	return len(referential.Errors) == 0
}

func (referential *Referential) Id() ReferentialId {
	return referential.id
}

func (referential *Referential) Slug() ReferentialSlug {
	return referential.slug
}

func (referential *Referential) Setting(key string) string {
	return referential.Settings[key]
}

func (referential *Referential) StartedAt() time.Time {
	return referential.startedAt
}

// WIP: Interface ?
func (referential *Referential) CollectManager() CollectManagerInterface {
	return referential.collectManager
}

func (referential *Referential) Model() model.Model {
	return referential.model
}

func (referential *Referential) ModelGuardian() *ModelGuardian {
	return referential.modelGuardian
}

func (referential *Referential) Partners() Partners {
	return referential.partners
}

func (referential *Referential) Start() {
	referential.startedAt = referential.Clock().Now()

	referential.partners.Start()
	referential.modelGuardian.Start()
}

func (referential *Referential) Stop() {
	referential.partners.Stop()
	referential.modelGuardian.Stop()
}

func (referential *Referential) Save() (ok bool) {
	ok = referential.manager.Save(referential)
	return
}

func (referential *Referential) NewTransaction() *model.Transaction {
	return model.NewTransaction(referential.model)
}

func (referential *Referential) MarshalJSON() ([]byte, error) {
	type Alias Referential
	aux := struct {
		Id           ReferentialId
		Slug         ReferentialSlug
		NextReloadAt *time.Time `json:",omitempty"`
		Partners     Partners   `json:",omitempty"`
		*Alias
	}{
		Id:    referential.id,
		Slug:  referential.slug,
		Alias: (*Alias)(referential),
	}

	if !referential.nextReloadAt.IsZero() {
		aux.NextReloadAt = &referential.nextReloadAt
	}
	if !referential.partners.IsEmpty() {
		aux.Partners = referential.partners
	}

	return json.Marshal(&aux)
}

func (referential *Referential) Definition() *APIReferential {
	settings := map[string]string{}
	for k, v := range referential.Settings {
		settings[k] = v
	}

	return &APIReferential{
		id:       referential.id,
		Slug:     referential.slug,
		Settings: settings,
		Errors:   NewErrors(),
		manager:  referential.manager,
	}
}

func (referential *Referential) SetDefinition(apiReferential *APIReferential) {
	initialReloadAt := referential.Setting(REFERENTIAL_SETTING_MODEL_RELOAD_AT)

	referential.slug = apiReferential.Slug
	referential.Settings = apiReferential.Settings

	if initialReloadAt != referential.Setting(REFERENTIAL_SETTING_MODEL_RELOAD_AT) {
		referential.setNextReloadAt()
	}
}

func (referential *Referential) NextReloadAt() time.Time {
	return referential.nextReloadAt
}

func (referential *Referential) ReloadModel() {
	logger.Log.Printf("Reset Model")
	referential.model = referential.model.Clone()
	referential.setNextReloadAt()
}

func (referential *Referential) setNextReloadAt() {
	reloadHour := referential.Setting(REFERENTIAL_SETTING_MODEL_RELOAD_AT)
	hour, minute := 4, 0

	if len(reloadHour) == 5 {
		hour, _ = strconv.Atoi(reloadHour[0:2])
		minute, _ = strconv.Atoi(reloadHour[3:5])
	}
	now := referential.Clock().Now()

	day := now.Day()
	if now.Hour() >= hour && now.Minute() >= minute {
		day += 1
	}

	referential.nextReloadAt = time.Date(now.Year(), now.Month(), day, hour, minute, 0, 0, now.Location())
	logger.Log.Printf("Next reload at: %v", referential.nextReloadAt)
}

func (referential *Referential) Load() {
	referential.Partners().Load()
	referential.model.Load(string(referential.id))
}

type MemoryReferentials struct {
	model.UUIDConsumer

	byId map[ReferentialId]*Referential
}

func NewMemoryReferentials() *MemoryReferentials {
	return &MemoryReferentials{
		byId: make(map[ReferentialId]*Referential),
	}
}

func CurrentReferentials() Referentials {
	return referentials
}

func (manager *MemoryReferentials) New(slug ReferentialSlug) *Referential {
	referential := manager.new()
	referential.slug = slug
	return referential
}

func (manager *MemoryReferentials) new() *Referential {
	model := model.NewMemoryModel()

	referential := &Referential{
		manager:  manager,
		model:    model,
		Settings: make(map[string]string),
	}

	referential.partners = NewPartnerManager(referential)
	referential.collectManager = NewCollectManager(referential)

	referential.modelGuardian = NewModelGuardian(referential)
	referential.setNextReloadAt()

	return referential
}

func (manager *MemoryReferentials) Find(id ReferentialId) *Referential {
	referential, _ := manager.byId[id]
	return referential
}

func (manager *MemoryReferentials) FindBySlug(slug ReferentialSlug) *Referential {
	for _, referential := range manager.byId {
		if referential.slug == slug {
			return referential
		}
	}
	return nil
}

func (manager *MemoryReferentials) FindAll() (referentials []*Referential) {
	if len(manager.byId) == 0 {
		return []*Referential{}
	}
	for _, referential := range manager.byId {
		referentials = append(referentials, referential)
	}
	return
}

func (manager *MemoryReferentials) Save(referential *Referential) bool {
	if referential.id == "" {
		referential.id = ReferentialId(manager.NewUUID())
	}
	referential.manager = manager
	referential.collectManager.HandleStopAreaUpdateEvent(model.NewStopAreaUpdateManager(referential))
	manager.byId[referential.id] = referential
	return true
}

func (manager *MemoryReferentials) Delete(referential *Referential) bool {
	delete(manager.byId, referential.id)
	return true
}

func (manager *MemoryReferentials) Load() error {
	var selectReferentials []struct {
		Referential_id string
		Slug           string
		Settings       sql.NullString
	}

	_, err := model.Database.Select(&selectReferentials, "select * from referentials")
	if err != nil {
		return err
	}

	for _, r := range selectReferentials {
		referential := manager.new()
		referential.id = ReferentialId(r.Referential_id)
		referential.slug = ReferentialSlug(r.Slug)

		if r.Settings.Valid && len(r.Settings.String) > 0 {
			if err = json.Unmarshal([]byte(r.Settings.String), &referential.Settings); err != nil {
				return err
			}
		}

		manager.Save(referential)
		referential.Load()
	}

	logger.Log.Debugf("Loaded Referentials from database")
	return nil
}

func (manager *MemoryReferentials) Start() {
	for _, referential := range manager.byId {
		referential.Start()
	}
}

type ReferentialsConsumer struct {
	referentials Referentials
}

func (consumer *ReferentialsConsumer) SetReferentials(referentials Referentials) {
	consumer.referentials = referentials
}

func (consumer *ReferentialsConsumer) CurrentReferentials() Referentials {
	if consumer.referentials == nil {
		consumer.referentials = CurrentReferentials()
	}
	return consumer.referentials
}
<commit_msg>Add SituationUpdateManager to referential ColelctManager<commit_after>package core

import (
	"database/sql"
	"encoding/json"
	"strconv"
	"time"

	"github.com/af83/edwig/logger"
	"github.com/af83/edwig/model"
)

type ReferentialId string
type ReferentialSlug string

const (
	REFERENTIAL_SETTING_MODEL_RELOAD_AT = "model.reload_at"
)

type Referential struct {
	model.ClockConsumer

	id   ReferentialId
	slug ReferentialSlug

	Settings map[string]string `json:"Settings,omitempty"`

	collectManager CollectManagerInterface
	manager        Referentials
	model          *model.MemoryModel
	modelGuardian  *ModelGuardian
	partners       Partners
	startedAt      time.Time
	nextReloadAt   time.Time
}

type Referentials interface {
	model.Startable

	New(slug ReferentialSlug) *Referential
	Find(id ReferentialId) *Referential
	FindBySlug(slug ReferentialSlug) *Referential
	FindAll() []*Referential
	Save(stopArea *Referential) bool
	Delete(stopArea *Referential) bool
	Load() error
}

var referentials = NewMemoryReferentials()

type APIReferential struct {
	id       ReferentialId
	Slug     ReferentialSlug   `json:"Slug,omitempty"`
	Errors   Errors            `json:"Errors,omitempty"`
	Settings map[string]string `json:"Settings,omitempty"`

	manager Referentials
}

func (referential *APIReferential) Id() ReferentialId {
	return referential.id
}

func (referential *APIReferential) Validate() bool {
	referential.Errors = NewErrors()

	if referential.Slug == "" {
		referential.Errors.Add("Slug", ERROR_BLANK)
	}

	// Check Slug uniqueness
	for _, existingReferential := range referential.manager.FindAll() {
		if existingReferential.id != referential.Id() {
			if referential.Slug == existingReferential.slug {
				referential.Errors.Add("Slug", ERROR_UNIQUE)
			}
		}
	}

	return len(referential.Errors) == 0
}

func (referential *Referential) Id() ReferentialId {
	return referential.id
}

func (referential *Referential) Slug() ReferentialSlug {
	return referential.slug
}

func (referential *Referential) Setting(key string) string {
	return referential.Settings[key]
}

func (referential *Referential) StartedAt() time.Time {
	return referential.startedAt
}

// WIP: Interface ?
func (referential *Referential) CollectManager() CollectManagerInterface {
	return referential.collectManager
}

func (referential *Referential) Model() model.Model {
	return referential.model
}

func (referential *Referential) ModelGuardian() *ModelGuardian {
	return referential.modelGuardian
}

func (referential *Referential) Partners() Partners {
	return referential.partners
}

func (referential *Referential) Start() {
	referential.startedAt = referential.Clock().Now()

	referential.partners.Start()
	referential.modelGuardian.Start()
}

func (referential *Referential) Stop() {
	referential.partners.Stop()
	referential.modelGuardian.Stop()
}

func (referential *Referential) Save() (ok bool) {
	ok = referential.manager.Save(referential)
	return
}

func (referential *Referential) NewTransaction() *model.Transaction {
	return model.NewTransaction(referential.model)
}

func (referential *Referential) MarshalJSON() ([]byte, error) {
	type Alias Referential
	aux := struct {
		Id           ReferentialId
		Slug         ReferentialSlug
		NextReloadAt *time.Time `json:",omitempty"`
		Partners     Partners   `json:",omitempty"`
		*Alias
	}{
		Id:    referential.id,
		Slug:  referential.slug,
		Alias: (*Alias)(referential),
	}

	if !referential.nextReloadAt.IsZero() {
		aux.NextReloadAt = &referential.nextReloadAt
	}
	if !referential.partners.IsEmpty() {
		aux.Partners = referential.partners
	}

	return json.Marshal(&aux)
}

func (referential *Referential) Definition() *APIReferential {
	settings := map[string]string{}
	for k, v := range referential.Settings {
		settings[k] = v
	}

	return &APIReferential{
		id:       referential.id,
		Slug:     referential.slug,
		Settings: settings,
		Errors:   NewErrors(),
		manager:  referential.manager,
	}
}

func (referential *Referential) SetDefinition(apiReferential *APIReferential) {
	initialReloadAt := referential.Setting(REFERENTIAL_SETTING_MODEL_RELOAD_AT)

	referential.slug = apiReferential.Slug
	referential.Settings = apiReferential.Settings

	if initialReloadAt != referential.Setting(REFERENTIAL_SETTING_MODEL_RELOAD_AT) {
		referential.setNextReloadAt()
	}
}

func (referential *Referential) NextReloadAt() time.Time {
	return referential.nextReloadAt
}

func (referential *Referential) ReloadModel() {
	logger.Log.Printf("Reset Model")
	referential.model = referential.model.Clone()
	referential.setNextReloadAt()
}

func (referential *Referential) setNextReloadAt() {
	reloadHour := referential.Setting(REFERENTIAL_SETTING_MODEL_RELOAD_AT)
	hour, minute := 4, 0

	if len(reloadHour) == 5 {
		hour, _ = strconv.Atoi(reloadHour[0:2])
		minute, _ = strconv.Atoi(reloadHour[3:5])
	}
	now := referential.Clock().Now()

	day := now.Day()
	if now.Hour() >= hour && now.Minute() >= minute {
		day += 1
	}

	referential.nextReloadAt = time.Date(now.Year(), now.Month(), day, hour, minute, 0, 0, now.Location())
	logger.Log.Printf("Next reload at: %v", referential.nextReloadAt)
}

func (referential *Referential) Load() {
	referential.Partners().Load()
	referential.model.Load(string(referential.id))
}

type MemoryReferentials struct {
	model.UUIDConsumer

	byId map[ReferentialId]*Referential
}

func NewMemoryReferentials() *MemoryReferentials {
	return &MemoryReferentials{
		byId: make(map[ReferentialId]*Referential),
	}
}

func CurrentReferentials() Referentials {
	return referentials
}

func (manager *MemoryReferentials) New(slug ReferentialSlug) *Referential {
	referential := manager.new()
	referential.slug = slug
	return referential
}

func (manager *MemoryReferentials) new() *Referential {
	model := model.NewMemoryModel()

	referential := &Referential{
		manager:  manager,
		model:    model,
		Settings: make(map[string]string),
	}

	referential.partners = NewPartnerManager(referential)
	referential.collectManager = NewCollectManager(referential)

	referential.modelGuardian = NewModelGuardian(referential)
	referential.setNextReloadAt()

	return referential
}

func (manager *MemoryReferentials) Find(id ReferentialId) *Referential {
	referential, _ := manager.byId[id]
	return referential
}

func (manager *MemoryReferentials) FindBySlug(slug ReferentialSlug) *Referential {
	for _, referential := range manager.byId {
		if referential.slug == slug {
			return referential
		}
	}
	return nil
}

func (manager *MemoryReferentials) FindAll() (referentials []*Referential) {
	if len(manager.byId) == 0 {
		return []*Referential{}
	}
	for _, referential := range manager.byId {
		referentials = append(referentials, referential)
	}
	return
}

func (manager *MemoryReferentials) Save(referential *Referential) bool {
	if referential.id == "" {
		referential.id = ReferentialId(manager.NewUUID())
	}
	referential.manager = manager
	referential.collectManager.HandleStopAreaUpdateEvent(model.NewStopAreaUpdateManager(referential))
	referential.collectManager.HandleSituationUpdateEvent(model.NewSituationUpdateManager(referential))
	manager.byId[referential.id] = referential
	return true
}

func (manager *MemoryReferentials) Delete(referential *Referential) bool {
	delete(manager.byId, referential.id)
	return true
}

func (manager *MemoryReferentials) Load() error {
	var selectReferentials []struct {
		Referential_id string
		Slug           string
		Settings       sql.NullString
	}

	_, err := model.Database.Select(&selectReferentials, "select * from referentials")
	if err != nil {
		return err
	}

	for _, r := range selectReferentials {
		referential := manager.new()
		referential.id = ReferentialId(r.Referential_id)
		referential.slug = ReferentialSlug(r.Slug)

		if r.Settings.Valid && len(r.Settings.String) > 0 {
			if err = json.Unmarshal([]byte(r.Settings.String), &referential.Settings); err != nil {
				return err
			}
		}

		manager.Save(referential)
		referential.Load()
	}

	logger.Log.Debugf("Loaded Referentials from database")
	return nil
}

func (manager *MemoryReferentials) Start() {
	for _, referential := range manager.byId {
		referential.Start()
	}
}

type ReferentialsConsumer struct {
	referentials Referentials
}

func (consumer *ReferentialsConsumer) SetReferentials(referentials Referentials) {
	consumer.referentials = referentials
}

func (consumer *ReferentialsConsumer) CurrentReferentials() Referentials {
	if consumer.referentials == nil {
		consumer.referentials = CurrentReferentials()
	}
	return consumer.referentials
}
<|endoftext|>


<commit_before>// Copyright 2022 Tomas Machalek <tomas.machalek@gmail.com>
// Copyright 2022 Charles University, Faculty of Arts,
//                Institute of the Czech National Corpus
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mysql

import (
	"database/sql"
	"fmt"
	"log"
	"strings"

	"github.com/czcorpus/vert-tagextract/v2/db"
)

// dropExisting drops existing tables/views.
// It is safe to call this even if one or more of these does not exist.
// Please note that the groupedCorpusName argument represents a derived corpus name
// which is able to group multipe (aligned) corpora together.E.g. 'intercorp_v13_cs'
// and 'intercorp_v13_en' will likely groupedName 'intercorp_v13'. For single corpora,
// the groupedCorpusName is the same as the original one.
func dropExisting(database *sql.DB, groupedCorpusName string) error {
	log.Print("Attempting to drop possible existing tables and views...")
	var err error
	_, err = database.Exec("DROP TABLE IF EXISTS cache")
	if err != nil {
		return fmt.Errorf("failed to drop table 'cache': %s", err)
	}
	_, err = database.Exec(fmt.Sprintf("DROP VIEW IF EXISTS %s_bibliography", groupedCorpusName))
	if err != nil {
		return fmt.Errorf("failed to drop view '%s_bibliography': %s", groupedCorpusName, err)
	}
	_, err = database.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s_liveattrs_entry", groupedCorpusName))
	if err != nil {
		return fmt.Errorf("failed to drop table '%s_liveattrs_entry': %s", groupedCorpusName, err)
	}
	_, err = database.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s_colcounts", groupedCorpusName))
	if err != nil {
		return fmt.Errorf("failed to drop table '%s_colcounts': %s", groupedCorpusName, err)
	}
	log.Print("...DONE")
	return nil
}

// generateColNames produces a list of structural
// attribute names as used in database
// (i.e. [structname]_[attr_name]) out of lists
// of structural attributes defined in the configuration.
// (see _examples/*.json)
func generateColNames(structures map[string][]string) []string {
	numAttrs := 0
	for _, v := range structures {
		numAttrs += len(v)
	}
	ans := make([]string, numAttrs)
	i := 0
	for k, v := range structures {
		for _, a := range v {
			ans[i] = fmt.Sprintf("%s_%s", k, a)
			i++
		}
	}
	return ans
}

// generateAuxColDefs creates definitions for
// auxiliary columns (num of positions, num of words etc.)
func generateAuxColDefs(hasSelfJoin bool) []string {
	ans := make([]string, 4)
	ans[0] = "poscount INTEGER"
	ans[1] = "wordcount INTEGER"
	ans[2] = "corpus_id VARCHAR(63)"
	if hasSelfJoin {
		ans[3] = "item_id VARCHAR(127)"

	} else {
		ans = ans[:3]
	}
	return ans
}

func createAuxIndices(database *sql.DB, groupedCorpusName string, cols []string) error {
	var err error
	for _, c := range cols {
		_, err = database.Exec(
			fmt.Sprintf("CREATE INDEX %s_%s_idx ON %s_liveattrs_entry(%s)",
				groupedCorpusName, c, groupedCorpusName, c))
		if err != nil {
			return err
		}
		log.Printf("Created custom index %s_%s_idx on %s_liveattrs_entry(%s)",
			groupedCorpusName, c, groupedCorpusName, c)
	}
	return nil
}

// generateViewColDefs creates definitions for
// bibliography view
func generateViewColDefs(cols []string, idAttr string) []string {
	ans := make([]string, len(cols))
	for i, c := range cols {
		if c != idAttr {
			ans[i] = c

		} else {
			ans[i] = fmt.Sprintf("%s AS id", c)
		}
	}
	return ans
}

// createBibView creates a database view needed
// by liveattrs to fetch bibliography information.
func createBibView(database *sql.DB, groupedCorpusName string, cols []string, idAttr string) error {
	colDefs := generateViewColDefs(cols, idAttr)
	_, err := database.Exec(fmt.Sprintf(
		"CREATE VIEW %s_bibliography AS SELECT %s FROM %s_liveattrs_entry",
		groupedCorpusName, joinArgs(colDefs), groupedCorpusName))
	if err != nil {
		return err
	}
	return nil
}

// createSchema creates all the required tables, views and indices
func createSchema(
	database *sql.DB,
	groupedCorpusName string,
	structures map[string][]string,
	indexedCols []string,
	useSelfJoin bool,
	countColumns []int,
) error {
	log.Print("Attempting to create tables and views...")

	cols := generateColNames(structures)
	colsDefs := make([]string, len(cols))
	for i, col := range cols {
		colsDefs[i] = fmt.Sprintf("%s VARCHAR(255)", col)
	}
	auxColDefs := generateAuxColDefs(useSelfJoin)
	allCollsDefs := append(colsDefs, auxColDefs...)
	_, dbErr := database.Exec(
		fmt.Sprintf("CREATE TABLE %s_liveattrs_entry (id INTEGER PRIMARY KEY auto_increment, %s)", groupedCorpusName, joinArgs(allCollsDefs)))
	if dbErr != nil {
		return fmt.Errorf("failed to create table '%s_liveattrs_entry': %s", groupedCorpusName, dbErr)
	}

	if useSelfJoin {
		_, dbErr = database.Exec(fmt.Sprintf(
			"CREATE UNIQUE INDEX %s_liveattrs_entry_item_id_corpus_id_idx ON %s_liveattrs_entry(item_id, corpus_id)",
			groupedCorpusName, groupedCorpusName))
		if dbErr != nil {
			return fmt.Errorf(
				"failed to create index %s_liveattrs_entry_item_id_corpus_id_idx on %s_liveattrs_entry(item_id, corpus_id): %s",
				groupedCorpusName, groupedCorpusName, dbErr)
		}
	}
	dbErr = createAuxIndices(database, groupedCorpusName, indexedCols)
	if dbErr != nil {
		return fmt.Errorf("failed to create a custom index: %s", dbErr)
	}

	if len(countColumns) > 0 {
		columns := db.GenerateColCountNames(countColumns)
		colDefs := db.GenerateColCountNames(countColumns)
		for i, c := range colDefs {
			colDefs[i] = c + " VARCHAR(127)"
		}
		_, dbErr = database.Exec(fmt.Sprintf(
			"CREATE TABLE %s_colcounts (%s, corpus_id VARCHAR(127), count INTEGER, arf INTEGER, PRIMARY KEY(%s))",
			groupedCorpusName, strings.Join(colDefs, ", "), strings.Join(columns, ", ")))
		if dbErr != nil {
			return fmt.Errorf("failed to create table '%s_colcounts': %s", groupedCorpusName, dbErr)
		}
		_, dbErr = database.Exec(fmt.Sprintf(
			"CREATE INDEX %s_colcounts_corpus_id_idx ON %s_colcounts(corpus_id)",
			groupedCorpusName, groupedCorpusName))
		if dbErr != nil {
			return fmt.Errorf(
				"failed to create index colcounts_corpus_id_idx on %s_colcounts(corpus_id): %s",
				groupedCorpusName, dbErr)
		}
	}
	log.Print("...DONE")
	return nil
}
<commit_msg>Quote table names as they may contain problematic characters<commit_after>// Copyright 2022 Tomas Machalek <tomas.machalek@gmail.com>
// Copyright 2022 Charles University, Faculty of Arts,
//                Institute of the Czech National Corpus
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mysql

import (
	"database/sql"
	"fmt"
	"log"
	"strings"

	"github.com/czcorpus/vert-tagextract/v2/db"
)

// dropExisting drops existing tables/views.
// It is safe to call this even if one or more of these does not exist.
// Please note that the groupedCorpusName argument represents a derived corpus name
// which is able to group multipe (aligned) corpora together.E.g. 'intercorp_v13_cs'
// and 'intercorp_v13_en' will likely groupedName 'intercorp_v13'. For single corpora,
// the groupedCorpusName is the same as the original one.
func dropExisting(database *sql.DB, groupedCorpusName string) error {
	log.Print("Attempting to drop possible existing tables and views...")
	var err error
	_, err = database.Exec("DROP TABLE IF EXISTS cache")
	if err != nil {
		return fmt.Errorf("failed to drop table 'cache': %s", err)
	}
	_, err = database.Exec(fmt.Sprintf("DROP VIEW IF EXISTS `%s_bibliography`", groupedCorpusName))
	if err != nil {
		return fmt.Errorf("failed to drop view `%s_bibliography`: %s", groupedCorpusName, err)
	}
	_, err = database.Exec(fmt.Sprintf("DROP TABLE IF EXISTS `%s_liveattrs_entry`", groupedCorpusName))
	if err != nil {
		return fmt.Errorf("failed to drop table '%s_liveattrs_entry': %s", groupedCorpusName, err)
	}
	_, err = database.Exec(fmt.Sprintf("DROP TABLE IF EXISTS `%s_colcounts`", groupedCorpusName))
	if err != nil {
		return fmt.Errorf("failed to drop table `%s_colcounts`: %s", groupedCorpusName, err)
	}
	log.Print("...DONE")
	return nil
}

// generateColNames produces a list of structural
// attribute names as used in database
// (i.e. [structname]_[attr_name]) out of lists
// of structural attributes defined in the configuration.
// (see _examples/*.json)
func generateColNames(structures map[string][]string) []string {
	numAttrs := 0
	for _, v := range structures {
		numAttrs += len(v)
	}
	ans := make([]string, numAttrs)
	i := 0
	for k, v := range structures {
		for _, a := range v {
			ans[i] = fmt.Sprintf("%s_%s", k, a)
			i++
		}
	}
	return ans
}

// generateAuxColDefs creates definitions for
// auxiliary columns (num of positions, num of words etc.)
func generateAuxColDefs(hasSelfJoin bool) []string {
	ans := make([]string, 4)
	ans[0] = "poscount INTEGER"
	ans[1] = "wordcount INTEGER"
	ans[2] = "corpus_id VARCHAR(63)"
	if hasSelfJoin {
		ans[3] = "item_id VARCHAR(127)"

	} else {
		ans = ans[:3]
	}
	return ans
}

func createAuxIndices(database *sql.DB, groupedCorpusName string, cols []string) error {
	var err error
	for _, c := range cols {
		_, err = database.Exec(
			fmt.Sprintf("CREATE INDEX %s_%s_idx ON `%s_liveattrs_entry`(%s)",
				groupedCorpusName, c, groupedCorpusName, c))
		if err != nil {
			return err
		}
		log.Printf("Created custom index %s_%s_idx on `%s_liveattrs_entry`(%s)",
			groupedCorpusName, c, groupedCorpusName, c)
	}
	return nil
}

// generateViewColDefs creates definitions for
// bibliography view
func generateViewColDefs(cols []string, idAttr string) []string {
	ans := make([]string, len(cols))
	for i, c := range cols {
		if c != idAttr {
			ans[i] = c

		} else {
			ans[i] = fmt.Sprintf("%s AS id", c)
		}
	}
	return ans
}

// createBibView creates a database view needed
// by liveattrs to fetch bibliography information.
func createBibView(database *sql.DB, groupedCorpusName string, cols []string, idAttr string) error {
	colDefs := generateViewColDefs(cols, idAttr)
	_, err := database.Exec(fmt.Sprintf(
		"CREATE VIEW %s_bibliography AS SELECT %s FROM `%s_liveattrs_entry`",
		groupedCorpusName, joinArgs(colDefs), groupedCorpusName))
	if err != nil {
		return err
	}
	return nil
}

// createSchema creates all the required tables, views and indices
func createSchema(
	database *sql.DB,
	groupedCorpusName string,
	structures map[string][]string,
	indexedCols []string,
	useSelfJoin bool,
	countColumns []int,
) error {
	log.Print("Attempting to create tables and views...")

	cols := generateColNames(structures)
	colsDefs := make([]string, len(cols))
	for i, col := range cols {
		colsDefs[i] = fmt.Sprintf("%s VARCHAR(255)", col)
	}
	auxColDefs := generateAuxColDefs(useSelfJoin)
	allCollsDefs := append(colsDefs, auxColDefs...)
	_, dbErr := database.Exec(
		fmt.Sprintf("CREATE TABLE `%s_liveattrs_entry` (id INTEGER PRIMARY KEY auto_increment, %s)", groupedCorpusName, joinArgs(allCollsDefs)))
	if dbErr != nil {
		return fmt.Errorf("failed to create table '%s_liveattrs_entry': %s", groupedCorpusName, dbErr)
	}

	if useSelfJoin {
		_, dbErr = database.Exec(fmt.Sprintf(
			"CREATE UNIQUE INDEX `%s_liveattrs_entry_item_id_corpus_id_idx` ON `%s_liveattrs_entry`(item_id, corpus_id)",
			groupedCorpusName, groupedCorpusName))
		if dbErr != nil {
			return fmt.Errorf(
				"failed to create index `%s_liveattrs_entry_item_id_corpus_id_idx` on `%s_liveattrs_entry`(item_id, corpus_id): %s",
				groupedCorpusName, groupedCorpusName, dbErr)
		}
	}
	dbErr = createAuxIndices(database, groupedCorpusName, indexedCols)
	if dbErr != nil {
		return fmt.Errorf("failed to create a custom index: %s", dbErr)
	}

	if len(countColumns) > 0 {
		columns := db.GenerateColCountNames(countColumns)
		colDefs := db.GenerateColCountNames(countColumns)
		for i, c := range colDefs {
			colDefs[i] = c + " VARCHAR(127)"
		}
		_, dbErr = database.Exec(fmt.Sprintf(
			"CREATE TABLE %s_colcounts (%s, corpus_id VARCHAR(127), count INTEGER, arf INTEGER, PRIMARY KEY(%s))",
			groupedCorpusName, strings.Join(colDefs, ", "), strings.Join(columns, ", ")))
		if dbErr != nil {
			return fmt.Errorf("failed to create table '%s_colcounts': %s", groupedCorpusName, dbErr)
		}
		_, dbErr = database.Exec(fmt.Sprintf(
			"CREATE INDEX %s_colcounts_corpus_id_idx ON %s_colcounts(corpus_id)",
			groupedCorpusName, groupedCorpusName))
		if dbErr != nil {
			return fmt.Errorf(
				"failed to create index colcounts_corpus_id_idx on %s_colcounts(corpus_id): %s",
				groupedCorpusName, dbErr)
		}
	}
	log.Print("...DONE")
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dao

import (
	"fmt"
	"time"

	"github.com/astaxie/beego/orm"
	"github.com/vmware/harbor/src/common/models"
)

// AddRepository adds a repo to the database.
func AddRepository(repo models.RepoRecord) error {
	if repo.ProjectID == 0 {
		return fmt.Errorf("invalid project ID: %d", repo.ProjectID)
	}

	o := GetOrmer()
	now := time.Now()
	repo.CreationTime = now
	repo.UpdateTime = now
	_, err := o.Insert(&repo)
	return err
}

// GetRepositoryByName ...
func GetRepositoryByName(name string) (*models.RepoRecord, error) {
	o := GetOrmer()
	r := models.RepoRecord{Name: name}
	err := o.Read(&r, "Name")
	if err == orm.ErrNoRows {
		return nil, nil
	}
	return &r, err
}

// DeleteRepository ...
func DeleteRepository(name string) error {
	o := GetOrmer()
	_, err := o.QueryTable("repository").Filter("name", name).Delete()
	return err
}

// UpdateRepository ...
func UpdateRepository(repo models.RepoRecord) error {
	o := GetOrmer()
	repo.UpdateTime = time.Now()
	_, err := o.Update(&repo)
	return err
}

// IncreasePullCount ...
func IncreasePullCount(name string) (err error) {
	o := GetOrmer()
	num, err := o.QueryTable("repository").Filter("name", name).Update(
		orm.Params{
			"pull_count":  orm.ColValue(orm.ColAdd, 1),
			"update_time": time.Now(),
		})
	if err != nil {
		return err
	}
	if num == 0 {
		return fmt.Errorf("Failed to increase repository pull count with name: %s", name)
	}
	return nil
}

//RepositoryExists returns whether the repository exists according to its name.
func RepositoryExists(name string) bool {
	o := GetOrmer()
	return o.QueryTable("repository").Filter("name", name).Exist()
}

//GetTopRepos returns the most popular repositories whose project ID is
// in projectIDs
func GetTopRepos(projectIDs []int64, n int) ([]*models.RepoRecord, error) {
	repositories := []*models.RepoRecord{}
	if len(projectIDs) == 0 {
		return repositories, nil
	}

	_, err := GetOrmer().QueryTable(&models.RepoRecord{}).
		Filter("project_id__in", projectIDs).
		OrderBy("-pull_count").
		Limit(n).
		All(&repositories)

	return repositories, err
}

// GetTotalOfRepositories ...
func GetTotalOfRepositories(query ...*models.RepositoryQuery) (int64, error) {
	sql, params := repositoryQueryConditions(query...)
	sql = `select count(*) ` + sql
	var total int64
	if err := GetOrmer().Raw(sql, params).QueryRow(&total); err != nil {
		return 0, err
	}
	return total, nil
}

// GetRepositories ...
func GetRepositories(query ...*models.RepositoryQuery) ([]*models.RepoRecord, error) {
	repositories := []*models.RepoRecord{}

	sql, params := repositoryQueryConditions(query...)
	sql = `select r.repository_id, r.name, r.project_id, r.description, r.pull_count,
	r.star_count, r.creation_time, r.update_time ` + sql + `order by r.name `
	if len(query) > 0 && query[0] != nil {
		page, size := query[0].Page, query[0].Size
		if size > 0 {
			sql += `limit ? `
			params = append(params, size)
			if page > 0 {
				sql += `offset ? `
				params = append(params, size*(page-1))
			}
		}
	}

	if _, err := GetOrmer().Raw(sql, params).QueryRows(&repositories); err != nil {
		return nil, err
	}
	return repositories, nil
}

func repositoryQueryConditions(query ...*models.RepositoryQuery) (string, []interface{}) {
	params := []interface{}{}
	sql := `from repository r `
	if len(query) == 0 || query[0] == nil {
		return sql, params
	}
	q := query[0]
	if len(q.ProjectName) > 0 {
		sql += `join project p on r.project_id = p.project_id `
	}

	if q.LabelID > 0 {
		sql += `join harbor_resource_label rl on r.repository_id = rl.resource_id
		and rl.resource_type = 'r' `
	}
	sql += `where 1=1 `

	if len(q.Name) > 0 {
		sql += `and r.name = ? `
		params = append(params, q.Name)
	}

	if len(q.ProjectIDs) > 0 {
		sql += fmt.Sprintf(`and r.project_id in ( %s ) `,
			paramPlaceholder(len(q.ProjectIDs)))
		params = append(params, q.ProjectIDs)
	}

	if len(q.ProjectName) > 0 {
		sql += `and p.name = ? `
		params = append(params, q.ProjectName)
	}

	if q.LabelID > 0 {
		sql += `and rl.label_id = ? `
		params = append(params, q.LabelID)
	}

	return sql, params
}
<commit_msg>Fuzzy matching repository name in GET repositories API<commit_after>// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dao

import (
	"fmt"
	"time"

	"github.com/astaxie/beego/orm"
	"github.com/vmware/harbor/src/common/models"
)

// AddRepository adds a repo to the database.
func AddRepository(repo models.RepoRecord) error {
	if repo.ProjectID == 0 {
		return fmt.Errorf("invalid project ID: %d", repo.ProjectID)
	}

	o := GetOrmer()
	now := time.Now()
	repo.CreationTime = now
	repo.UpdateTime = now
	_, err := o.Insert(&repo)
	return err
}

// GetRepositoryByName ...
func GetRepositoryByName(name string) (*models.RepoRecord, error) {
	o := GetOrmer()
	r := models.RepoRecord{Name: name}
	err := o.Read(&r, "Name")
	if err == orm.ErrNoRows {
		return nil, nil
	}
	return &r, err
}

// DeleteRepository ...
func DeleteRepository(name string) error {
	o := GetOrmer()
	_, err := o.QueryTable("repository").Filter("name", name).Delete()
	return err
}

// UpdateRepository ...
func UpdateRepository(repo models.RepoRecord) error {
	o := GetOrmer()
	repo.UpdateTime = time.Now()
	_, err := o.Update(&repo)
	return err
}

// IncreasePullCount ...
func IncreasePullCount(name string) (err error) {
	o := GetOrmer()
	num, err := o.QueryTable("repository").Filter("name", name).Update(
		orm.Params{
			"pull_count":  orm.ColValue(orm.ColAdd, 1),
			"update_time": time.Now(),
		})
	if err != nil {
		return err
	}
	if num == 0 {
		return fmt.Errorf("Failed to increase repository pull count with name: %s", name)
	}
	return nil
}

//RepositoryExists returns whether the repository exists according to its name.
func RepositoryExists(name string) bool {
	o := GetOrmer()
	return o.QueryTable("repository").Filter("name", name).Exist()
}

//GetTopRepos returns the most popular repositories whose project ID is
// in projectIDs
func GetTopRepos(projectIDs []int64, n int) ([]*models.RepoRecord, error) {
	repositories := []*models.RepoRecord{}
	if len(projectIDs) == 0 {
		return repositories, nil
	}

	_, err := GetOrmer().QueryTable(&models.RepoRecord{}).
		Filter("project_id__in", projectIDs).
		OrderBy("-pull_count").
		Limit(n).
		All(&repositories)

	return repositories, err
}

// GetTotalOfRepositories ...
func GetTotalOfRepositories(query ...*models.RepositoryQuery) (int64, error) {
	sql, params := repositoryQueryConditions(query...)
	sql = `select count(*) ` + sql
	var total int64
	if err := GetOrmer().Raw(sql, params).QueryRow(&total); err != nil {
		return 0, err
	}
	return total, nil
}

// GetRepositories ...
func GetRepositories(query ...*models.RepositoryQuery) ([]*models.RepoRecord, error) {
	repositories := []*models.RepoRecord{}

	sql, params := repositoryQueryConditions(query...)
	sql = `select r.repository_id, r.name, r.project_id, r.description, r.pull_count,
	r.star_count, r.creation_time, r.update_time ` + sql + `order by r.name `
	if len(query) > 0 && query[0] != nil {
		page, size := query[0].Page, query[0].Size
		if size > 0 {
			sql += `limit ? `
			params = append(params, size)
			if page > 0 {
				sql += `offset ? `
				params = append(params, size*(page-1))
			}
		}
	}

	if _, err := GetOrmer().Raw(sql, params).QueryRows(&repositories); err != nil {
		return nil, err
	}
	return repositories, nil
}

func repositoryQueryConditions(query ...*models.RepositoryQuery) (string, []interface{}) {
	params := []interface{}{}
	sql := `from repository r `
	if len(query) == 0 || query[0] == nil {
		return sql, params
	}
	q := query[0]
	if len(q.ProjectName) > 0 {
		sql += `join project p on r.project_id = p.project_id `
	}

	if q.LabelID > 0 {
		sql += `join harbor_resource_label rl on r.repository_id = rl.resource_id
		and rl.resource_type = 'r' `
	}
	sql += `where 1=1 `

	if len(q.Name) > 0 {
		sql += `and r.name like ? `
		params = append(params, "%"+Escape(q.Name)+"%")
	}

	if len(q.ProjectIDs) > 0 {
		sql += fmt.Sprintf(`and r.project_id in ( %s ) `,
			paramPlaceholder(len(q.ProjectIDs)))
		params = append(params, q.ProjectIDs)
	}

	if len(q.ProjectName) > 0 {
		sql += `and p.name = ? `
		params = append(params, q.ProjectName)
	}

	if q.LabelID > 0 {
		sql += `and rl.label_id = ? `
		params = append(params, q.LabelID)
	}

	return sql, params
}
<|endoftext|>


<commit_before>package jsontree

import (
	"bufio"
	"fmt"
	"io"
)

type GetValueFn func() Value

func DeserializeNode(r io.Reader, getValFn GetValueFn) (*Node, error) {
	node := new(Node)
	p := newParser(r, getValFn)
	for p.Scan() {
		path, value := p.Data()
		node.Key = path[0]
		if len(path) == 1 {
			if node.Value != nil {
				return node, fmt.Errorf("invalid json. Expected 1 root node")
			}
			node.Value = value
		} else {
			node.getOrAdd(path[1:]...).Value = value
		}
	}
	return node, p.Err()
}

type readFn func() (next readFn, err error)

type parser struct {
	r     *bufio.Reader
	valFn GetValueFn
	next  readFn
	err   error
	mode  int
	path  stack
	value Value
	eof   bool
}

func newParser(r io.Reader, getValFn GetValueFn) *parser {
	p := &parser{valFn: getValFn}
	if br, ok := r.(*bufio.Reader); ok {
		p.r = br
	} else {
		p.r = bufio.NewReader(r)
	}
	p.next = p.readOpenBracket
	return p
}

func (p *parser) Scan() bool {
	if p.err != nil {
		return false
	}
	p.value = nil
	// Scan until we've hit a value
	for p.value == nil {
		p.next, p.err = p.next()
		if p.err != nil {
			return false
		}
	}
	return true
}

func (p *parser) Data() (path []string, value Value) {
	path = p.path
	value = p.value
	return path, value
}

func (p *parser) Err() error {
	if p.err == io.EOF {
		if p.eof {
			return nil
		} else {
			return fmt.Errorf("reader returned io.EOF before expected")
		}
	} else {
		return p.err
	}
}

func (p *parser) readByte(bWant byte, next readFn) (readFn, error) {
	if bGot, err := p.r.ReadByte(); err != nil {
		return nil, err
	} else if bGot != bWant {
		return nil, fmt.Errorf("Read '%s', want '%s'", string(bGot), string(bWant))
	} else {
		return next, nil
	}
}

func (p *parser) readOpenBracket() (readFn, error) {
	return p.readByte('{', p.readQuotedKey)
}

func (p *parser) readCloseBracket() (readFn, error) {
	if _, err := p.readByte('}', nil); err != nil {
		return nil, err
	}
	p.path.Pop()
	if len(p.path) == 0 {
		p.eof = true
		if b, err := p.r.ReadByte(); err == nil {
			return nil, fmt.Errorf("expected end of input. Got '%s'", string(b))
		} else {
			return nil, err
		}
	}
	if bs, err := p.r.Peek(1); err != nil {
		return nil, err
	} else {
		switch bs[0] {
		case '}':
			return p.readCloseBracket, nil
		case ',':
			return p.readComma, nil
		default:
			return nil, fmt.Errorf(`Read '%s', want '{' or '"'`, string(bs))
		}
	}
}

func (p *parser) readQuotedString() ([]byte, error) {
	if _, err := p.readByte('"', nil); err != nil {
		return nil, err
	}
	var bs []byte
	for {
		b, err := p.r.ReadByte()
		if err != nil {
			return nil, err
		}
		if b == '\\' { // escape
			bs = append(bs, b)
			if b, err := p.r.ReadByte(); err != nil {
				return nil, err
			} else {
				bs = append(bs, b)
			}
			continue
		}
		if b == '"' {
			break
		}
		bs = append(bs, b)
	}
	return bs, nil
}

func (p *parser) readQuotedKey() (readFn, error) {
	if bs, err := p.readQuotedString(); err != nil {
		return nil, err
	} else {
		p.path.Push(string(bs))
	}
	// Following the key should be a column
	if _, err := p.readByte(':', nil); err != nil {
		return nil, err
	}
	// And following the column is eiter a sub node or a value
	if bs, err := p.r.Peek(1); err != nil {
		return nil, err
	} else {
		switch bs[0] {
		case '{':
			// Consume the byte
			if _, err := p.r.ReadByte(); err != nil {
				return nil, err
			}
			return p.readQuotedKey, nil
		case '"':
			return p.readQuotedValue, nil
		default:
			return nil, fmt.Errorf(`Read %s, want '{' or '"'`, string(bs))
		}
	}
}

func (p *parser) readQuotedValue() (readFn, error) {
	if bs, err := p.readQuotedString(); err != nil {
		return nil, err
	} else {
		p.value = p.valFn()
		p.err = p.value.Deserialize(bs)
	}
	// Following the value is either a sibling node or a closing bracket
	if bs, err := p.r.Peek(1); err != nil {
		return nil, err
	} else {
		switch bs[0] {
		case '}':
			return p.readCloseBracket, nil
		case ',':
			return p.readComma, nil
		default:
			return nil, fmt.Errorf(`Read '%s', want '}' or ','`, string(bs))
		}
	}
}

func (p *parser) readComma() (readFn, error) {
	p.path.Pop()
	return p.readByte(',', p.readQuotedKey)
}

type stack []string

func (s *stack) Push(v string) {
	*s = append(*s, v)
}

func (s *stack) Pop() {
	if n := len(*s); n > 0 {
		*s = (*s)[:n-1]
	}
}
<commit_msg>Fixed typo in comment<commit_after>package jsontree

import (
	"bufio"
	"fmt"
	"io"
)

type GetValueFn func() Value

func DeserializeNode(r io.Reader, getValFn GetValueFn) (*Node, error) {
	node := new(Node)
	p := newParser(r, getValFn)
	for p.Scan() {
		path, value := p.Data()
		node.Key = path[0]
		if len(path) == 1 {
			if node.Value != nil {
				return node, fmt.Errorf("invalid json. Expected 1 root node")
			}
			node.Value = value
		} else {
			node.getOrAdd(path[1:]...).Value = value
		}
	}
	return node, p.Err()
}

type readFn func() (next readFn, err error)

type parser struct {
	r     *bufio.Reader
	valFn GetValueFn
	next  readFn
	err   error
	mode  int
	path  stack
	value Value
	eof   bool
}

func newParser(r io.Reader, getValFn GetValueFn) *parser {
	p := &parser{valFn: getValFn}
	if br, ok := r.(*bufio.Reader); ok {
		p.r = br
	} else {
		p.r = bufio.NewReader(r)
	}
	p.next = p.readOpenBracket
	return p
}

func (p *parser) Scan() bool {
	if p.err != nil {
		return false
	}
	p.value = nil
	// Scan until we've hit a value
	for p.value == nil {
		p.next, p.err = p.next()
		if p.err != nil {
			return false
		}
	}
	return true
}

func (p *parser) Data() (path []string, value Value) {
	path = p.path
	value = p.value
	return path, value
}

func (p *parser) Err() error {
	if p.err == io.EOF {
		if p.eof {
			return nil
		} else {
			return fmt.Errorf("reader returned io.EOF before expected")
		}
	} else {
		return p.err
	}
}

func (p *parser) readByte(bWant byte, next readFn) (readFn, error) {
	if bGot, err := p.r.ReadByte(); err != nil {
		return nil, err
	} else if bGot != bWant {
		return nil, fmt.Errorf("Read '%s', want '%s'", string(bGot), string(bWant))
	} else {
		return next, nil
	}
}

func (p *parser) readOpenBracket() (readFn, error) {
	return p.readByte('{', p.readQuotedKey)
}

func (p *parser) readCloseBracket() (readFn, error) {
	if _, err := p.readByte('}', nil); err != nil {
		return nil, err
	}
	p.path.Pop()
	if len(p.path) == 0 {
		p.eof = true
		if b, err := p.r.ReadByte(); err == nil {
			return nil, fmt.Errorf("expected end of input. Got '%s'", string(b))
		} else {
			return nil, err
		}
	}
	if bs, err := p.r.Peek(1); err != nil {
		return nil, err
	} else {
		switch bs[0] {
		case '}':
			return p.readCloseBracket, nil
		case ',':
			return p.readComma, nil
		default:
			return nil, fmt.Errorf(`Read '%s', want '{' or '"'`, string(bs))
		}
	}
}

func (p *parser) readQuotedString() ([]byte, error) {
	if _, err := p.readByte('"', nil); err != nil {
		return nil, err
	}
	var bs []byte
	for {
		b, err := p.r.ReadByte()
		if err != nil {
			return nil, err
		}
		if b == '\\' { // escape
			bs = append(bs, b)
			if b, err := p.r.ReadByte(); err != nil {
				return nil, err
			} else {
				bs = append(bs, b)
			}
			continue
		}
		if b == '"' {
			break
		}
		bs = append(bs, b)
	}
	return bs, nil
}

func (p *parser) readQuotedKey() (readFn, error) {
	if bs, err := p.readQuotedString(); err != nil {
		return nil, err
	} else {
		p.path.Push(string(bs))
	}
	// Following the key should be a column
	if _, err := p.readByte(':', nil); err != nil {
		return nil, err
	}
	// And following the column is either a sub node or a value
	if bs, err := p.r.Peek(1); err != nil {
		return nil, err
	} else {
		switch bs[0] {
		case '{':
			// Consume the byte
			if _, err := p.r.ReadByte(); err != nil {
				return nil, err
			}
			return p.readQuotedKey, nil
		case '"':
			return p.readQuotedValue, nil
		default:
			return nil, fmt.Errorf(`Read %s, want '{' or '"'`, string(bs))
		}
	}
}

func (p *parser) readQuotedValue() (readFn, error) {
	if bs, err := p.readQuotedString(); err != nil {
		return nil, err
	} else {
		p.value = p.valFn()
		p.err = p.value.Deserialize(bs)
	}
	// Following the value is either a sibling node or a closing bracket
	if bs, err := p.r.Peek(1); err != nil {
		return nil, err
	} else {
		switch bs[0] {
		case '}':
			return p.readCloseBracket, nil
		case ',':
			return p.readComma, nil
		default:
			return nil, fmt.Errorf(`Read '%s', want '}' or ','`, string(bs))
		}
	}
}

func (p *parser) readComma() (readFn, error) {
	p.path.Pop()
	return p.readByte(',', p.readQuotedKey)
}

type stack []string

func (s *stack) Push(v string) {
	*s = append(*s, v)
}

func (s *stack) Pop() {
	if n := len(*s); n > 0 {
		*s = (*s)[:n-1]
	}
}
<|endoftext|>