<commit_before>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ast

import (
	"encoding/json"
	"fmt"
	"github.com/prometheus/prometheus/utility"
	"sort"
	"strings"
	"time"
)

type OutputFormat int

const (
	TEXT OutputFormat = iota
	JSON
)

func binOpTypeToString(opType BinOpType) string {
	opTypeMap := map[BinOpType]string{
		ADD: "+",
		SUB: "-",
		MUL: "*",
		DIV: "/",
		MOD: "%",
		GT:  ">",
		LT:  "<",
		EQ:  "==",
		NE:  "!=",
		GE:  ">=",
		LE:  "<=",
	}
	return opTypeMap[opType]
}

func aggrTypeToString(aggrType AggrType) string {
	aggrTypeMap := map[AggrType]string{
		SUM: "SUM",
		AVG: "AVG",
		MIN: "MIN",
		MAX: "MAX",
	}
	return aggrTypeMap[aggrType]
}

func exprTypeToString(exprType ExprType) string {
	exprTypeMap := map[ExprType]string{
		SCALAR: "scalar",
		VECTOR: "vector",
		MATRIX: "matrix",
		STRING: "string",
	}
	return exprTypeMap[exprType]
}

func (vector Vector) ToString() string {
	metricStrings := []string{}
	for _, sample := range vector {
		metricName, ok := sample.Metric["name"]
		if !ok {
			panic("Tried to print vector without metric name")
		}
		labelStrings := []string{}
		for label, value := range sample.Metric {
			if label != "name" {
				// TODO escape special chars in label values here and elsewhere.
				labelStrings = append(labelStrings, fmt.Sprintf("%v='%v'", label, value))
			}
		}
		sort.Strings(labelStrings)
		metricStrings = append(metricStrings,
			fmt.Sprintf("%v{%v} => %v @[%v]",
				metricName,
				strings.Join(labelStrings, ","),
				sample.Value, sample.Timestamp))
	}
	sort.Strings(metricStrings)
	return strings.Join(metricStrings, "\n")
}

func (matrix Matrix) ToString() string {
	metricStrings := []string{}
	for _, sampleSet := range matrix {
		metricName, ok := sampleSet.Metric["name"]
		if !ok {
			panic("Tried to print matrix without metric name")
		}
		labelStrings := []string{}
		for label, value := range sampleSet.Metric {
			if label != "name" {
				labelStrings = append(labelStrings, fmt.Sprintf("%v='%v'", label, value))
			}
		}
		sort.Strings(labelStrings)
		valueStrings := []string{}
		for _, value := range sampleSet.Values {
			valueStrings = append(valueStrings,
				fmt.Sprintf("\n%v @[%v]", value.Value, value.Timestamp))
		}
		metricStrings = append(metricStrings,
			fmt.Sprintf("%v{%v} => %v",
				metricName,
				strings.Join(labelStrings, ","),
				strings.Join(valueStrings, ", ")))
	}
	sort.Strings(metricStrings)
	return strings.Join(metricStrings, "\n")
}

func ErrorToJSON(err error) string {
	errorStruct := struct {
		Type  string
		Value string
	}{
		Type:  "error",
		Value: err.Error(),
	}

	errorJSON, err := json.MarshalIndent(errorStruct, "", "\t")
	if err != nil {
		return ""
	}
	return string(errorJSON)
}

func TypedValueToJSON(data interface{}, typeStr string) string {
	dataStruct := struct {
		Type  string
		Value interface{}
	}{
		Type:  typeStr,
		Value: data,
	}
	dataJSON, err := json.MarshalIndent(dataStruct, "", "\t")
	if err != nil {
		return ErrorToJSON(err)
	}
	return string(dataJSON)
}

func EvalToString(node Node, timestamp *time.Time, format OutputFormat) string {
	switch node.Type() {
	case SCALAR:
		scalar := node.(ScalarNode).Eval(timestamp)
		switch format {
		case TEXT:
			return fmt.Sprintf("scalar: %v", scalar)
		case JSON:
			return TypedValueToJSON(scalar, "scalar")
		}
	case VECTOR:
		vector := node.(VectorNode).Eval(timestamp)
		switch format {
		case TEXT:
			return vector.ToString()
		case JSON:
			return TypedValueToJSON(vector, "vector")
		}
	case MATRIX:
		matrix := node.(MatrixNode).Eval(timestamp)
		switch format {
		case TEXT:
			return matrix.ToString()
		case JSON:
			return TypedValueToJSON(matrix, "matrix")
		}
	case STRING:
		str := node.(StringNode).Eval(timestamp)
		switch format {
		case TEXT:
			return str
		case JSON:
			return TypedValueToJSON(str, "string")
		}
	}
	panic("Switch didn't cover all node types")
}

func (node *VectorLiteral) ToString() string {
	metricName, ok := node.labels["name"]
	if !ok {
		panic("Tried to print vector without metric name")
	}
	labelStrings := []string{}
	for label, value := range node.labels {
		if label != "name" {
			labelStrings = append(labelStrings, fmt.Sprintf("%v='%v'", label, value))
		}
	}
	sort.Strings(labelStrings)
	return fmt.Sprintf("%v{%v}", metricName, strings.Join(labelStrings, ","))
}

func (node *MatrixLiteral) ToString() string {
	vectorString := (&VectorLiteral{labels: node.labels}).ToString()
	intervalString := fmt.Sprintf("['%v']", utility.DurationToString(node.interval))
	return vectorString + intervalString
}

func (node *ScalarLiteral) NodeTreeToDotGraph() string {
	return fmt.Sprintf("%#p[label=\"%v\"];\n", node, node.value)
}

func functionArgsToDotGraph(node Node, args []Node) string {
	graph := ""
	for _, arg := range args {
		graph += fmt.Sprintf("%#p -> %#p;\n", node, arg)
	}
	for _, arg := range args {
		graph += arg.NodeTreeToDotGraph()
	}
	return graph
}

func (node *ScalarFunctionCall) NodeTreeToDotGraph() string {
	graph := fmt.Sprintf("%#p[label=\"%v\"];\n", node, node.function.name)
	graph += functionArgsToDotGraph(node, node.args)
	return graph
}

func (node *ScalarArithExpr) NodeTreeToDotGraph() string {
	graph := fmt.Sprintf("%#p[label=\"%v\"];\n", node, binOpTypeToString(node.opType))
	graph += fmt.Sprintf("%#p -> %#p;\n", node, node.lhs)
	graph += fmt.Sprintf("%#p -> %#p;\n", node, node.rhs)
	graph += node.lhs.NodeTreeToDotGraph()
	graph += node.rhs.NodeTreeToDotGraph()
	return graph
}

func (node *VectorLiteral) NodeTreeToDotGraph() string {
	return fmt.Sprintf("%#p[label=\"%v\"];\n", node, node.ToString())
}

func (node *VectorFunctionCall) NodeTreeToDotGraph() string {
	graph := fmt.Sprintf("%#p[label=\"%v\"];\n", node, node.function.name)
	graph += functionArgsToDotGraph(node, node.args)
	return graph
}

func (node *VectorAggregation) NodeTreeToDotGraph() string {
	groupByStrings := []string{}
	for _, label := range node.groupBy {
		groupByStrings = append(groupByStrings, string(label))
	}

	graph := fmt.Sprintf("%#p[label=\"%v BY (%v)\"]\n",
		node,
		aggrTypeToString(node.aggrType),
		strings.Join(groupByStrings, ", "))
	graph += fmt.Sprintf("%#p -> %#p;\n", node, node.vector)
	graph += node.vector.NodeTreeToDotGraph()
	return graph
}

func (node *VectorArithExpr) NodeTreeToDotGraph() string {
	graph := fmt.Sprintf("%#p[label=\"%v\"];\n", node, binOpTypeToString(node.opType))
	graph += fmt.Sprintf("%#p -> %#p;\n", node, node.lhs)
	graph += fmt.Sprintf("%#p -> %#p;\n", node, node.rhs)
	graph += node.lhs.NodeTreeToDotGraph()
	graph += node.rhs.NodeTreeToDotGraph()
	return graph
}

func (node *MatrixLiteral) NodeTreeToDotGraph() string {
	return fmt.Sprintf("%#p[label=\"%v\"];\n", node, node.ToString())
}

func (node *StringLiteral) NodeTreeToDotGraph() string {
	return fmt.Sprintf("%#p[label=\"'%v'\"];\n", node.str)
}

func (node *StringFunctionCall) NodeTreeToDotGraph() string {
	graph := fmt.Sprintf("%#p[label=\"%v\"];\n", node, node.function.name)
	graph += functionArgsToDotGraph(node, node.args)
	return graph
}
<commit_msg>Niladic ``ToString()`` to idiomatic ``String()``.<commit_after>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ast

import (
	"encoding/json"
	"fmt"
	"github.com/prometheus/prometheus/utility"
	"sort"
	"strings"
	"time"
)

type OutputFormat int

const (
	TEXT OutputFormat = iota
	JSON
)

func binOpTypeToString(opType BinOpType) string {
	opTypeMap := map[BinOpType]string{
		ADD: "+",
		SUB: "-",
		MUL: "*",
		DIV: "/",
		MOD: "%",
		GT:  ">",
		LT:  "<",
		EQ:  "==",
		NE:  "!=",
		GE:  ">=",
		LE:  "<=",
	}
	return opTypeMap[opType]
}

func aggrTypeToString(aggrType AggrType) string {
	aggrTypeMap := map[AggrType]string{
		SUM: "SUM",
		AVG: "AVG",
		MIN: "MIN",
		MAX: "MAX",
	}
	return aggrTypeMap[aggrType]
}

func exprTypeToString(exprType ExprType) string {
	exprTypeMap := map[ExprType]string{
		SCALAR: "scalar",
		VECTOR: "vector",
		MATRIX: "matrix",
		STRING: "string",
	}
	return exprTypeMap[exprType]
}

func (vector Vector) String() string {
	metricStrings := []string{}
	for _, sample := range vector {
		metricName, ok := sample.Metric["name"]
		if !ok {
			panic("Tried to print vector without metric name")
		}
		labelStrings := []string{}
		for label, value := range sample.Metric {
			if label != "name" {
				// TODO escape special chars in label values here and elsewhere.
				labelStrings = append(labelStrings, fmt.Sprintf("%v='%v'", label, value))
			}
		}
		sort.Strings(labelStrings)
		metricStrings = append(metricStrings,
			fmt.Sprintf("%v{%v} => %v @[%v]",
				metricName,
				strings.Join(labelStrings, ","),
				sample.Value, sample.Timestamp))
	}
	sort.Strings(metricStrings)
	return strings.Join(metricStrings, "\n")
}

func (matrix Matrix) String() string {
	metricStrings := []string{}
	for _, sampleSet := range matrix {
		metricName, ok := sampleSet.Metric["name"]
		if !ok {
			panic("Tried to print matrix without metric name")
		}
		labelStrings := []string{}
		for label, value := range sampleSet.Metric {
			if label != "name" {
				labelStrings = append(labelStrings, fmt.Sprintf("%v='%v'", label, value))
			}
		}
		sort.Strings(labelStrings)
		valueStrings := []string{}
		for _, value := range sampleSet.Values {
			valueStrings = append(valueStrings,
				fmt.Sprintf("\n%v @[%v]", value.Value, value.Timestamp))
		}
		metricStrings = append(metricStrings,
			fmt.Sprintf("%v{%v} => %v",
				metricName,
				strings.Join(labelStrings, ","),
				strings.Join(valueStrings, ", ")))
	}
	sort.Strings(metricStrings)
	return strings.Join(metricStrings, "\n")
}

func ErrorToJSON(err error) string {
	errorStruct := struct {
		Type  string
		Value string
	}{
		Type:  "error",
		Value: err.Error(),
	}

	errorJSON, err := json.MarshalIndent(errorStruct, "", "\t")
	if err != nil {
		return ""
	}
	return string(errorJSON)
}

func TypedValueToJSON(data interface{}, typeStr string) string {
	dataStruct := struct {
		Type  string
		Value interface{}
	}{
		Type:  typeStr,
		Value: data,
	}
	dataJSON, err := json.MarshalIndent(dataStruct, "", "\t")
	if err != nil {
		return ErrorToJSON(err)
	}
	return string(dataJSON)
}

func EvalToString(node Node, timestamp *time.Time, format OutputFormat) string {
	switch node.Type() {
	case SCALAR:
		scalar := node.(ScalarNode).Eval(timestamp)
		switch format {
		case TEXT:
			return fmt.Sprintf("scalar: %v", scalar)
		case JSON:
			return TypedValueToJSON(scalar, "scalar")
		}
	case VECTOR:
		vector := node.(VectorNode).Eval(timestamp)
		switch format {
		case TEXT:
			return vector.String()
		case JSON:
			return TypedValueToJSON(vector, "vector")
		}
	case MATRIX:
		matrix := node.(MatrixNode).Eval(timestamp)
		switch format {
		case TEXT:
			return matrix.String()
		case JSON:
			return TypedValueToJSON(matrix, "matrix")
		}
	case STRING:
		str := node.(StringNode).Eval(timestamp)
		switch format {
		case TEXT:
			return str
		case JSON:
			return TypedValueToJSON(str, "string")
		}
	}
	panic("Switch didn't cover all node types")
}

func (node *VectorLiteral) String() string {
	metricName, ok := node.labels["name"]
	if !ok {
		panic("Tried to print vector without metric name")
	}
	labelStrings := []string{}
	for label, value := range node.labels {
		if label != "name" {
			labelStrings = append(labelStrings, fmt.Sprintf("%v='%v'", label, value))
		}
	}
	sort.Strings(labelStrings)
	return fmt.Sprintf("%v{%v}", metricName, strings.Join(labelStrings, ","))
}

func (node *MatrixLiteral) String() string {
	vectorString := (&VectorLiteral{labels: node.labels}).String()
	intervalString := fmt.Sprintf("['%v']", utility.DurationToString(node.interval))
	return vectorString + intervalString
}

func (node *ScalarLiteral) NodeTreeToDotGraph() string {
	return fmt.Sprintf("%#p[label=\"%v\"];\n", node, node.value)
}

func functionArgsToDotGraph(node Node, args []Node) string {
	graph := ""
	for _, arg := range args {
		graph += fmt.Sprintf("%#p -> %#p;\n", node, arg)
	}
	for _, arg := range args {
		graph += arg.NodeTreeToDotGraph()
	}
	return graph
}

func (node *ScalarFunctionCall) NodeTreeToDotGraph() string {
	graph := fmt.Sprintf("%#p[label=\"%v\"];\n", node, node.function.name)
	graph += functionArgsToDotGraph(node, node.args)
	return graph
}

func (node *ScalarArithExpr) NodeTreeToDotGraph() string {
	graph := fmt.Sprintf("%#p[label=\"%v\"];\n", node, binOpTypeToString(node.opType))
	graph += fmt.Sprintf("%#p -> %#p;\n", node, node.lhs)
	graph += fmt.Sprintf("%#p -> %#p;\n", node, node.rhs)
	graph += node.lhs.NodeTreeToDotGraph()
	graph += node.rhs.NodeTreeToDotGraph()
	return graph
}

func (node *VectorLiteral) NodeTreeToDotGraph() string {
	return fmt.Sprintf("%#p[label=\"%v\"];\n", node, node.String())
}

func (node *VectorFunctionCall) NodeTreeToDotGraph() string {
	graph := fmt.Sprintf("%#p[label=\"%v\"];\n", node, node.function.name)
	graph += functionArgsToDotGraph(node, node.args)
	return graph
}

func (node *VectorAggregation) NodeTreeToDotGraph() string {
	groupByStrings := []string{}
	for _, label := range node.groupBy {
		groupByStrings = append(groupByStrings, string(label))
	}

	graph := fmt.Sprintf("%#p[label=\"%v BY (%v)\"]\n",
		node,
		aggrTypeToString(node.aggrType),
		strings.Join(groupByStrings, ", "))
	graph += fmt.Sprintf("%#p -> %#p;\n", node, node.vector)
	graph += node.vector.NodeTreeToDotGraph()
	return graph
}

func (node *VectorArithExpr) NodeTreeToDotGraph() string {
	graph := fmt.Sprintf("%#p[label=\"%v\"];\n", node, binOpTypeToString(node.opType))
	graph += fmt.Sprintf("%#p -> %#p;\n", node, node.lhs)
	graph += fmt.Sprintf("%#p -> %#p;\n", node, node.rhs)
	graph += node.lhs.NodeTreeToDotGraph()
	graph += node.rhs.NodeTreeToDotGraph()
	return graph
}

func (node *MatrixLiteral) NodeTreeToDotGraph() string {
	return fmt.Sprintf("%#p[label=\"%v\"];\n", node, node.String())
}

func (node *StringLiteral) NodeTreeToDotGraph() string {
	return fmt.Sprintf("%#p[label=\"'%v'\"];\n", node.str, node.str)
}

func (node *StringFunctionCall) NodeTreeToDotGraph() string {
	graph := fmt.Sprintf("%#p[label=\"%v\"];\n", node, node.function.name)
	graph += functionArgsToDotGraph(node, node.args)
	return graph
}
<|endoftext|>


<commit_before>// Copyright 2018 ThoughtWorks, Inc.

// This file is part of Gauge.

// Gauge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Gauge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Gauge.  If not, see <http://www.gnu.org/licenses/>.

package runner

import (
	"context"
	"fmt"
	"io"
	"net"
	"os/exec"
	"time"

	"github.com/getgauge/gauge/config"
	"github.com/getgauge/gauge/gauge_messages"
	gm "github.com/getgauge/gauge/gauge_messages"
	"github.com/getgauge/gauge/logger"
	"github.com/getgauge/gauge/manifest"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

const (
	host  = "127.0.0.1"
	oneGB = 1024 * 1024 * 1024
)

// GrpcRunner handles grpc messages.
type GrpcRunner struct {
	cmd          *exec.Cmd
	conn         *grpc.ClientConn
	LegacyClient gm.LspServiceClient
	RunnerClient gm.RunnerClient
	Timeout      time.Duration
	info         *RunnerInfo
	IsExecuting  bool
}

func (r *GrpcRunner) invokeLegacyLSPService(message *gm.Message) (*gm.Message, error) {
	switch message.MessageType {
	case gm.Message_CacheFileRequest:
		_, err := r.LegacyClient.CacheFile(context.Background(), message.CacheFileRequest)
		return &gm.Message{}, err
	case gm.Message_StepNamesRequest:
		response, err := r.LegacyClient.GetStepNames(context.Background(), message.StepNamesRequest)
		return &gm.Message{StepNamesResponse: response}, err
	case gm.Message_StepPositionsRequest:
		response, err := r.LegacyClient.GetStepPositions(context.Background(), message.StepPositionsRequest)
		return &gm.Message{StepPositionsResponse: response}, err
	case gm.Message_ImplementationFileListRequest:
		response, err := r.LegacyClient.GetImplementationFiles(context.Background(), &gm.Empty{})
		return &gm.Message{ImplementationFileListResponse: response}, err
	case gm.Message_StubImplementationCodeRequest:
		response, err := r.LegacyClient.ImplementStub(context.Background(), message.StubImplementationCodeRequest)
		return &gm.Message{FileDiff: response}, err
	case gm.Message_StepValidateRequest:
		response, err := r.LegacyClient.ValidateStep(context.Background(), message.StepValidateRequest)
		return &gm.Message{MessageType: gm.Message_StepValidateResponse, StepValidateResponse: response}, err
	case gm.Message_RefactorRequest:
		response, err := r.LegacyClient.Refactor(context.Background(), message.RefactorRequest)
		return &gm.Message{MessageType: gm.Message_RefactorResponse, RefactorResponse: response}, err
	case gm.Message_StepNameRequest:
		response, err := r.LegacyClient.GetStepName(context.Background(), message.StepNameRequest)
		return &gm.Message{MessageType: gm.Message_StepNameResponse, StepNameResponse: response}, err
	case gm.Message_ImplementationFileGlobPatternRequest:
		response, err := r.LegacyClient.GetGlobPatterns(context.Background(), &gm.Empty{})
		return &gm.Message{MessageType: gm.Message_ImplementationFileGlobPatternRequest, ImplementationFileGlobPatternResponse: response}, err
	case gm.Message_KillProcessRequest:
		_, err := r.LegacyClient.KillProcess(context.Background(), message.KillProcessRequest)
		return &gm.Message{}, err
	default:
		return nil, nil
	}
}

func (r *GrpcRunner) invokeServiceFor(message *gm.Message) (*gm.Message, error) {
	switch message.MessageType {
	case gm.Message_SuiteDataStoreInit:
		response, err := r.RunnerClient.InitializeSuiteDataStore(context.Background(), message.SuiteDataStoreInitRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_SpecDataStoreInit:
		response, err := r.RunnerClient.InitializeSpecDataStore(context.Background(), message.SpecDataStoreInitRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_ScenarioDataStoreInit:
		response, err := r.RunnerClient.InitializeScenarioDataStore(context.Background(), message.ScenarioDataStoreInitRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_ExecutionStarting:
		response, err := r.RunnerClient.StartExecution(context.Background(), message.ExecutionStartingRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_SpecExecutionStarting:
		response, err := r.RunnerClient.StartSpecExecution(context.Background(), message.SpecExecutionStartingRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_ScenarioExecutionStarting:
		response, err := r.RunnerClient.StartScenarioExecution(context.Background(), message.ScenarioExecutionStartingRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_StepExecutionStarting:
		response, err := r.RunnerClient.StartStepExecution(context.Background(), message.StepExecutionStartingRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_ExecuteStep:
		response, err := r.RunnerClient.ExecuteStep(context.Background(), message.ExecuteStepRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_StepExecutionEnding:
		response, err := r.RunnerClient.FinishStepExecution(context.Background(), message.StepExecutionEndingRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_ScenarioExecutionEnding:
		response, err := r.RunnerClient.FinishScenarioExecution(context.Background(), message.ScenarioExecutionEndingRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_SpecExecutionEnding:
		response, err := r.RunnerClient.FinishSpecExecution(context.Background(), message.SpecExecutionEndingRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_ExecutionEnding:
		response, err := r.RunnerClient.FinishExecution(context.Background(), message.ExecutionEndingRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err

	case gm.Message_CacheFileRequest:
		_, err := r.RunnerClient.CacheFile(context.Background(), message.CacheFileRequest)
		return &gm.Message{}, err
	case gm.Message_StepNamesRequest:
		response, err := r.RunnerClient.GetStepNames(context.Background(), message.StepNamesRequest)
		return &gm.Message{StepNamesResponse: response}, err
	case gm.Message_StepPositionsRequest:
		response, err := r.RunnerClient.GetStepPositions(context.Background(), message.StepPositionsRequest)
		return &gm.Message{StepPositionsResponse: response}, err
	case gm.Message_ImplementationFileListRequest:
		response, err := r.RunnerClient.GetImplementationFiles(context.Background(), &gm.Empty{})
		return &gm.Message{ImplementationFileListResponse: response}, err
	case gm.Message_StubImplementationCodeRequest:
		response, err := r.RunnerClient.ImplementStub(context.Background(), message.StubImplementationCodeRequest)
		return &gm.Message{FileDiff: response}, err
	case gm.Message_RefactorRequest:
		response, err := r.RunnerClient.Refactor(context.Background(), message.RefactorRequest)
		return &gm.Message{MessageType: gm.Message_RefactorResponse, RefactorResponse: response}, err
	case gm.Message_StepNameRequest:
		response, err := r.RunnerClient.GetStepName(context.Background(), message.StepNameRequest)
		return &gm.Message{MessageType: gm.Message_StepNameResponse, StepNameResponse: response}, err
	case gm.Message_ImplementationFileGlobPatternRequest:
		response, err := r.RunnerClient.GetGlobPatterns(context.Background(), &gm.Empty{})
		return &gm.Message{MessageType: gm.Message_ImplementationFileGlobPatternRequest, ImplementationFileGlobPatternResponse: response}, err
	case gm.Message_StepValidateRequest:
		response, err := r.RunnerClient.ValidateStep(context.Background(), message.StepValidateRequest)
		return &gm.Message{MessageType: gm.Message_StepValidateResponse, StepValidateResponse: response}, err
	case gm.Message_KillProcessRequest:
		_, err := r.RunnerClient.Kill(context.Background(), message.KillProcessRequest)
		errStatus, _ := status.FromError(err)
		if errStatus.Code() == codes.Canceled {
			// Ref https://www.grpc.io/docs/guides/error/#general-errors
			// GRPC_STATUS_UNAVAILABLE is thrown when Server is shutting down. Ignore it here.
			return &gm.Message{}, nil
		}
		return &gm.Message{}, err
	default:
		return nil, nil
	}
}

func (r *GrpcRunner) invokeRPC(message *gm.Message, resChan chan *gm.Message, errChan chan error) {
	var res *gm.Message
	var err error
	if r.LegacyClient != nil {
		res, err = r.invokeLegacyLSPService(message)
	} else {
		res, err = r.invokeServiceFor(message)
	}
	if err != nil {
		errChan <- err
	} else {
		resChan <- res
	}
}

func (r *GrpcRunner) executeMessage(message *gm.Message, timeout time.Duration) (*gm.Message, error) {
	resChan := make(chan *gm.Message)
	errChan := make(chan error)
	go r.invokeRPC(message, resChan, errChan)

	timer := setupTimer(timeout, errChan, message.GetMessageType().String())
	defer stopTimer(timer)

	select {
	case response := <-resChan:
		return response, nil
	case err := <-errChan:
		return nil, err
	}
}

// ExecuteMessageWithTimeout process request and give back the response
func (r *GrpcRunner) ExecuteMessageWithTimeout(message *gm.Message) (*gm.Message, error) {
	return r.executeMessage(message, r.Timeout)
}

// ExecuteAndGetStatus executes a given message and response without timeout.
func (r *GrpcRunner) ExecuteAndGetStatus(m *gm.Message) *gm.ProtoExecutionResult {
	res, err := r.executeMessage(m, 0)
	if err != nil {
		return &gauge_messages.ProtoExecutionResult{Failed: true, ErrorMessage: err.Error()}
	}
	return res.ExecutionStatusResponse.ExecutionResult
}

// Alive check if the runner process is still alive
func (r *GrpcRunner) Alive() bool {
	ps := r.cmd.ProcessState
	return ps == nil || !ps.Exited()
}

// Kill closes the grpc connection and kills the process
func (r *GrpcRunner) Kill() error {
	if r.IsExecuting {
		return nil
	}
	m := &gm.Message{
		MessageType:        gm.Message_KillProcessRequest,
		KillProcessRequest: &gm.KillProcessRequest{},
	}
	m, err := r.executeMessage(m, r.Timeout)
	if m == nil || err != nil {
		return err
	}
	if r.conn == nil && r.cmd == nil {
		return nil
	}
	defer r.conn.Close()
	if r.Alive() {
		exited := make(chan bool, 1)
		go func() {
			for {
				if r.Alive() {
					time.Sleep(100 * time.Millisecond)
				} else {
					exited <- true
					return
				}
			}
		}()

		select {
		case done := <-exited:
			if done {
				logger.Debugf(true, "Runner with PID:%d has exited", r.cmd.Process.Pid)
				return nil
			}
		case <-time.After(config.PluginKillTimeout()):
			logger.Warningf(true, "Killing runner with PID:%d forcefully", r.cmd.Process.Pid)
			return r.cmd.Process.Kill()
		}
	}
	return nil
}

// Connection return the client connection
func (r *GrpcRunner) Connection() net.Conn {
	return nil
}

// IsMultithreaded tells if the runner has multithreaded capability
func (r *GrpcRunner) IsMultithreaded() bool {
	return r.info.Multithreaded
}

// Info gives the information about runner
func (r *GrpcRunner) Info() *RunnerInfo {
	return r.info
}

// Pid return the runner's command pid
func (r *GrpcRunner) Pid() int {
	return r.cmd.Process.Pid
}

// StartGrpcRunner makes a connection with grpc server
func StartGrpcRunner(m *manifest.Manifest, stdout io.Writer, stderr io.Writer, timeout time.Duration, shouldWriteToStdout bool) (*GrpcRunner, error) {
	portChan := make(chan string)
	logWriter := &logger.LogWriter{
		Stderr: logger.NewCustomWriter(portChan, stderr, m.Language),
		Stdout: logger.NewCustomWriter(portChan, stdout, m.Language),
	}
	cmd, info, err := runRunnerCommand(m, "0", false, logWriter)
	go func() {
		err = cmd.Wait()
		if err != nil {
			logger.Errorf(true, "Error occurred while waiting for runner process to finish.\nError : %s", err.Error())
		}
	}()
	if err != nil {
		return nil, err
	}

	var port string
	select {
	case port = <-portChan:
		close(portChan)
	case <-time.After(config.RunnerConnectionTimeout()):
		return nil, fmt.Errorf("Timed out connecting to %s", m.Language)
	}
	logger.Debugf(true, "Attempting to connect to grpc server at port: %s", port)
	conn, err := grpc.Dial(fmt.Sprintf("%s:%s", host, port),
		grpc.WithInsecure(),
		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(oneGB), grpc.MaxCallSendMsgSize(oneGB)),
		grpc.WithBlock())
	logger.Debugf(true, "Successfully made the connection with runner with port: %s", port)
	if err != nil {
		return nil, err
	}
	r := &GrpcRunner{cmd: cmd, conn: conn, Timeout: timeout, info: info}

	if info.GRPCSupport {
		r.RunnerClient = gm.NewRunnerClient(conn)
	} else {
		r.LegacyClient = gm.NewLspServiceClient(conn)
	}
	return r, nil
}

func setupTimer(timeout time.Duration, errChan chan error, messageType string) *time.Timer {
	if timeout > 0 {
		return time.AfterFunc(timeout, func() {
			errChan <- fmt.Errorf("request timed out for message %s", messageType)
		})
	}
	return nil
}

func stopTimer(timer *time.Timer) {
	if timer != nil && !timer.Stop() {
		<-timer.C
	}
}
<commit_msg>return err thrown by runner process.<commit_after>// Copyright 2018 ThoughtWorks, Inc.

// This file is part of Gauge.

// Gauge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Gauge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Gauge.  If not, see <http://www.gnu.org/licenses/>.

package runner

import (
	"context"
	"fmt"
	"io"
	"net"
	"os/exec"
	"time"

	"github.com/getgauge/gauge/config"
	"github.com/getgauge/gauge/gauge_messages"
	gm "github.com/getgauge/gauge/gauge_messages"
	"github.com/getgauge/gauge/logger"
	"github.com/getgauge/gauge/manifest"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

const (
	host  = "127.0.0.1"
	oneGB = 1024 * 1024 * 1024
)

// GrpcRunner handles grpc messages.
type GrpcRunner struct {
	cmd          *exec.Cmd
	conn         *grpc.ClientConn
	LegacyClient gm.LspServiceClient
	RunnerClient gm.RunnerClient
	Timeout      time.Duration
	info         *RunnerInfo
	IsExecuting  bool
}

func (r *GrpcRunner) invokeLegacyLSPService(message *gm.Message) (*gm.Message, error) {
	switch message.MessageType {
	case gm.Message_CacheFileRequest:
		_, err := r.LegacyClient.CacheFile(context.Background(), message.CacheFileRequest)
		return &gm.Message{}, err
	case gm.Message_StepNamesRequest:
		response, err := r.LegacyClient.GetStepNames(context.Background(), message.StepNamesRequest)
		return &gm.Message{StepNamesResponse: response}, err
	case gm.Message_StepPositionsRequest:
		response, err := r.LegacyClient.GetStepPositions(context.Background(), message.StepPositionsRequest)
		return &gm.Message{StepPositionsResponse: response}, err
	case gm.Message_ImplementationFileListRequest:
		response, err := r.LegacyClient.GetImplementationFiles(context.Background(), &gm.Empty{})
		return &gm.Message{ImplementationFileListResponse: response}, err
	case gm.Message_StubImplementationCodeRequest:
		response, err := r.LegacyClient.ImplementStub(context.Background(), message.StubImplementationCodeRequest)
		return &gm.Message{FileDiff: response}, err
	case gm.Message_StepValidateRequest:
		response, err := r.LegacyClient.ValidateStep(context.Background(), message.StepValidateRequest)
		return &gm.Message{MessageType: gm.Message_StepValidateResponse, StepValidateResponse: response}, err
	case gm.Message_RefactorRequest:
		response, err := r.LegacyClient.Refactor(context.Background(), message.RefactorRequest)
		return &gm.Message{MessageType: gm.Message_RefactorResponse, RefactorResponse: response}, err
	case gm.Message_StepNameRequest:
		response, err := r.LegacyClient.GetStepName(context.Background(), message.StepNameRequest)
		return &gm.Message{MessageType: gm.Message_StepNameResponse, StepNameResponse: response}, err
	case gm.Message_ImplementationFileGlobPatternRequest:
		response, err := r.LegacyClient.GetGlobPatterns(context.Background(), &gm.Empty{})
		return &gm.Message{MessageType: gm.Message_ImplementationFileGlobPatternRequest, ImplementationFileGlobPatternResponse: response}, err
	case gm.Message_KillProcessRequest:
		_, err := r.LegacyClient.KillProcess(context.Background(), message.KillProcessRequest)
		return &gm.Message{}, err
	default:
		return nil, nil
	}
}

func (r *GrpcRunner) invokeServiceFor(message *gm.Message) (*gm.Message, error) {
	switch message.MessageType {
	case gm.Message_SuiteDataStoreInit:
		response, err := r.RunnerClient.InitializeSuiteDataStore(context.Background(), message.SuiteDataStoreInitRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_SpecDataStoreInit:
		response, err := r.RunnerClient.InitializeSpecDataStore(context.Background(), message.SpecDataStoreInitRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_ScenarioDataStoreInit:
		response, err := r.RunnerClient.InitializeScenarioDataStore(context.Background(), message.ScenarioDataStoreInitRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_ExecutionStarting:
		response, err := r.RunnerClient.StartExecution(context.Background(), message.ExecutionStartingRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_SpecExecutionStarting:
		response, err := r.RunnerClient.StartSpecExecution(context.Background(), message.SpecExecutionStartingRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_ScenarioExecutionStarting:
		response, err := r.RunnerClient.StartScenarioExecution(context.Background(), message.ScenarioExecutionStartingRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_StepExecutionStarting:
		response, err := r.RunnerClient.StartStepExecution(context.Background(), message.StepExecutionStartingRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_ExecuteStep:
		response, err := r.RunnerClient.ExecuteStep(context.Background(), message.ExecuteStepRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_StepExecutionEnding:
		response, err := r.RunnerClient.FinishStepExecution(context.Background(), message.StepExecutionEndingRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_ScenarioExecutionEnding:
		response, err := r.RunnerClient.FinishScenarioExecution(context.Background(), message.ScenarioExecutionEndingRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_SpecExecutionEnding:
		response, err := r.RunnerClient.FinishSpecExecution(context.Background(), message.SpecExecutionEndingRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err
	case gm.Message_ExecutionEnding:
		response, err := r.RunnerClient.FinishExecution(context.Background(), message.ExecutionEndingRequest)
		return &gm.Message{MessageType: gm.Message_ExecutionStatusResponse, ExecutionStatusResponse: response}, err

	case gm.Message_CacheFileRequest:
		_, err := r.RunnerClient.CacheFile(context.Background(), message.CacheFileRequest)
		return &gm.Message{}, err
	case gm.Message_StepNamesRequest:
		response, err := r.RunnerClient.GetStepNames(context.Background(), message.StepNamesRequest)
		return &gm.Message{StepNamesResponse: response}, err
	case gm.Message_StepPositionsRequest:
		response, err := r.RunnerClient.GetStepPositions(context.Background(), message.StepPositionsRequest)
		return &gm.Message{StepPositionsResponse: response}, err
	case gm.Message_ImplementationFileListRequest:
		response, err := r.RunnerClient.GetImplementationFiles(context.Background(), &gm.Empty{})
		return &gm.Message{ImplementationFileListResponse: response}, err
	case gm.Message_StubImplementationCodeRequest:
		response, err := r.RunnerClient.ImplementStub(context.Background(), message.StubImplementationCodeRequest)
		return &gm.Message{FileDiff: response}, err
	case gm.Message_RefactorRequest:
		response, err := r.RunnerClient.Refactor(context.Background(), message.RefactorRequest)
		return &gm.Message{MessageType: gm.Message_RefactorResponse, RefactorResponse: response}, err
	case gm.Message_StepNameRequest:
		response, err := r.RunnerClient.GetStepName(context.Background(), message.StepNameRequest)
		return &gm.Message{MessageType: gm.Message_StepNameResponse, StepNameResponse: response}, err
	case gm.Message_ImplementationFileGlobPatternRequest:
		response, err := r.RunnerClient.GetGlobPatterns(context.Background(), &gm.Empty{})
		return &gm.Message{MessageType: gm.Message_ImplementationFileGlobPatternRequest, ImplementationFileGlobPatternResponse: response}, err
	case gm.Message_StepValidateRequest:
		response, err := r.RunnerClient.ValidateStep(context.Background(), message.StepValidateRequest)
		return &gm.Message{MessageType: gm.Message_StepValidateResponse, StepValidateResponse: response}, err
	case gm.Message_KillProcessRequest:
		_, err := r.RunnerClient.Kill(context.Background(), message.KillProcessRequest)
		errStatus, _ := status.FromError(err)
		if errStatus.Code() == codes.Canceled {
			// Ref https://www.grpc.io/docs/guides/error/#general-errors
			// GRPC_STATUS_UNAVAILABLE is thrown when Server is shutting down. Ignore it here.
			return &gm.Message{}, nil
		}
		return &gm.Message{}, err
	default:
		return nil, nil
	}
}

func (r *GrpcRunner) invokeRPC(message *gm.Message, resChan chan *gm.Message, errChan chan error) {
	var res *gm.Message
	var err error
	if r.LegacyClient != nil {
		res, err = r.invokeLegacyLSPService(message)
	} else {
		res, err = r.invokeServiceFor(message)
	}
	if err != nil {
		errChan <- err
	} else {
		resChan <- res
	}
}

func (r *GrpcRunner) executeMessage(message *gm.Message, timeout time.Duration) (*gm.Message, error) {
	resChan := make(chan *gm.Message)
	errChan := make(chan error)
	go r.invokeRPC(message, resChan, errChan)

	timer := setupTimer(timeout, errChan, message.GetMessageType().String())
	defer stopTimer(timer)

	select {
	case response := <-resChan:
		return response, nil
	case err := <-errChan:
		return nil, err
	}
}

// ExecuteMessageWithTimeout process request and give back the response
func (r *GrpcRunner) ExecuteMessageWithTimeout(message *gm.Message) (*gm.Message, error) {
	return r.executeMessage(message, r.Timeout)
}

// ExecuteAndGetStatus executes a given message and response without timeout.
func (r *GrpcRunner) ExecuteAndGetStatus(m *gm.Message) *gm.ProtoExecutionResult {
	res, err := r.executeMessage(m, 0)
	if err != nil {
		return &gauge_messages.ProtoExecutionResult{Failed: true, ErrorMessage: err.Error()}
	}
	return res.ExecutionStatusResponse.ExecutionResult
}

// Alive check if the runner process is still alive
func (r *GrpcRunner) Alive() bool {
	ps := r.cmd.ProcessState
	return ps == nil || !ps.Exited()
}

// Kill closes the grpc connection and kills the process
func (r *GrpcRunner) Kill() error {
	if r.IsExecuting {
		return nil
	}
	m := &gm.Message{
		MessageType:        gm.Message_KillProcessRequest,
		KillProcessRequest: &gm.KillProcessRequest{},
	}
	m, err := r.executeMessage(m, r.Timeout)
	if m == nil || err != nil {
		return err
	}
	if r.conn == nil && r.cmd == nil {
		return nil
	}
	defer r.conn.Close()
	if r.Alive() {
		exited := make(chan bool, 1)
		go func() {
			for {
				if r.Alive() {
					time.Sleep(100 * time.Millisecond)
				} else {
					exited <- true
					return
				}
			}
		}()

		select {
		case done := <-exited:
			if done {
				logger.Debugf(true, "Runner with PID:%d has exited", r.cmd.Process.Pid)
				return nil
			}
		case <-time.After(config.PluginKillTimeout()):
			logger.Warningf(true, "Killing runner with PID:%d forcefully", r.cmd.Process.Pid)
			return r.cmd.Process.Kill()
		}
	}
	return nil
}

// Connection return the client connection
func (r *GrpcRunner) Connection() net.Conn {
	return nil
}

// IsMultithreaded tells if the runner has multithreaded capability
func (r *GrpcRunner) IsMultithreaded() bool {
	return r.info.Multithreaded
}

// Info gives the information about runner
func (r *GrpcRunner) Info() *RunnerInfo {
	return r.info
}

// Pid return the runner's command pid
func (r *GrpcRunner) Pid() int {
	return r.cmd.Process.Pid
}

// StartGrpcRunner makes a connection with grpc server
func StartGrpcRunner(m *manifest.Manifest, stdout io.Writer, stderr io.Writer, timeout time.Duration, shouldWriteToStdout bool) (*GrpcRunner, error) {
	portChan := make(chan string)
	errChan := make(chan error)
	logWriter := &logger.LogWriter{
		Stderr: logger.NewCustomWriter(portChan, stderr, m.Language),
		Stdout: logger.NewCustomWriter(portChan, stdout, m.Language),
	}
	cmd, info, err := runRunnerCommand(m, "0", false, logWriter)
	if err != nil {
		return nil, fmt.Errorf("Error occurred while starting runner process.\nError : %w", err)
	}

	go func() {
		err = cmd.Wait()
		if err != nil {
			e := fmt.Errorf("Error occurred while waiting for runner process to finish.\nError : %w", err)
			logger.Errorf(true, e.Error())
			errChan <- e
		}
		errChan <- nil
	}()

	var port string
	select {
	case port = <-portChan:
		close(portChan)
	case err = <-errChan:
		return nil, err
	case <-time.After(config.RunnerConnectionTimeout()):
		return nil, fmt.Errorf("Timed out connecting to %s", m.Language)
	}
	logger.Debugf(true, "Attempting to connect to grpc server at port: %s", port)
	conn, err := grpc.Dial(fmt.Sprintf("%s:%s", host, port),
		grpc.WithInsecure(),
		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(oneGB), grpc.MaxCallSendMsgSize(oneGB)),
		grpc.WithBlock())
	logger.Debugf(true, "Successfully made the connection with runner with port: %s", port)
	if err != nil {
		return nil, err
	}
	r := &GrpcRunner{cmd: cmd, conn: conn, Timeout: timeout, info: info}

	if info.GRPCSupport {
		r.RunnerClient = gm.NewRunnerClient(conn)
	} else {
		r.LegacyClient = gm.NewLspServiceClient(conn)
	}
	return r, nil
}

func setupTimer(timeout time.Duration, errChan chan error, messageType string) *time.Timer {
	if timeout > 0 {
		return time.AfterFunc(timeout, func() {
			errChan <- fmt.Errorf("request timed out for message %s", messageType)
		})
	}
	return nil
}

func stopTimer(timer *time.Timer) {
	if timer != nil && !timer.Stop() {
		<-timer.C
	}
}
<|endoftext|>


<commit_before>package handler

import (
	"net/http"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"

	"github.com/Lunchr/luncher-api/db"
	"github.com/Lunchr/luncher-api/router"
	"github.com/Lunchr/luncher-api/session"
	"github.com/deiwin/facebook"
	"golang.org/x/oauth2"
)

// RedirectToFBForLogin returns a handler that redirects the user to Facebook to log in
func RedirectToFBForLogin(sessionManager session.Manager, auther facebook.Authenticator) router.Handler {
	return func(w http.ResponseWriter, r *http.Request) *router.HandlerError {
		session := sessionManager.GetOrInit(w, r)
		redirectURL := auther.AuthURL(session)
		http.Redirect(w, r, redirectURL, http.StatusSeeOther)
		return nil
	}
}

// RedirectedFromFBForLogin returns a handler that receives the user and page tokens for the
// user who has just logged in through Facebook. Updates the user and page
// access tokens in the DB
func RedirectedFromFBForLogin(sessionManager session.Manager, auther facebook.Authenticator, usersCollection db.Users, restaurantsCollection db.Restaurants) router.Handler {
	return func(w http.ResponseWriter, r *http.Request) *router.HandlerError {
		session := sessionManager.GetOrInit(w, r)
		tok, handlerErr := getLongTermToken(session, r, auther)
		if handlerErr != nil {
			return handlerErr
		}
		fbUserID, err := getUserID(tok, auther)
		if err != nil {
			return router.NewHandlerError(err, "Failed to get the user information from Facebook", http.StatusInternalServerError)
		}
		user, err := usersCollection.GetFbID(fbUserID)
		if err == mgo.ErrNotFound {
			return router.NewHandlerError(err, "User not registered", http.StatusForbidden)
		} else if err != nil {
			return router.NewHandlerError(err, "Failed to find the user from DB", http.StatusInternalServerError)
		}
		err = storeAccessTokensInDB(user.ID, fbUserID, tok, session, usersCollection)
		if err != nil {
			return router.NewHandlerError(err, "Failed to persist Facebook login information", http.StatusInternalServerError)
		}
		pageID, handlerErr := getPageID(fbUserID, usersCollection, restaurantsCollection)
		if handlerErr != nil {
			return handlerErr
		}
		if pageID != "" {
			pageAccessToken, err := auther.PageAccessToken(tok, pageID)
			if err != nil {
				if err == facebook.ErrNoSuchPage {
					return router.NewHandlerError(err, "Access denied by Facebook to the managed page", http.StatusForbidden)
				}
				return router.NewHandlerError(err, "Failed to get access to the Facebook page", http.StatusInternalServerError)
			}
			err = usersCollection.SetPageAccessToken(fbUserID, pageAccessToken)
			if err != nil {
				return router.NewHandlerError(err, "Failed to persist Facebook login information", http.StatusInternalServerError)
			}
		}
		http.Redirect(w, r, "/#/admin", http.StatusSeeOther)
		return nil
	}
}

func getLongTermToken(session string, r *http.Request, auther facebook.Authenticator) (*oauth2.Token, *router.HandlerError) {
	tok, err := auther.Token(session, r)
	if err != nil {
		if err == facebook.ErrMissingState {
			return nil, router.NewHandlerError(err, "Expecting a 'state' value", http.StatusBadRequest)
		} else if err == facebook.ErrInvalidState {
			return nil, router.NewHandlerError(err, "Invalid 'state' value", http.StatusForbidden)
		} else if err == facebook.ErrMissingCode {
			return nil, router.NewHandlerError(err, "Expecting a 'code' value", http.StatusBadRequest)
		}
		return nil, router.NewHandlerError(err, "Failed to connect to Facebook", http.StatusInternalServerError)
	}
	return tok, nil
}

func storeAccessTokensInDB(userID bson.ObjectId, fbUserID string, tok *oauth2.Token, sessionID string, usersCollection db.Users) error {
	err := usersCollection.SetAccessToken(fbUserID, *tok)
	if err != nil {
		return err
	}
	return usersCollection.SetSessionID(userID, sessionID)
}

func getUserID(tok *oauth2.Token, auther facebook.Authenticator) (string, error) {
	api := auther.APIConnection(tok)
	user, err := api.Me()
	if err != nil {
		return "", err
	}
	return user.ID, nil
}

func getPageID(userID string, usersCollection db.Users, restaurantsCollection db.Restaurants) (string, *router.HandlerError) {
	user, err := usersCollection.GetFbID(userID)
	if err != nil {
		return "", router.NewHandlerError(err, "Failed to find a user in DB related to this Facebook User ID", http.StatusInternalServerError)
	}
	restaurant, err := restaurantsCollection.GetID(user.RestaurantIDs[0])
	if err != nil {
		return "", router.NewHandlerError(err, "Failed to find the restaurant in the DB", http.StatusInternalServerError)
	}
	return restaurant.FacebookPageID, nil
}
<commit_msg>Avoid duplicate DB queries<commit_after>package handler

import (
	"net/http"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"

	"github.com/Lunchr/luncher-api/db"
	"github.com/Lunchr/luncher-api/db/model"
	"github.com/Lunchr/luncher-api/router"
	"github.com/Lunchr/luncher-api/session"
	"github.com/deiwin/facebook"
	"golang.org/x/oauth2"
)

// RedirectToFBForLogin returns a handler that redirects the user to Facebook to log in
func RedirectToFBForLogin(sessionManager session.Manager, auther facebook.Authenticator) router.Handler {
	return func(w http.ResponseWriter, r *http.Request) *router.HandlerError {
		session := sessionManager.GetOrInit(w, r)
		redirectURL := auther.AuthURL(session)
		http.Redirect(w, r, redirectURL, http.StatusSeeOther)
		return nil
	}
}

// RedirectedFromFBForLogin returns a handler that receives the user and page tokens for the
// user who has just logged in through Facebook. Updates the user and page
// access tokens in the DB
func RedirectedFromFBForLogin(sessionManager session.Manager, auther facebook.Authenticator, usersCollection db.Users, restaurantsCollection db.Restaurants) router.Handler {
	return func(w http.ResponseWriter, r *http.Request) *router.HandlerError {
		session := sessionManager.GetOrInit(w, r)
		tok, handlerErr := getLongTermToken(session, r, auther)
		if handlerErr != nil {
			return handlerErr
		}
		fbUserID, err := getUserID(tok, auther)
		if err != nil {
			return router.NewHandlerError(err, "Failed to get the user information from Facebook", http.StatusInternalServerError)
		}
		user, err := usersCollection.GetFbID(fbUserID)
		if err == mgo.ErrNotFound {
			return router.NewHandlerError(err, "User not registered", http.StatusForbidden)
		} else if err != nil {
			return router.NewHandlerError(err, "Failed to find the user from DB", http.StatusInternalServerError)
		}
		err = storeAccessTokensInDB(user.ID, fbUserID, tok, session, usersCollection)
		if err != nil {
			return router.NewHandlerError(err, "Failed to persist Facebook login information", http.StatusInternalServerError)
		}
		pageID, handlerErr := getPageID(user, restaurantsCollection)
		if handlerErr != nil {
			return handlerErr
		}
		if pageID != "" {
			pageAccessToken, err := auther.PageAccessToken(tok, pageID)
			if err != nil {
				if err == facebook.ErrNoSuchPage {
					return router.NewHandlerError(err, "Access denied by Facebook to the managed page", http.StatusForbidden)
				}
				return router.NewHandlerError(err, "Failed to get access to the Facebook page", http.StatusInternalServerError)
			}
			err = usersCollection.SetPageAccessToken(fbUserID, pageAccessToken)
			if err != nil {
				return router.NewHandlerError(err, "Failed to persist Facebook login information", http.StatusInternalServerError)
			}
		}
		http.Redirect(w, r, "/#/admin", http.StatusSeeOther)
		return nil
	}
}

func getLongTermToken(session string, r *http.Request, auther facebook.Authenticator) (*oauth2.Token, *router.HandlerError) {
	tok, err := auther.Token(session, r)
	if err != nil {
		if err == facebook.ErrMissingState {
			return nil, router.NewHandlerError(err, "Expecting a 'state' value", http.StatusBadRequest)
		} else if err == facebook.ErrInvalidState {
			return nil, router.NewHandlerError(err, "Invalid 'state' value", http.StatusForbidden)
		} else if err == facebook.ErrMissingCode {
			return nil, router.NewHandlerError(err, "Expecting a 'code' value", http.StatusBadRequest)
		}
		return nil, router.NewHandlerError(err, "Failed to connect to Facebook", http.StatusInternalServerError)
	}
	return tok, nil
}

func storeAccessTokensInDB(userID bson.ObjectId, fbUserID string, tok *oauth2.Token, sessionID string, usersCollection db.Users) error {
	err := usersCollection.SetAccessToken(fbUserID, *tok)
	if err != nil {
		return err
	}
	return usersCollection.SetSessionID(userID, sessionID)
}

func getUserID(tok *oauth2.Token, auther facebook.Authenticator) (string, error) {
	api := auther.APIConnection(tok)
	user, err := api.Me()
	if err != nil {
		return "", err
	}
	return user.ID, nil
}

func getPageID(user *model.User, restaurantsCollection db.Restaurants) (string, *router.HandlerError) {
	restaurant, err := restaurantsCollection.GetID(user.RestaurantIDs[0])
	if err != nil {
		return "", router.NewHandlerError(err, "Failed to find the restaurant in the DB", http.StatusInternalServerError)
	}
	return restaurant.FacebookPageID, nil
}
<|endoftext|>


<commit_before>package filer

import (
	"bytes"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

// onMetadataChangeEvent is triggered after filer processed change events from local or remote filers
func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse) {
	f.maybeReloadFilerConfiguration(event)
	f.maybeReloadRemoteStorageConfigurationAndMapping(event)
	f.onBucketEvents(event)
}

func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) {
	message := event.EventNotification
	for _, sig := range message.Signatures {
		if sig == f.Signature {
			return
		}
	}
	if f.DirBucketsPath == event.Directory {
		if filer_pb.IsCreate(event) {
			if message.NewEntry.IsDirectory {
				f.Store.OnBucketCreation(message.NewEntry.Name)
			}
		}
		if filer_pb.IsDelete(event) {
			if message.OldEntry.IsDirectory {
				f.Store.OnBucketDeletion(message.OldEntry.Name)
			}
		}
	}
}

func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataResponse) {
	if DirectoryEtcSeaweedFS != event.Directory {
		if DirectoryEtcSeaweedFS != event.EventNotification.NewParentPath {
			return
		}
	}

	entry := event.EventNotification.NewEntry
	if entry == nil {
		return
	}

	glog.V(0).Infof("procesing %v", event)
	if entry.Name == FilerConfName {
		f.reloadFilerConfiguration(entry)
	}
}

func (f *Filer) readEntry(chunks []*filer_pb.FileChunk, size uint64) ([]byte, error) {
	var buf bytes.Buffer
	err := StreamContent(f.MasterClient, &buf, chunks, 0, int64(size))
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func (f *Filer) reloadFilerConfiguration(entry *filer_pb.Entry) {
	fc := NewFilerConf()
	err := fc.loadFromChunks(f, entry.Content, entry.Chunks, FileSize(entry))
	if err != nil {
		glog.Errorf("read filer conf chunks: %v", err)
		return
	}
	f.FilerConf = fc
}

func (f *Filer) LoadFilerConf() {
	fc := NewFilerConf()
	err := util.Retry("loadFilerConf", func() error {
		return fc.loadFromFiler(f)
	})
	if err != nil {
		glog.Errorf("read filer conf: %v", err)
		return
	}
	f.FilerConf = fc
}

////////////////////////////////////
// load and maintain remote storages
////////////////////////////////////
func (f *Filer) LoadRemoteStorageConfAndMapping() {
	if err := f.RemoteStorage.LoadRemoteStorageConfigurationsAndMapping(f); err != nil {
		glog.Errorf("read remote conf and mapping: %v", err)
		return
	}
}
func (f *Filer) maybeReloadRemoteStorageConfigurationAndMapping(event *filer_pb.SubscribeMetadataResponse) {
	// FIXME add reloading
}
<commit_msg>fix: Buckets are not created and deleted correctly on the filer with the same signature when they are created and deleted<commit_after>package filer

import (
	"bytes"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

// onMetadataChangeEvent is triggered after filer processed change events from local or remote filers
func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse) {
	f.maybeReloadFilerConfiguration(event)
	f.maybeReloadRemoteStorageConfigurationAndMapping(event)
	f.onBucketEvents(event)
}

func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) {
	message := event.EventNotification

	if f.DirBucketsPath == event.Directory {
		if filer_pb.IsCreate(event) {
			if message.NewEntry.IsDirectory {
				f.Store.OnBucketCreation(message.NewEntry.Name)
			}
		}
		if filer_pb.IsDelete(event) {
			if message.OldEntry.IsDirectory {
				f.Store.OnBucketDeletion(message.OldEntry.Name)
			}
		}
	}
}

func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataResponse) {
	if DirectoryEtcSeaweedFS != event.Directory {
		if DirectoryEtcSeaweedFS != event.EventNotification.NewParentPath {
			return
		}
	}

	entry := event.EventNotification.NewEntry
	if entry == nil {
		return
	}

	glog.V(0).Infof("procesing %v", event)
	if entry.Name == FilerConfName {
		f.reloadFilerConfiguration(entry)
	}
}

func (f *Filer) readEntry(chunks []*filer_pb.FileChunk, size uint64) ([]byte, error) {
	var buf bytes.Buffer
	err := StreamContent(f.MasterClient, &buf, chunks, 0, int64(size))
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func (f *Filer) reloadFilerConfiguration(entry *filer_pb.Entry) {
	fc := NewFilerConf()
	err := fc.loadFromChunks(f, entry.Content, entry.Chunks, FileSize(entry))
	if err != nil {
		glog.Errorf("read filer conf chunks: %v", err)
		return
	}
	f.FilerConf = fc
}

func (f *Filer) LoadFilerConf() {
	fc := NewFilerConf()
	err := util.Retry("loadFilerConf", func() error {
		return fc.loadFromFiler(f)
	})
	if err != nil {
		glog.Errorf("read filer conf: %v", err)
		return
	}
	f.FilerConf = fc
}

////////////////////////////////////
// load and maintain remote storages
////////////////////////////////////
func (f *Filer) LoadRemoteStorageConfAndMapping() {
	if err := f.RemoteStorage.LoadRemoteStorageConfigurationsAndMapping(f); err != nil {
		glog.Errorf("read remote conf and mapping: %v", err)
		return
	}
}
func (f *Filer) maybeReloadRemoteStorageConfigurationAndMapping(event *filer_pb.SubscribeMetadataResponse) {
	// FIXME add reloading
}
<|endoftext|>


<commit_before>package shell

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"

	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func init() {
	Commands = append(Commands, &commandVolumeFsck{})
}

type commandVolumeFsck struct {
	env *CommandEnv
}

func (c *commandVolumeFsck) Name() string {
	return "volume.fsck"
}

func (c *commandVolumeFsck) Help() string {
	return `check all volumes to find entries not used by the filer

	Important assumption!!!
		the system is all used by one filer.

	This command works this way:
	1. collect all file ids from all volumes, as set A
	2. collect all file ids from the filer, as set B
	3. find out the set A subtract B

`
}

func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	c.env = commandEnv

	// collect all volume id locations
	volumeIdToServer, err := c.collectVolumeIds()
	if err != nil {
		return fmt.Errorf("failed to collect all volume locations: %v", err)
	}

	// create a temp folder
	tempFolder, err := ioutil.TempDir("", "sw_fsck")
	if err != nil {
		return fmt.Errorf("failed to create temp folder: %v", err)
	}
	// fmt.Fprintf(writer, "working directory: %s\n", tempFolder)

	// collect each volume file ids
	for volumeId, vinfo := range volumeIdToServer {
		err = c.collectOneVolumeFileIds(tempFolder, volumeId, vinfo)
		if err != nil {
			return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, err)
		}
	}

	// collect all filer file ids
	if err = c.collectFilerFileIds(tempFolder, volumeIdToServer); err != nil {
		return fmt.Errorf("failed to collect file ids from filer: %v", err)
	}

	// volume file ids substract filer file ids
	var totalOrphanChunkCount, totalOrphanDataSize uint64
	for volumeId, vinfo := range volumeIdToServer {
		orphanChunkCount, orphanDataSize, checkErr := c.oneVolumeFileIdsSubtractFilerFileIds(tempFolder, volumeId, writer)
		if checkErr != nil {
			return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr)
		}
		totalOrphanChunkCount += orphanChunkCount
		totalOrphanDataSize += orphanDataSize
	}

	if totalOrphanChunkCount > 0 {
		fmt.Fprintf(writer, "\ntotal\t%d orphan entries\t%d bytes not used by filer http://%s:%d/\n",
			totalOrphanChunkCount, totalOrphanDataSize, c.env.option.FilerHost, c.env.option.FilerPort)
		fmt.Fprintf(writer, "This could be normal if multiple filers or no filers are used.\n")
	} else {
		fmt.Fprintf(writer, "no orphan data\n")
	}

	os.RemoveAll(tempFolder)

	return nil
}

func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, volumeId uint32, vinfo VInfo) error {

	return operation.WithVolumeServerClient(vinfo.server, c.env.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {

		copyFileClient, err := volumeServerClient.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{
			VolumeId:                 volumeId,
			Ext:                      ".idx",
			CompactionRevision:       math.MaxUint32,
			StopOffset:               math.MaxInt64,
			Collection:               vinfo.collection,
			IsEcVolume:               vinfo.isEcVolume,
			IgnoreSourceFileNotFound: false,
		})
		if err != nil {
			return fmt.Errorf("failed to start copying volume %d.idx: %v", volumeId, err)
		}

		err = writeToFile(copyFileClient, getVolumeFileIdFile(tempFolder, volumeId))
		if err != nil {
			return fmt.Errorf("failed to copy %d.idx from %s: %v", volumeId, vinfo.server, err)
		}

		return nil

	})

}

func (c *commandVolumeFsck) collectFilerFileIds(tempFolder string, volumeIdToServer map[uint32]VInfo) error {

	files := make(map[uint32]*os.File)
	for vid := range volumeIdToServer {
		dst, openErr := os.OpenFile(getFilerFileIdFile(tempFolder, vid), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		if openErr != nil {
			return fmt.Errorf("failed to create file %s: %v", getFilerFileIdFile(tempFolder, vid), openErr)
		}
		files[vid] = dst
	}
	defer func() {
		for _, f := range files {
			f.Close()
		}
	}()

	type Item struct {
		vid     uint32
		fileKey uint64
	}
	return doTraverseBfsAndSaving(c.env, nil, "/", false, func(outputChan chan interface{}) {
		buffer := make([]byte, 8)
		for item := range outputChan {
			i := item.(*Item)
			util.Uint64toBytes(buffer, i.fileKey)
			files[i.vid].Write(buffer)
		}
	}, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) {
		for _, chunk := range entry.Entry.Chunks {
			outputChan <- &Item{
				vid:     chunk.Fid.VolumeId,
				fileKey: chunk.Fid.FileKey,
			}
		}
		return nil
	})
}

func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(tempFolder string, volumeId uint32, writer io.Writer) (orphanChunkCount, orphanDataSize uint64, err error) {

	db := needle_map.NewMemDb()
	defer db.Close()

	if err = db.LoadFromIdx(getVolumeFileIdFile(tempFolder, volumeId)); err != nil {
		return
	}

	filerFileIdsData, err := ioutil.ReadFile(getFilerFileIdFile(tempFolder, volumeId))
	if err != nil {
		return
	}

	dataLen := len(filerFileIdsData)
	if dataLen%8 != 0 {
		return 0, 0, fmt.Errorf("filer data is corrupted")
	}

	for i := 0; i < len(filerFileIdsData); i += 8 {
		fileKey := util.BytesToUint64(filerFileIdsData[i : i+8])
		db.Delete(types.NeedleId(fileKey))
	}

	db.AscendingVisit(func(n needle_map.NeedleValue) error {
		// fmt.Printf("%d,%x\n", volumeId, n.Key)
		orphanChunkCount++
		orphanDataSize += uint64(n.Size)
		return nil
	})

	if orphanChunkCount > 0 {
		fmt.Fprintf(writer, "volume %d\t%d orphan entries\t%d bytes\n", volumeId, orphanChunkCount, orphanDataSize)
	}

	return

}

type VInfo struct {
	server     string
	collection string
	isEcVolume bool
}

func (c *commandVolumeFsck) collectVolumeIds() (volumeIdToServer map[uint32]VInfo, err error) {

	volumeIdToServer = make(map[uint32]VInfo)
	var resp *master_pb.VolumeListResponse
	err = c.env.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
		resp, err = client.VolumeList(context.Background(), &master_pb.VolumeListRequest{})
		return err
	})
	if err != nil {
		return
	}

	eachDataNode(resp.TopologyInfo, func(dc string, rack RackId, t *master_pb.DataNodeInfo) {
		for _, vi := range t.VolumeInfos {
			volumeIdToServer[vi.Id] = VInfo{
				server:     t.Id,
				collection: vi.Collection,
				isEcVolume: false,
			}
		}
		for _, ecShardInfo := range t.EcShardInfos {
			volumeIdToServer[ecShardInfo.Id] = VInfo{
				server:     t.Id,
				collection: ecShardInfo.Collection,
				isEcVolume: true,
			}
		}
	})

	return
}

func getVolumeFileIdFile(tempFolder string, vid uint32) string {
	return filepath.Join(tempFolder, fmt.Sprintf("%d.idx", vid))
}

func getFilerFileIdFile(tempFolder string, vid uint32) string {
	return filepath.Join(tempFolder, fmt.Sprintf("%d.fid", vid))
}

func writeToFile(client volume_server_pb.VolumeServer_CopyFileClient, fileName string) error {
	flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
	dst, err := os.OpenFile(fileName, flags, 0644)
	if err != nil {
		return nil
	}
	defer dst.Close()

	for {
		resp, receiveErr := client.Recv()
		if receiveErr == io.EOF {
			break
		}
		if receiveErr != nil {
			return fmt.Errorf("receiving %s: %v", fileName, receiveErr)
		}
		dst.Write(resp.FileContent)
	}
	return nil
}
<commit_msg>shell: volume.fsck add options<commit_after>package shell

import (
	"context"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"
	"time"

	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func init() {
	Commands = append(Commands, &commandVolumeFsck{})
}

type commandVolumeFsck struct {
	env *CommandEnv
}

func (c *commandVolumeFsck) Name() string {
	return "volume.fsck"
}

func (c *commandVolumeFsck) Help() string {
	return `check all volumes to find entries not used by the filer

	Important assumption!!!
		the system is all used by one filer.

	This command works this way:
	1. collect all file ids from all volumes, as set A
	2. collect all file ids from the filer, as set B
	3. find out the set A subtract B

`
}

func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	fsckCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	verbose := fsckCommand.Bool("v", false, "verbose mode")
	applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, "<expert only> delete data not referenced by the filer")
	if err = fsckCommand.Parse(args); err != nil {
		return nil
	}

	c.env = commandEnv

	// collect all volume id locations
	volumeIdToServer, err := c.collectVolumeIds(*verbose)
	if err != nil {
		return fmt.Errorf("failed to collect all volume locations: %v", err)
	}

	// create a temp folder
	tempFolder, err := ioutil.TempDir("", "sw_fsck")
	if err != nil {
		return fmt.Errorf("failed to create temp folder: %v", err)
	}
	if *verbose {
		fmt.Fprintf(writer, "working directory: %s\n", tempFolder)
	}
	defer os.RemoveAll(tempFolder)

	// collect each volume file ids
	for volumeId, vinfo := range volumeIdToServer {
		err = c.collectOneVolumeFileIds(tempFolder, volumeId, vinfo, *verbose)
		if err != nil {
			return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, err)
		}
	}

	// collect all filer file ids
	if err = c.collectFilerFileIds(tempFolder, volumeIdToServer, *verbose); err != nil {
		return fmt.Errorf("failed to collect file ids from filer: %v", err)
	}

	// volume file ids substract filer file ids
	var totalInUseCount, totalOrphanChunkCount, totalOrphanDataSize uint64
	for volumeId, vinfo := range volumeIdToServer {
		inUseCount, orphanChunkCount, orphanDataSize, checkErr := c.oneVolumeFileIdsSubtractFilerFileIds(tempFolder, volumeId, writer, *verbose)
		if checkErr != nil {
			return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr)
		}
		totalInUseCount += inUseCount
		totalOrphanChunkCount += orphanChunkCount
		totalOrphanDataSize += orphanDataSize
	}

	if totalOrphanChunkCount == 0 {
		fmt.Fprintf(writer, "no orphan data\n")
	}

	pct := float64(totalOrphanChunkCount*100) / (float64(totalOrphanChunkCount + totalInUseCount))
	fmt.Fprintf(writer, "\nTotal\t\tentries:%d\torphan:%d\t%.2f%%\t%dB\n",
		totalOrphanChunkCount+totalInUseCount, totalOrphanChunkCount, pct, totalOrphanDataSize)

	fmt.Fprintf(writer, "This could be normal if multiple filers or no filers are used.\n")

	if *applyPurging {
		fmt.Fprintf(writer, "\nstarting to destroy your data ...\n")
		time.Sleep(30 * time.Second)
		fmt.Fprintf(writer, "just kidding. Not implemented yet.\n")
	}

	return nil
}

func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, volumeId uint32, vinfo VInfo, verbose bool) error {

	return operation.WithVolumeServerClient(vinfo.server, c.env.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {

		copyFileClient, err := volumeServerClient.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{
			VolumeId:                 volumeId,
			Ext:                      ".idx",
			CompactionRevision:       math.MaxUint32,
			StopOffset:               math.MaxInt64,
			Collection:               vinfo.collection,
			IsEcVolume:               vinfo.isEcVolume,
			IgnoreSourceFileNotFound: false,
		})
		if err != nil {
			return fmt.Errorf("failed to start copying volume %d.idx: %v", volumeId, err)
		}

		err = writeToFile(copyFileClient, getVolumeFileIdFile(tempFolder, volumeId))
		if err != nil {
			return fmt.Errorf("failed to copy %d.idx from %s: %v", volumeId, vinfo.server, err)
		}

		return nil

	})

}

func (c *commandVolumeFsck) collectFilerFileIds(tempFolder string, volumeIdToServer map[uint32]VInfo, verbose bool) error {

	files := make(map[uint32]*os.File)
	for vid := range volumeIdToServer {
		dst, openErr := os.OpenFile(getFilerFileIdFile(tempFolder, vid), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		if openErr != nil {
			return fmt.Errorf("failed to create file %s: %v", getFilerFileIdFile(tempFolder, vid), openErr)
		}
		files[vid] = dst
	}
	defer func() {
		for _, f := range files {
			f.Close()
		}
	}()

	type Item struct {
		vid     uint32
		fileKey uint64
	}
	return doTraverseBfsAndSaving(c.env, nil, "/", false, func(outputChan chan interface{}) {
		buffer := make([]byte, 8)
		for item := range outputChan {
			i := item.(*Item)
			util.Uint64toBytes(buffer, i.fileKey)
			files[i.vid].Write(buffer)
		}
	}, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) {
		for _, chunk := range entry.Entry.Chunks {
			outputChan <- &Item{
				vid:     chunk.Fid.VolumeId,
				fileKey: chunk.Fid.FileKey,
			}
		}
		return nil
	})
}

func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(tempFolder string, volumeId uint32, writer io.Writer, verbose bool) (inUseCount, orphanChunkCount, orphanDataSize uint64, err error) {

	db := needle_map.NewMemDb()
	defer db.Close()

	if err = db.LoadFromIdx(getVolumeFileIdFile(tempFolder, volumeId)); err != nil {
		return
	}

	filerFileIdsData, err := ioutil.ReadFile(getFilerFileIdFile(tempFolder, volumeId))
	if err != nil {
		return
	}

	dataLen := len(filerFileIdsData)
	if dataLen%8 != 0 {
		return 0, 0, 0, fmt.Errorf("filer data is corrupted")
	}

	for i := 0; i < len(filerFileIdsData); i += 8 {
		fileKey := util.BytesToUint64(filerFileIdsData[i : i+8])
		db.Delete(types.NeedleId(fileKey))
		inUseCount++
	}

	db.AscendingVisit(func(n needle_map.NeedleValue) error {
		// fmt.Printf("%d,%x\n", volumeId, n.Key)
		orphanChunkCount++
		orphanDataSize += uint64(n.Size)
		return nil
	})

	if orphanChunkCount > 0 {
		pct := float64(orphanChunkCount*100) / (float64(orphanChunkCount + inUseCount))
		fmt.Fprintf(writer, "volume:%d\tentries:%d\torphan:%d\t%.2f%%\t%dB\n",
			volumeId, orphanChunkCount+inUseCount, orphanChunkCount, pct, orphanDataSize)
	}

	return

}

type VInfo struct {
	server     string
	collection string
	isEcVolume bool
}

func (c *commandVolumeFsck) collectVolumeIds(verbose bool) (volumeIdToServer map[uint32]VInfo, err error) {

	volumeIdToServer = make(map[uint32]VInfo)
	var resp *master_pb.VolumeListResponse
	err = c.env.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
		resp, err = client.VolumeList(context.Background(), &master_pb.VolumeListRequest{})
		return err
	})
	if err != nil {
		return
	}

	eachDataNode(resp.TopologyInfo, func(dc string, rack RackId, t *master_pb.DataNodeInfo) {
		for _, vi := range t.VolumeInfos {
			volumeIdToServer[vi.Id] = VInfo{
				server:     t.Id,
				collection: vi.Collection,
				isEcVolume: false,
			}
		}
		for _, ecShardInfo := range t.EcShardInfos {
			volumeIdToServer[ecShardInfo.Id] = VInfo{
				server:     t.Id,
				collection: ecShardInfo.Collection,
				isEcVolume: true,
			}
		}
	})

	return
}

func getVolumeFileIdFile(tempFolder string, vid uint32) string {
	return filepath.Join(tempFolder, fmt.Sprintf("%d.idx", vid))
}

func getFilerFileIdFile(tempFolder string, vid uint32) string {
	return filepath.Join(tempFolder, fmt.Sprintf("%d.fid", vid))
}

func writeToFile(client volume_server_pb.VolumeServer_CopyFileClient, fileName string) error {
	flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
	dst, err := os.OpenFile(fileName, flags, 0644)
	if err != nil {
		return nil
	}
	defer dst.Close()

	for {
		resp, receiveErr := client.Recv()
		if receiveErr == io.EOF {
			break
		}
		if receiveErr != nil {
			return fmt.Errorf("receiving %s: %v", fileName, receiveErr)
		}
		dst.Write(resp.FileContent)
	}
	return nil
}
<|endoftext|>


<commit_before>package namedParameterQuery

import (
	"testing"
)

/*
	Represents a single test of query parsing.
	Given an [Input] query, if the actual result of parsing
	does not match the [Expected] string, the test fails
*/
type QueryParsingTest struct {
	Name string
	Input string
	Expected string
	ExpectedParameters int
}

/*
	Represents a single test of parameter parsing.
	Given the [Query] and a set of [Parameters], if the actual parameter output
	from GetParsedParameters() matches the given [ExpectedParameters].
	These tests specifically check type of output parameters, too.
*/
type ParameterParsingTest struct {

	Name string
	Query string
	Parameters []TestQueryParameter
	ExpectedParameters []interface{}
}

type TestQueryParameter struct {
	Name string
	Value interface{}
}

func TestQueryParsing(test *testing.T) {

	var query *NamedParameterQuery

	// Each of these represents a single test.
	queryParsingTests := []QueryParsingTest {
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = 1",
			Expected: "SELECT * FROM table WHERE col1 = 1",
			Name: "NoParameter",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = :name",
			Expected: "SELECT * FROM table WHERE col1 = ?",
			ExpectedParameters: 1,
			Name: "SingleParameter",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = :name AND col2 = :occupation",
			Expected: "SELECT * FROM table WHERE col1 = ? AND col2 = ?",
			ExpectedParameters: 2,
			Name: "TwoParameters",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = :name AND col2 = :occupation",
			Expected: "SELECT * FROM table WHERE col1 = ? AND col2 = ?",
			ExpectedParameters: 2,
			Name: "OneParameterMultipleTimes",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 IN (:something, :else)",
			Expected: "SELECT * FROM table WHERE col1 IN (?, ?)",
			ExpectedParameters: 2,
			Name: "ParametersInParenthesis",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = ':literal' AND col2 LIKE ':literal'",
			Expected: "SELECT * FROM table WHERE col1 = ':literal' AND col2 LIKE ':literal'",
			Name: "ParametersInQuotes",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = ':literal' AND col2 = :literal AND col3 LIKE ':literal'",
			Expected: "SELECT * FROM table WHERE col1 = ':literal' AND col2 = ? AND col3 LIKE ':literal'",
			ExpectedParameters: 1,
			Name: "ParametersInQuotes2",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = :foo AND col2 IN (SELECT id FROM tabl2 WHERE col10 = :bar)",
			Expected: "SELECT * FROM table WHERE col1 = ? AND col2 IN (SELECT id FROM tabl2 WHERE col10 = ?)",
			ExpectedParameters: 2,
			Name: "ParametersInSubclause",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = :1234567890 AND col2 = :0987654321",
			Expected: "SELECT * FROM table WHERE col1 = ? AND col2 = ?",
			ExpectedParameters: 2,
			Name: "NumericParameters",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = :ABCDEFGHIJKLMNOPQRSTUVWXYZ",
			Expected: "SELECT * FROM table WHERE col1 = ?",
			ExpectedParameters: 1,
			Name: "CapsParameters",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = :abc123ABC098",
			Expected: "SELECT * FROM table WHERE col1 = ?",
			ExpectedParameters: 1,
			Name: "AltcapsParameters",
		},
	}

	// Run each test.
	for _, parsingTest := range queryParsingTests {

		query = NewNamedParameterQuery(parsingTest.Input)

		// test query texts
		if(query.GetParsedQuery() != parsingTest.Expected) {
			test.Log("Test '", parsingTest.Name, "': Expected query text did not match actual parsed output")
			test.Log("Actual: ", query.GetParsedQuery())
			test.Fail()
		}

		// test parameters
		if(len(query.GetParsedParameters()) != parsingTest.ExpectedParameters) {
			test.Log("Test '", parsingTest.Name, "': Expected parameters did not match actual parsed parameter count")
			test.Fail()
		}
	}
}

/*
	Tests to ensure that setting parameter values turns out correct when using GetParsedParameters().
	These tests ensure correct positioning and type.
*/
func TestParameterReplacement(test *testing.T) {

	var query *NamedParameterQuery

	queryVariableTests := []ParameterParsingTest {
		ParameterParsingTest {

			Name: "SingleStringParameter",
			Query: "SELECT * FROM table WHERE col1 = :foo",
			Parameters: []TestQueryParameter {
				TestQueryParameter {
					Name: "foo",
					Value: "bar",
				},
			},
			ExpectedParameters: []interface{} {
				"bar",
			},
		},
		ParameterParsingTest {

			Name: "TwoStringParameter",
			Query: "SELECT * FROM table WHERE col1 = :foo AND col2 = :foo2",
			Parameters: []TestQueryParameter {
				TestQueryParameter {
					Name: "foo",
					Value: "bar",
				},
				TestQueryParameter {
					Name: "foo2",
					Value: "bart",
				},
			},
			ExpectedParameters: []interface{} {
				"bar", "bart",
			},
		},
		ParameterParsingTest {

			Name: "TwiceOccurringParameter",
			Query: "SELECT * FROM table WHERE col1 = :foo AND col2 = :foo",
			Parameters: []TestQueryParameter {
				TestQueryParameter {
					Name: "foo",
					Value: "bar",
				},
			},
			ExpectedParameters: []interface{} {
				"bar", "bar",
			},
		},
		ParameterParsingTest {

			Name: "ParameterTyping",
			Query: "SELECT * FROM table WHERE col1 = :str AND col2 = :int AND col3 = :pi",
			Parameters: []TestQueryParameter {
				TestQueryParameter {
					Name: "str",
					Value: "foo",
				},
				TestQueryParameter {
					Name: "int",
					Value: 1,
				},
				TestQueryParameter {
					Name: "pi",
					Value: 3.14,
				},
			},
			ExpectedParameters: []interface{} {
				"foo", 1, 3.14,
			},
		},
		ParameterParsingTest {

			Name: "ParameterOrdering",
			Query: "SELECT * FROM table WHERE col1 = :foo AND col2 = :bar AND col3 = :foo AND col4 = :foo AND col5 = :bar",
			Parameters: []TestQueryParameter {
				TestQueryParameter {
					Name: "foo",
					Value: "something",
				},
				TestQueryParameter {
					Name: "bar",
					Value: "else",
				},
			},
			ExpectedParameters: []interface{} {
				"something", "else", "something", "something", "else",
			},
		},
		ParameterParsingTest {

			Name: "ParameterCaseSensitivity",
			Query: "SELECT * FROM table WHERE col1 = :foo AND col2 = :FOO",
			Parameters: []TestQueryParameter {
				TestQueryParameter {
					Name: "foo",
					Value: "baz",
				},
				TestQueryParameter {
					Name: "FOO",
					Value: "quux",
				},
			},
			ExpectedParameters: []interface{} {
				"baz", "quux",
			},
		},
	}

	// run variable tests.
	for _, variableTest := range queryVariableTests {

		// parse query and set values.
		query = NewNamedParameterQuery(variableTest.Query)
		for _, queryVariable := range variableTest.Parameters {
			query.SetValue(queryVariable.Name, queryVariable.Value)
		}

		// Test outputs
		for index, queryVariable := range query.GetParsedParameters() {

			if(queryVariable != variableTest.ExpectedParameters[index]) {
				test.Log("Test '", variableTest.Name, "' did not produce the expected parameter output. Actual: '", queryVariable, "', Expected: '", variableTest.ExpectedParameters[index], "'")
				test.Fail()
			}
		}
	}
}
<commit_msg>Added tests for parameter maps.<commit_after>package namedParameterQuery

import (
	"testing"
)

/*
	Represents a single test of query parsing.
	Given an [Input] query, if the actual result of parsing
	does not match the [Expected] string, the test fails
*/
type QueryParsingTest struct {
	Name string
	Input string
	Expected string
	ExpectedParameters int
}

/*
	Represents a single test of parameter parsing.
	Given the [Query] and a set of [Parameters], if the actual parameter output
	from GetParsedParameters() matches the given [ExpectedParameters].
	These tests specifically check type of output parameters, too.
*/
type ParameterParsingTest struct {

	Name string
	Query string
	Parameters []TestQueryParameter
	ExpectedParameters []interface{}
}

type TestQueryParameter struct {
	Name string
	Value interface{}
}

func TestQueryParsing(test *testing.T) {

	var query *NamedParameterQuery

	// Each of these represents a single test.
	queryParsingTests := []QueryParsingTest {
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = 1",
			Expected: "SELECT * FROM table WHERE col1 = 1",
			Name: "NoParameter",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = :name",
			Expected: "SELECT * FROM table WHERE col1 = ?",
			ExpectedParameters: 1,
			Name: "SingleParameter",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = :name AND col2 = :occupation",
			Expected: "SELECT * FROM table WHERE col1 = ? AND col2 = ?",
			ExpectedParameters: 2,
			Name: "TwoParameters",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = :name AND col2 = :occupation",
			Expected: "SELECT * FROM table WHERE col1 = ? AND col2 = ?",
			ExpectedParameters: 2,
			Name: "OneParameterMultipleTimes",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 IN (:something, :else)",
			Expected: "SELECT * FROM table WHERE col1 IN (?, ?)",
			ExpectedParameters: 2,
			Name: "ParametersInParenthesis",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = ':literal' AND col2 LIKE ':literal'",
			Expected: "SELECT * FROM table WHERE col1 = ':literal' AND col2 LIKE ':literal'",
			Name: "ParametersInQuotes",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = ':literal' AND col2 = :literal AND col3 LIKE ':literal'",
			Expected: "SELECT * FROM table WHERE col1 = ':literal' AND col2 = ? AND col3 LIKE ':literal'",
			ExpectedParameters: 1,
			Name: "ParametersInQuotes2",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = :foo AND col2 IN (SELECT id FROM tabl2 WHERE col10 = :bar)",
			Expected: "SELECT * FROM table WHERE col1 = ? AND col2 IN (SELECT id FROM tabl2 WHERE col10 = ?)",
			ExpectedParameters: 2,
			Name: "ParametersInSubclause",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = :1234567890 AND col2 = :0987654321",
			Expected: "SELECT * FROM table WHERE col1 = ? AND col2 = ?",
			ExpectedParameters: 2,
			Name: "NumericParameters",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = :ABCDEFGHIJKLMNOPQRSTUVWXYZ",
			Expected: "SELECT * FROM table WHERE col1 = ?",
			ExpectedParameters: 1,
			Name: "CapsParameters",
		},
		QueryParsingTest {
			Input: "SELECT * FROM table WHERE col1 = :abc123ABC098",
			Expected: "SELECT * FROM table WHERE col1 = ?",
			ExpectedParameters: 1,
			Name: "AltcapsParameters",
		},
	}

	// Run each test.
	for _, parsingTest := range queryParsingTests {

		query = NewNamedParameterQuery(parsingTest.Input)

		// test query texts
		if(query.GetParsedQuery() != parsingTest.Expected) {
			test.Log("Test '", parsingTest.Name, "': Expected query text did not match actual parsed output")
			test.Log("Actual: ", query.GetParsedQuery())
			test.Fail()
		}

		// test parameters
		if(len(query.GetParsedParameters()) != parsingTest.ExpectedParameters) {
			test.Log("Test '", parsingTest.Name, "': Expected parameters did not match actual parsed parameter count")
			test.Fail()
		}
	}
}

/*
	Tests to ensure that setting parameter values turns out correct when using GetParsedParameters().
	These tests ensure correct positioning and type.
*/
func TestParameterReplacement(test *testing.T) {

	var query *NamedParameterQuery
	var parameterMap map[string]interface{}

	queryVariableTests := []ParameterParsingTest {
		ParameterParsingTest {

			Name: "SingleStringParameter",
			Query: "SELECT * FROM table WHERE col1 = :foo",
			Parameters: []TestQueryParameter {
				TestQueryParameter {
					Name: "foo",
					Value: "bar",
				},
			},
			ExpectedParameters: []interface{} {
				"bar",
			},
		},
		ParameterParsingTest {

			Name: "TwoStringParameter",
			Query: "SELECT * FROM table WHERE col1 = :foo AND col2 = :foo2",
			Parameters: []TestQueryParameter {
				TestQueryParameter {
					Name: "foo",
					Value: "bar",
				},
				TestQueryParameter {
					Name: "foo2",
					Value: "bart",
				},
			},
			ExpectedParameters: []interface{} {
				"bar", "bart",
			},
		},
		ParameterParsingTest {

			Name: "TwiceOccurringParameter",
			Query: "SELECT * FROM table WHERE col1 = :foo AND col2 = :foo",
			Parameters: []TestQueryParameter {
				TestQueryParameter {
					Name: "foo",
					Value: "bar",
				},
			},
			ExpectedParameters: []interface{} {
				"bar", "bar",
			},
		},
		ParameterParsingTest {

			Name: "ParameterTyping",
			Query: "SELECT * FROM table WHERE col1 = :str AND col2 = :int AND col3 = :pi",
			Parameters: []TestQueryParameter {
				TestQueryParameter {
					Name: "str",
					Value: "foo",
				},
				TestQueryParameter {
					Name: "int",
					Value: 1,
				},
				TestQueryParameter {
					Name: "pi",
					Value: 3.14,
				},
			},
			ExpectedParameters: []interface{} {
				"foo", 1, 3.14,
			},
		},
		ParameterParsingTest {

			Name: "ParameterOrdering",
			Query: "SELECT * FROM table WHERE col1 = :foo AND col2 = :bar AND col3 = :foo AND col4 = :foo AND col5 = :bar",
			Parameters: []TestQueryParameter {
				TestQueryParameter {
					Name: "foo",
					Value: "something",
				},
				TestQueryParameter {
					Name: "bar",
					Value: "else",
				},
			},
			ExpectedParameters: []interface{} {
				"something", "else", "something", "something", "else",
			},
		},
		ParameterParsingTest {

			Name: "ParameterCaseSensitivity",
			Query: "SELECT * FROM table WHERE col1 = :foo AND col2 = :FOO",
			Parameters: []TestQueryParameter {
				TestQueryParameter {
					Name: "foo",
					Value: "baz",
				},
				TestQueryParameter {
					Name: "FOO",
					Value: "quux",
				},
			},
			ExpectedParameters: []interface{} {
				"baz", "quux",
			},
		},
	}

	// run variable tests.
	for _, variableTest := range queryVariableTests {

		// parse query and set values.
		parameterMap = make(map[string]interface{}, 8)
		query = NewNamedParameterQuery(variableTest.Query)

		for _, queryVariable := range variableTest.Parameters {
			query.SetValue(queryVariable.Name, queryVariable.Value)
			parameterMap[queryVariable.Name] = queryVariable.Value
		}

		// Test outputs
		for index, queryVariable := range query.GetParsedParameters() {

			if(queryVariable != variableTest.ExpectedParameters[index]) {
				test.Log("Test '", variableTest.Name, "' did not produce the expected parameter output. Actual: '", queryVariable, "', Expected: '", variableTest.ExpectedParameters[index], "'")
				test.Fail()
			}
		}

		query = NewNamedParameterQuery(variableTest.Query)
		query.SetValuesFromMap(parameterMap)

		// test map parameter outputs.
		for index, queryVariable := range query.GetParsedParameters() {

			if(queryVariable != variableTest.ExpectedParameters[index]) {
				test.Log("Test '", variableTest.Name, "' did not produce the expected parameter output when using parameter map. Actual: '", queryVariable, "', Expected: '", variableTest.ExpectedParameters[index], "'")
				test.Fail()
			}
		}
	}
}
<|endoftext|>