<commit_before>package scheduler

import (
	"fmt"
	"flag"
	"net"
	"io/ioutil"

	log "github.com/golang/glog"
	"golang.org/x/net/context"

	mesos "github.com/mesos/mesos-go/mesosproto"
	sched "github.com/mesos/mesos-go/scheduler"
	"github.com/mesos/mesos-go/auth"
	"github.com/mesos/mesos-go/auth/sasl"
	"github.com/mesos/mesos-go/auth/sasl/mech"
	util "github.com/mesos/mesos-go/mesosutil"
	"github.com/gogo/protobuf/proto"
)

var (
	consulServer = flag.String("consul_server", "", "CloudFoundry Consul server to join")
	address = flag.String("address", "127.0.0.1", "Binding address for artifact server")
	authProvider = flag.String("mesos_authentication_provider", sasl.ProviderName,
		fmt.Sprintf("Authentication provider to use, default is SASL that supports mechanisms: %+v", mech.ListSupported()))
	master              = flag.String("master", "127.0.0.1:5050", "Master address <ip:port>")
	mesosAuthPrincipal  = flag.String("mesos_authentication_principal", "", "Mesos authentication principal.")
	mesosAuthSecretFile = flag.String("mesos_authentication_secret_file", "", "Mesos authentication secret file.")
)

func InitializeScheduler(auctionRunner *AuctionRunner) *SchedulerRunner {
	exec := prepareExecutorInfo()
	fwinfo := &mesos.FrameworkInfo{
		User: proto.String(""), // Mesos-go will fill in user.
		Name: proto.String("Diego Scheduler"),
	}

	cred := (*mesos.Credential)(nil)
	if *mesosAuthPrincipal != "" {
		fwinfo.Principal = proto.String(*mesosAuthPrincipal)
		secret, err := ioutil.ReadFile(*mesosAuthSecretFile)
		if err != nil {
			log.Fatal(err)
		}
		cred = &mesos.Credential{
			Principal: proto.String(*mesosAuthPrincipal),
			Secret:    secret,
		}
	}
	bindingAddress := parseIP(*address)

	digoScheduler := NewDiegoScheduler(exec, auctionRunner.LrpAuctions, auctionRunner.TaskAuctions, auctionRunner.AuctionResults)
	config := sched.DriverConfig{
		Scheduler:      digoScheduler,
		Framework:      fwinfo,
		Master:         *master,
		Credential:     cred,
		BindingAddress: bindingAddress,
		WithAuthContext: func(ctx context.Context) context.Context {
			ctx = auth.WithLoginProvider(ctx, *authProvider)
			ctx = sasl.WithBindingAddress(ctx, bindingAddress)
			return ctx
		},
	}
	driver, err := sched.NewMesosSchedulerDriver(config)

	if err != nil {
		log.Fatal("Unable to create a SchedulerDriver ", err.Error())
	}

	return NewSchedulerRunner(driver)

}

func prepareExecutorInfo() *mesos.ExecutorInfo {

	containerType := mesos.ContainerInfo_DOCKER
	containerNetwork := mesos.ContainerInfo_DockerInfo_HOST
	vcapDataVolumeMode := mesos.Volume_RW
	return &mesos.ExecutorInfo{
		ExecutorId: util.NewExecutorID("diego-executor"),
		Name:       proto.String("Diego Executor"),
		Source:     proto.String("diego-executor"),
		Container: &mesos.ContainerInfo{
			Type: &containerType,
			Volumes: []*mesos.Volume {
				&mesos.Volume{
					Mode: &vcapDataVolumeMode,
					ContainerPath: proto.String("/var/vcap/data"),
					HostPath: proto.String("data"),
				},
			},
			Docker: &mesos.ContainerInfo_DockerInfo{
				Image: proto.String("jianhuiz/diego-cell"),
				Network: &containerNetwork,
				Privileged: proto.Bool(true),
			},
		},
		Command: &mesos.CommandInfo {
			Environment: &mesos.Environment{
				Variables: []*mesos.Environment_Variable {
					&mesos.Environment_Variable{
						Name: proto.String("CONSUL_SERVER"),
						Value: proto.String(*consulServer),
					},
				},
			},
			Shell: proto.Bool(false),
			Value: proto.String("/executor"),
			Arguments: []string{ "-logtostderr=true", "-v=5" },
		},
	}
}

func parseIP(address string) net.IP {
	addr, err := net.LookupIP(address)
	if err != nil {
		log.Fatal(err)
	}
	if len(addr) < 1 {
		log.Fatalf("failed to parse IP from address '%v'", address)
	}
	return addr[0]
}
<commit_msg>add -etcd_url parameter as metron uses ETCD of CF; bind /sys/fs/cgroup (maybe not needed)<commit_after>package scheduler

import (
	"fmt"
	"flag"
	"net"
	"io/ioutil"

	log "github.com/golang/glog"
	"golang.org/x/net/context"

	mesos "github.com/mesos/mesos-go/mesosproto"
	sched "github.com/mesos/mesos-go/scheduler"
	"github.com/mesos/mesos-go/auth"
	"github.com/mesos/mesos-go/auth/sasl"
	"github.com/mesos/mesos-go/auth/sasl/mech"
	util "github.com/mesos/mesos-go/mesosutil"
	"github.com/gogo/protobuf/proto"
)

var (
	consulServer = flag.String("consul_server", "", "CloudFoundry Consul server to join")
	etcdUrl = flag.String("etcd_url", "", "CloudFoundry ETCD URL")
	address = flag.String("address", "127.0.0.1", "Binding address for artifact server")
	authProvider = flag.String("mesos_authentication_provider", sasl.ProviderName,
		fmt.Sprintf("Authentication provider to use, default is SASL that supports mechanisms: %+v", mech.ListSupported()))
	master              = flag.String("master", "127.0.0.1:5050", "Master address <ip:port>")
	mesosAuthPrincipal  = flag.String("mesos_authentication_principal", "", "Mesos authentication principal.")
	mesosAuthSecretFile = flag.String("mesos_authentication_secret_file", "", "Mesos authentication secret file.")
)

func InitializeScheduler(auctionRunner *AuctionRunner) *SchedulerRunner {
	exec := prepareExecutorInfo()
	fwinfo := &mesos.FrameworkInfo{
		User: proto.String(""), // Mesos-go will fill in user.
		Name: proto.String("Diego Scheduler"),
	}

	cred := (*mesos.Credential)(nil)
	if *mesosAuthPrincipal != "" {
		fwinfo.Principal = proto.String(*mesosAuthPrincipal)
		secret, err := ioutil.ReadFile(*mesosAuthSecretFile)
		if err != nil {
			log.Fatal(err)
		}
		cred = &mesos.Credential{
			Principal: proto.String(*mesosAuthPrincipal),
			Secret:    secret,
		}
	}
	bindingAddress := parseIP(*address)

	digoScheduler := NewDiegoScheduler(exec, auctionRunner.LrpAuctions, auctionRunner.TaskAuctions, auctionRunner.AuctionResults)
	config := sched.DriverConfig{
		Scheduler:      digoScheduler,
		Framework:      fwinfo,
		Master:         *master,
		Credential:     cred,
		BindingAddress: bindingAddress,
		WithAuthContext: func(ctx context.Context) context.Context {
			ctx = auth.WithLoginProvider(ctx, *authProvider)
			ctx = sasl.WithBindingAddress(ctx, bindingAddress)
			return ctx
		},
	}
	driver, err := sched.NewMesosSchedulerDriver(config)

	if err != nil {
		log.Fatal("Unable to create a SchedulerDriver ", err.Error())
	}

	return NewSchedulerRunner(driver)

}

func prepareExecutorInfo() *mesos.ExecutorInfo {

	containerType := mesos.ContainerInfo_DOCKER
	containerNetwork := mesos.ContainerInfo_DockerInfo_HOST
	vcapDataVolumeMode := mesos.Volume_RW
	return &mesos.ExecutorInfo{
		ExecutorId: util.NewExecutorID("diego-executor"),
		Name:       proto.String("Diego Executor"),
		Source:     proto.String("diego-executor"),
		Container: &mesos.ContainerInfo{
			Type: &containerType,
			Volumes: []*mesos.Volume {
				&mesos.Volume{
					Mode: &vcapDataVolumeMode,
					ContainerPath: proto.String("/var/vcap/data"),
					HostPath: proto.String("data"),
				},
				&mesos.Volume{
					Mode: &vcapDataVolumeMode,
					ContainerPath: proto.String("/sys/fs/cgroup"),
					HostPath: proto.String("/sys/fs/cgroup"),
				},
			},
			Docker: &mesos.ContainerInfo_DockerInfo{
				Image: proto.String("jianhuiz/diego-cell"),
				Network: &containerNetwork,
				Privileged: proto.Bool(true),
			},
		},
		Command: &mesos.CommandInfo {
			Environment: &mesos.Environment{
				Variables: []*mesos.Environment_Variable {
					&mesos.Environment_Variable{
						Name: proto.String("CONSUL_SERVER"),
						Value: proto.String(*consulServer),
					},
					&mesos.Environment_Variable{
						Name: proto.String("ETCD_URL"),
						Value: proto.String(*etcdUrl),
					},
				},
			},
			Shell: proto.Bool(false),
			Value: proto.String("/executor"),
			Arguments: []string{ "-logtostderr=true", "-v=5" },
		},
	}
}

func parseIP(address string) net.IP {
	addr, err := net.LookupIP(address)
	if err != nil {
		log.Fatal(err)
	}
	if len(addr) < 1 {
		log.Fatalf("failed to parse IP from address '%v'", address)
	}
	return addr[0]
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A driver to run the mapping library.
package main

import (
	"context"
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/transform" /* copybara-comment: transform */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/util/jsonutil" /* copybara-comment: jsonutil */
	"github.com/golang/protobuf/proto" /* copybara-comment: proto */

	dhpb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: data_harmonization_go_proto */
	hpb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: harmonization_go_proto */
	httppb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: http_go_proto */
	libpb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: library_go_proto */
	fileutil "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/util/ioutil" /* copybara-comment: ioutil */

)

const fileWritePerm = 0666

type stringSlice []string

// String joins the slice into a semicolon-separated string.
func (s *stringSlice) String() string {
	if s == nil {
		return ""
	}
	return strings.Join(*s, ";")
}

// Set splits the given semicolon-separated string into this stringSlice.
func (s *stringSlice) Set(v string) error {
	*s = strings.Split(v, ";")
	return nil
}

var (
	inputFile         = flag.String("input_file_spec", "", "Input data file or glob pattern (JSON).")
	outputDir         = flag.String("output_dir", "", "Path to the directory where the output will be written to. Leave empty to print to stdout.")
	mappingFile       = flag.String("mapping_file_spec", "", "Mapping file (DHML file).")
	harmonizeCodeDir  = flag.String("harmonize_code_dir_spec", "", "Path to the directory where the FHIR ConceptMaps that should be used for harmozing codes are.")
	harmonizeUnitFile = flag.String("harmonize_unit_spec", "", "Unit harmonization file (textproto)")
	libDir            = flag.String("lib_dir_spec", "", "Path to the directory where the libraries are.")
	dhConfigFile      = flag.String("data_harmonization_config_file_spec", "", "Data Harmonization config (textproto). If this flag is specified, other configs cannot be specified.")

	verbose = flag.Bool("verbose", false, "Enables outputting full trace of operations at the end.")

)

const (
	dhmlExtension      = ".dhml"
	textProtoExtension = ".textproto"
	jsonExtension      = ".json"
	inputExtension     = ".input"
	outputExtension    = ".output.json"
)

func outputFileName(outputPath, inputFilePath string) string {
	f := filepath.Base(inputFilePath)
	f = strings.TrimSuffix(f, jsonExtension)
	f = strings.TrimSuffix(f, inputExtension)
	return filepath.Join(outputPath, f+outputExtension)
}

func libConfigs(path string) []*libpb.LibraryConfig {
	if path == "" {
		return nil
	}
	fs := fileutil.MustReadDir(path, "library dir")

	var libs []*libpb.LibraryConfig
	for _, f := range fs {

		var lbc *libpb.LibraryConfig
		lbc = &libpb.LibraryConfig{UserLibraries: []*libpb.UserLibrary{
			&libpb.UserLibrary{
				Type: hpb.MappingType_MAPPING_LANGUAGE,
				Path: &httppb.Location{Location: &httppb.Location_LocalPath{LocalPath: f}},
			}}}

		libs = append(libs, lbc)
	}
	return libs
}

func codeHarmonizationConfig(path string) *hpb.CodeHarmonizationConfig {
	if path == "" {
		return nil
	}
	fs := fileutil.MustReadDir(path, "code harmonization dir")

	var locs []*httppb.Location
	for _, f := range fs {
		locs = append(locs, &httppb.Location{Location: &httppb.Location_LocalPath{LocalPath: f}})
	}
	return &hpb.CodeHarmonizationConfig{CodeLookup: locs}
}

func unitHarmonizationConfig(path string) *hpb.UnitHarmonizationConfig {
	if path == "" {
		return nil
	}
	return &hpb.UnitHarmonizationConfig{
		UnitConversion: &httppb.Location{
			Location: &httppb.Location_LocalPath{LocalPath: path},
		}}
}

func readInputs(pattern string) []string {
	fs := fileutil.MustReadGlob(pattern, "input_dir")

	var ret []string
	for _, f := range fs {
		fi, err := os.Stat(f)
		if err != nil {
			log.Fatalf("Failed to read input spec: %v", err)
		}
		if fi.IsDir() {
			continue
		}
		ret = append(ret, f)
	}
	return ret
}

func main() {
	flag.Parse()

	var dhConfig *dhpb.DataHarmonizationConfig

	if *dhConfigFile != "" {
		if *mappingFile != "" || *harmonizeCodeDir != "" || *harmonizeUnitFile != "" || *libDir != "" {
			log.Fatal("data_harmonization_config_file_spec flag should not be set along with other configuration flags " +
				"(mapping_file_spec, harmonize_code_dir_spec, harmonize_unit_spec, lib_dir_spec).")
		}
		n := fileutil.MustRead(*dhConfigFile, "data harmonization config")
		if err := proto.UnmarshalText(string(n), dhConfig); err != nil {
			log.Fatalf("Failed to parse data harmonization config")
		}
	} else {
		dhConfig = &dhpb.DataHarmonizationConfig{
			StructureMappingConfig: &hpb.StructureMappingConfig{
				Mapping: &hpb.StructureMappingConfig_MappingLanguageString{
					MappingLanguageString: string(fileutil.MustRead(*mappingFile, "mapping")),
				},
			},
			HarmonizationConfig:     codeHarmonizationConfig(*harmonizeCodeDir),
			UnitHarmonizationConfig: unitHarmonizationConfig(*harmonizeUnitFile),
			LibraryConfig:           libConfigs(*libDir),
		}
	}

	var tr *transform.Transformer
	var err error

	if tr, err = transform.NewTransformer(context.Background(), dhConfig); err != nil {
		log.Fatalf("Failed to load mapping config: %v", err)
	}

	tconfig := transform.TransformationConfigs{
		LogTrace: *verbose,
	}

	for _, f := range readInputs(*inputFile) {
		i := fileutil.MustRead(f, "input")

		ji := &jsonutil.JSONContainer{}
		if err := ji.UnmarshalJSON(i); err != nil {
			log.Fatalf("Failed to parse input JSON in file %v: %v", f, err)
		}

		res, err := tr.Transform(ji, tconfig)
		if err != nil {
			log.Fatalf("Mapping failed for input file %v: %v", f, err)
		}

		bres, err := json.MarshalIndent(res, "", "  ")
		if err != nil {
			log.Fatalf("Failed to serialize output: %v", err)
		}

		op := outputFileName(*outputDir, f)
		if *outputDir == "" {
			log.Printf("File %q\n\n%s\n", op, string(bres))
		} else {
			if err := ioutil.WriteFile(op, bres, fileWritePerm); err != nil {
				log.Fatalf("Could not write output file %q: %v", op, err)
			}
		}
	}
}
<commit_msg>Fix NPE when reading from DH config.<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A driver to run the mapping library.
package main

import (
	"context"
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/transform" /* copybara-comment: transform */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/util/jsonutil" /* copybara-comment: jsonutil */
	"github.com/golang/protobuf/proto" /* copybara-comment: proto */

	dhpb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: data_harmonization_go_proto */
	hpb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: harmonization_go_proto */
	httppb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: http_go_proto */
	libpb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: library_go_proto */
	fileutil "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/util/ioutil" /* copybara-comment: ioutil */

)

const fileWritePerm = 0666

type stringSlice []string

// String joins the slice into a semicolon-separated string.
func (s *stringSlice) String() string {
	if s == nil {
		return ""
	}
	return strings.Join(*s, ";")
}

// Set splits the given semicolon-separated string into this stringSlice.
func (s *stringSlice) Set(v string) error {
	*s = strings.Split(v, ";")
	return nil
}

var (
	inputFile         = flag.String("input_file_spec", "", "Input data file or glob pattern (JSON).")
	outputDir         = flag.String("output_dir", "", "Path to the directory where the output will be written to. Leave empty to print to stdout.")
	mappingFile       = flag.String("mapping_file_spec", "", "Mapping file (DHML file).")
	harmonizeCodeDir  = flag.String("harmonize_code_dir_spec", "", "Path to the directory where the FHIR ConceptMaps that should be used for harmozing codes are.")
	harmonizeUnitFile = flag.String("harmonize_unit_spec", "", "Unit harmonization file (textproto)")
	libDir            = flag.String("lib_dir_spec", "", "Path to the directory where the libraries are.")
	dhConfigFile      = flag.String("data_harmonization_config_file_spec", "", "Data Harmonization config (textproto). If this flag is specified, other configs cannot be specified.")

	verbose = flag.Bool("verbose", false, "Enables outputting full trace of operations at the end.")

)

const (
	dhmlExtension      = ".dhml"
	textProtoExtension = ".textproto"
	jsonExtension      = ".json"
	inputExtension     = ".input"
	outputExtension    = ".output.json"
)

func outputFileName(outputPath, inputFilePath string) string {
	f := filepath.Base(inputFilePath)
	f = strings.TrimSuffix(f, jsonExtension)
	f = strings.TrimSuffix(f, inputExtension)
	return filepath.Join(outputPath, f+outputExtension)
}

func libConfigs(path string) []*libpb.LibraryConfig {
	if path == "" {
		return nil
	}
	fs := fileutil.MustReadDir(path, "library dir")

	var libs []*libpb.LibraryConfig
	for _, f := range fs {

		var lbc *libpb.LibraryConfig
		lbc = &libpb.LibraryConfig{UserLibraries: []*libpb.UserLibrary{
			&libpb.UserLibrary{
				Type: hpb.MappingType_MAPPING_LANGUAGE,
				Path: &httppb.Location{Location: &httppb.Location_LocalPath{LocalPath: f}},
			}}}

		libs = append(libs, lbc)
	}
	return libs
}

func codeHarmonizationConfig(path string) *hpb.CodeHarmonizationConfig {
	if path == "" {
		return nil
	}
	fs := fileutil.MustReadDir(path, "code harmonization dir")

	var locs []*httppb.Location
	for _, f := range fs {
		locs = append(locs, &httppb.Location{Location: &httppb.Location_LocalPath{LocalPath: f}})
	}
	return &hpb.CodeHarmonizationConfig{CodeLookup: locs}
}

func unitHarmonizationConfig(path string) *hpb.UnitHarmonizationConfig {
	if path == "" {
		return nil
	}
	return &hpb.UnitHarmonizationConfig{
		UnitConversion: &httppb.Location{
			Location: &httppb.Location_LocalPath{LocalPath: path},
		}}
}

func readInputs(pattern string) []string {
	fs := fileutil.MustReadGlob(pattern, "input_dir")

	var ret []string
	for _, f := range fs {
		fi, err := os.Stat(f)
		if err != nil {
			log.Fatalf("Failed to read input spec: %v", err)
		}
		if fi.IsDir() {
			continue
		}
		ret = append(ret, f)
	}
	return ret
}

func main() {
	flag.Parse()

	dhConfig := &dhpb.DataHarmonizationConfig{}

	if *dhConfigFile != "" {
		if *mappingFile != "" || *harmonizeCodeDir != "" || *harmonizeUnitFile != "" || *libDir != "" {
			log.Fatal("data_harmonization_config_file_spec flag should not be set along with other configuration flags " +
				"(mapping_file_spec, harmonize_code_dir_spec, harmonize_unit_spec, lib_dir_spec).")
		}
		n := fileutil.MustRead(*dhConfigFile, "data harmonization config")
		if err := proto.UnmarshalText(string(n), dhConfig); err != nil {
			log.Fatalf("Failed to parse data harmonization config")
		}
	} else {
		dhConfig = &dhpb.DataHarmonizationConfig{
			StructureMappingConfig: &hpb.StructureMappingConfig{
				Mapping: &hpb.StructureMappingConfig_MappingLanguageString{
					MappingLanguageString: string(fileutil.MustRead(*mappingFile, "mapping")),
				},
			},
			HarmonizationConfig:     codeHarmonizationConfig(*harmonizeCodeDir),
			UnitHarmonizationConfig: unitHarmonizationConfig(*harmonizeUnitFile),
			LibraryConfig:           libConfigs(*libDir),
		}
	}

	var tr *transform.Transformer
	var err error

	if tr, err = transform.NewTransformer(context.Background(), dhConfig); err != nil {
		log.Fatalf("Failed to load mapping config: %v", err)
	}

	tconfig := transform.TransformationConfigs{
		LogTrace: *verbose,
	}

	for _, f := range readInputs(*inputFile) {
		i := fileutil.MustRead(f, "input")

		ji := &jsonutil.JSONContainer{}
		if err := ji.UnmarshalJSON(i); err != nil {
			log.Fatalf("Failed to parse input JSON in file %v: %v", f, err)
		}

		res, err := tr.Transform(ji, tconfig)
		if err != nil {
			log.Fatalf("Mapping failed for input file %v: %v", f, err)
		}

		bres, err := json.MarshalIndent(res, "", "  ")
		if err != nil {
			log.Fatalf("Failed to serialize output: %v", err)
		}

		op := outputFileName(*outputDir, f)
		if *outputDir == "" {
			log.Printf("File %q\n\n%s\n", op, string(bres))
		} else {
			if err := ioutil.WriteFile(op, bres, fileWritePerm); err != nil {
				log.Fatalf("Could not write output file %q: %v", op, err)
			}
		}
	}
}
<|endoftext|>


<commit_before>package action

import (
	"errors"
	"fmt"
	"io"
	"runtime"
	"strconv"
	"strings"
	"time"
	"unicode"

	"github.com/mithrandie/csvq/lib/cmd"
	"github.com/mithrandie/csvq/lib/parser"
	"github.com/mithrandie/csvq/lib/query"
)

func Run(input string, sourceFile string) error {
	SetSignalHandler()
	start := time.Now()

	defer func() {
		query.ReleaseResources()
		showStats(start)
	}()

	query.UpdateWaitTimeout()

	statements, err := parser.Parse(input, sourceFile)
	if err != nil {
		syntaxErr := err.(*parser.SyntaxError)
		return query.NewSyntaxError(syntaxErr.Message, syntaxErr.Line, syntaxErr.Char, syntaxErr.SourceFile)
	}

	proc := query.NewProcedure()
	flow, err := proc.Execute(statements)

	if err == nil && flow == query.TERMINATE {
		err = query.Commit(nil, proc.Filter)
	}

	if err != nil {
		return err
	}

	createSelectLog()

	return nil
}

func LaunchInteractiveShell() error {
	if cmd.IsReadableFromPipeOrRedirection() {
		return errors.New("input from pipe or redirection cannot be used in interactive shell")
	}
	cmd.SetWriteEncoding("UTF8")
	cmd.SetOut("")
	cmd.SetFormat("TEXT")
	cmd.SetWriteEncoding(",")
	cmd.SetWithoutHeader(false)

	SetSignalHandler()

	defer func() {
		query.ReleaseResources()
	}()

	var err error
	query.UpdateWaitTimeout()

	term, err := cmd.NewTerminal()
	if err != nil {
		return err
	}
	cmd.Terminal = term
	defer func() {
		cmd.Terminal.Teardown()
		cmd.Terminal = nil
	}()

	proc := query.NewProcedure()
	lines := []string{}

	for {
		line, e := cmd.Terminal.ReadLine()
		if e != nil {
			if e == io.EOF {
				break
			}
			return e
		}

		line = strings.TrimRightFunc(line, unicode.IsSpace)

		if len(lines) < 1 && len(line) < 1 {
			continue
		}

		if 0 < len(line) && line[len(line)-1] == '\\' {
			lines = append(lines, line[:len(line)-1])
			cmd.Terminal.SetContinuousPrompt()
			continue
		}

		lines = append(lines, line)

		if len(line) < 1 || line[len(line)-1] != ';' {
			cmd.Terminal.SetContinuousPrompt()
			continue
		}

		cmd.Terminal.SaveHistory(strings.Join(lines, " "))

		statements, e := parser.Parse(strings.Join(lines, "\n"), "")
		if e != nil {
			syntaxErr := e.(*parser.SyntaxError)
			e = query.NewSyntaxError(syntaxErr.Message, syntaxErr.Line, syntaxErr.Char, syntaxErr.SourceFile)
			if werr := cmd.Terminal.Write(e.Error() + "\n"); werr != nil {
				return werr
			}
			lines = lines[:0]
			cmd.Terminal.SetPrompt()
			continue
		}

		flow, e := proc.Execute(statements)
		if e != nil {
			if ex, ok := e.(*query.Exit); ok {
				err = ex
				break
			} else {
				if werr := cmd.Terminal.Write(e.Error() + "\n"); werr != nil {
					return werr
				}
				lines = lines[:0]
				cmd.Terminal.SetPrompt()
				continue
			}
		}

		if flow == query.EXIT {
			break
		}

		lines = lines[:0]
		cmd.Terminal.SetPrompt()
	}

	return err
}

func createSelectLog() error {
	flags := cmd.GetFlags()
	selectLog := query.ReadSelectLog()
	if 0 < len(flags.OutFile) && 0 < len(selectLog) {
		if err := cmd.CreateFile(flags.OutFile, selectLog); err != nil {
			return err
		}
	}
	return nil
}

func showStats(start time.Time) {
	flags := cmd.GetFlags()
	if !flags.Stats {
		return
	}
	var mem runtime.MemStats
	runtime.ReadMemStats(&mem)

	exectime := cmd.HumarizeNumber(fmt.Sprintf("%f", time.Since(start).Seconds()))
	alloc := cmd.HumarizeNumber(fmt.Sprintf("%v", mem.Alloc))
	talloc := cmd.HumarizeNumber(fmt.Sprintf("%v", mem.TotalAlloc))
	sys := cmd.HumarizeNumber(fmt.Sprintf("%v", mem.HeapSys))
	mallocs := cmd.HumarizeNumber(fmt.Sprintf("%v", mem.Mallocs))
	frees := cmd.HumarizeNumber(fmt.Sprintf("%v", mem.Frees))

	width := len(exectime)
	for _, v := range []string{alloc, talloc, sys, mallocs, frees} {
		if width < len(v) {
			width = len(v)
		}
	}
	w := strconv.Itoa(width)

	stats := fmt.Sprintf(
		" TotalTime: %"+w+"[2]s seconds %[1]s"+
			"     Alloc: %"+w+"[3]s bytes %[1]s"+
			"TotalAlloc: %"+w+"[4]s bytes %[1]s"+
			"   HeapSys: %"+w+"[5]s bytes %[1]s"+
			"   Mallocs: %"+w+"[6]s objects %[1]s"+
			"     Frees: %"+w+"[7]s objects %[1]s",
		"\n",
		exectime,
		alloc,
		talloc,
		sys,
		mallocs,
		frees,
	)
	cmd.ToStdout(stats)
}
<commit_msg>Add start-up message to the interactive shell.<commit_after>package action

import (
	"errors"
	"fmt"
	"io"
	"runtime"
	"strconv"
	"strings"
	"time"
	"unicode"

	"github.com/mithrandie/csvq/lib/cmd"
	"github.com/mithrandie/csvq/lib/parser"
	"github.com/mithrandie/csvq/lib/query"
)

func Run(input string, sourceFile string) error {
	SetSignalHandler()
	start := time.Now()

	defer func() {
		query.ReleaseResources()
		showStats(start)
	}()

	query.UpdateWaitTimeout()

	statements, err := parser.Parse(input, sourceFile)
	if err != nil {
		syntaxErr := err.(*parser.SyntaxError)
		return query.NewSyntaxError(syntaxErr.Message, syntaxErr.Line, syntaxErr.Char, syntaxErr.SourceFile)
	}

	proc := query.NewProcedure()
	flow, err := proc.Execute(statements)

	if err == nil && flow == query.TERMINATE {
		err = query.Commit(nil, proc.Filter)
	}

	if err != nil {
		return err
	}

	createSelectLog()

	return nil
}

func LaunchInteractiveShell() error {
	if cmd.IsReadableFromPipeOrRedirection() {
		return errors.New("input from pipe or redirection cannot be used in interactive shell")
	}
	cmd.SetWriteEncoding("UTF8")
	cmd.SetOut("")
	cmd.SetFormat("TEXT")
	cmd.SetWriteEncoding(",")
	cmd.SetWithoutHeader(false)

	SetSignalHandler()

	defer func() {
		query.ReleaseResources()
	}()

	var err error
	query.UpdateWaitTimeout()

	term, err := cmd.NewTerminal()
	if err != nil {
		return err
	}
	cmd.Terminal = term
	defer func() {
		cmd.Terminal.Teardown()
		cmd.Terminal = nil
	}()

	StartUpMessage := "csvq interactive shell\n" +
		"Press Ctrl+D or execute \"EXIT;\" to terminate this shell.\n\n"
	if werr := cmd.Terminal.Write(StartUpMessage); werr != nil {
		return werr
	}

	proc := query.NewProcedure()
	lines := []string{}

	for {
		line, e := cmd.Terminal.ReadLine()
		if e != nil {
			if e == io.EOF {
				break
			}
			return e
		}

		line = strings.TrimRightFunc(line, unicode.IsSpace)

		if len(lines) < 1 && len(line) < 1 {
			continue
		}

		if 0 < len(line) && line[len(line)-1] == '\\' {
			lines = append(lines, line[:len(line)-1])
			cmd.Terminal.SetContinuousPrompt()
			continue
		}

		lines = append(lines, line)

		if len(line) < 1 || line[len(line)-1] != ';' {
			cmd.Terminal.SetContinuousPrompt()
			continue
		}

		cmd.Terminal.SaveHistory(strings.Join(lines, " "))

		statements, e := parser.Parse(strings.Join(lines, "\n"), "")
		if e != nil {
			syntaxErr := e.(*parser.SyntaxError)
			e = query.NewSyntaxError(syntaxErr.Message, syntaxErr.Line, syntaxErr.Char, syntaxErr.SourceFile)
			if werr := cmd.Terminal.Write(e.Error() + "\n"); werr != nil {
				return werr
			}
			lines = lines[:0]
			cmd.Terminal.SetPrompt()
			continue
		}

		flow, e := proc.Execute(statements)
		if e != nil {
			if ex, ok := e.(*query.Exit); ok {
				err = ex
				break
			} else {
				if werr := cmd.Terminal.Write(e.Error() + "\n"); werr != nil {
					return werr
				}
				lines = lines[:0]
				cmd.Terminal.SetPrompt()
				continue
			}
		}

		if flow == query.EXIT {
			break
		}

		lines = lines[:0]
		cmd.Terminal.SetPrompt()
	}

	return err
}

func createSelectLog() error {
	flags := cmd.GetFlags()
	selectLog := query.ReadSelectLog()
	if 0 < len(flags.OutFile) && 0 < len(selectLog) {
		if err := cmd.CreateFile(flags.OutFile, selectLog); err != nil {
			return err
		}
	}
	return nil
}

func showStats(start time.Time) {
	flags := cmd.GetFlags()
	if !flags.Stats {
		return
	}
	var mem runtime.MemStats
	runtime.ReadMemStats(&mem)

	exectime := cmd.HumarizeNumber(fmt.Sprintf("%f", time.Since(start).Seconds()))
	alloc := cmd.HumarizeNumber(fmt.Sprintf("%v", mem.Alloc))
	talloc := cmd.HumarizeNumber(fmt.Sprintf("%v", mem.TotalAlloc))
	sys := cmd.HumarizeNumber(fmt.Sprintf("%v", mem.HeapSys))
	mallocs := cmd.HumarizeNumber(fmt.Sprintf("%v", mem.Mallocs))
	frees := cmd.HumarizeNumber(fmt.Sprintf("%v", mem.Frees))

	width := len(exectime)
	for _, v := range []string{alloc, talloc, sys, mallocs, frees} {
		if width < len(v) {
			width = len(v)
		}
	}
	w := strconv.Itoa(width)

	stats := fmt.Sprintf(
		" TotalTime: %"+w+"[2]s seconds %[1]s"+
			"     Alloc: %"+w+"[3]s bytes %[1]s"+
			"TotalAlloc: %"+w+"[4]s bytes %[1]s"+
			"   HeapSys: %"+w+"[5]s bytes %[1]s"+
			"   Mallocs: %"+w+"[6]s objects %[1]s"+
			"     Frees: %"+w+"[7]s objects %[1]s",
		"\n",
		exectime,
		alloc,
		talloc,
		sys,
		mallocs,
		frees,
	)
	cmd.ToStdout(stats)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"
)

type (
	Item struct {
		chip    bool
		element string
		floor   int
	}

	State struct {
		cost     int
		elevator int
		gens     []Item
		chips    []Item
	}

	Index struct {
		index int
		chip  bool
	}

	Search struct {
		cost   int
		target State
		open   []State
		closed []State
	}
)

func (i *Item) up(limit int) bool {
	if i.floor+1 < limit {
		i.floor += 1
		return true
	} else {
		return false
	}
}

func (i *Item) down(limit int) bool {
	if i.floor-1 >= limit {
		i.floor -= 1
		return true
	} else {
		return false
	}
}

func NewState() State {
	return State{
		cost:     0,
		elevator: 0,
		gens: []Item{
			Item{
				chip:    false,
				element: "s",
				floor:   0,
			},
			Item{
				chip:    false,
				element: "p",
				floor:   0,
			},
			Item{
				chip:    false,
				element: "t",
				floor:   1,
			},
			Item{
				chip:    false,
				element: "r",
				floor:   1,
			},
			Item{
				chip:    false,
				element: "c",
				floor:   1,
			},
		},
		chips: []Item{
			Item{
				chip:    true,
				element: "s",
				floor:   0,
			},
			Item{
				chip:    true,
				element: "p",
				floor:   0,
			},
			Item{
				chip:    true,
				element: "t",
				floor:   2,
			},
			Item{
				chip:    true,
				element: "r",
				floor:   1,
			},
			Item{
				chip:    true,
				element: "c",
				floor:   1,
			},
		},
	}
}

func NewTargetState() State {
	return State{
		cost:     0,
		elevator: 4,
		gens: []Item{
			Item{
				chip:    false,
				element: "s",
				floor:   4,
			},
			Item{
				chip:    false,
				element: "p",
				floor:   4,
			},
			Item{
				chip:    false,
				element: "t",
				floor:   4,
			},
			Item{
				chip:    false,
				element: "r",
				floor:   4,
			},
			Item{
				chip:    false,
				element: "c",
				floor:   4,
			},
		},
		chips: []Item{
			Item{
				chip:    true,
				element: "s",
				floor:   4,
			},
			Item{
				chip:    true,
				element: "p",
				floor:   4,
			},
			Item{
				chip:    true,
				element: "t",
				floor:   4,
			},
			Item{
				chip:    true,
				element: "r",
				floor:   4,
			},
			Item{
				chip:    true,
				element: "c",
				floor:   4,
			},
		},
	}
}

func EqualStates(a, b State) bool {
	if a.elevator != b.elevator {
		return false
	}

	for i, _ := range a.chips {
		if a.chips[i] != b.chips[i] || a.gens[i] != b.gens[i] {
			return false
		}
	}

	return true
}

func (s *State) copy() State {
	a := []Item{}
	b := []Item{}
	copy(a, s.gens)
	copy(b, s.chips)
	return State{
		cost:     s.cost,
		elevator: s.elevator,
		gens:     a,
		chips:    b,
	}
}

func (s *State) elevator_up(limit int) bool {
	if s.elevator+1 < limit {
		s.elevator++
		return true
	} else {
		return false
	}
}

func (s *State) elevator_down(limit int) bool {
	if s.elevator-1 >= limit {
		s.elevator--
		return true
	} else {
		return false
	}
}

func (s *State) up(chip bool, element, limit int) bool {
	if chip {
		return s.chips[element].up(limit)
	} else {
		return s.gens[element].up(limit)
	}
}

func (s *State) down(chip bool, element, limit int) bool {
	if chip {
		return s.chips[element].down(limit)
	} else {
		return s.gens[element].down(limit)
	}
}

func (s *State) incCost() {
	s.cost++
}

func (s *State) nextStates() []State {
	k := []State{}
	indicies := []Index{}
	for i := 0; i < len(s.chips); i++ {
		if s.elevator == s.chips[i].floor {
			indicies = append(indicies, Index{i, true})
		}
	}
	for i := 0; i < len(s.gens); i++ {
		if s.elevator == s.gens[i].floor {
			indicies = append(indicies, Index{i, false})
		}
	}

	for _, i := range indicies {
		j := s.copy()
		if j.elevator_up(4) && j.up(i.chip, i.index, 4) {
			j.incCost()
			k = append(k, j)
		}
		j = s.copy()
		if j.elevator_down(0) && j.down(i.chip, i.index, 0) {
			j.incCost()
			k = append(k, j)
		}
	}
	for n, i := range indicies {
		for l := n + 1; l < len(indicies); l++ {
			j := s.copy()
			if j.elevator_up(4) && j.up(i.chip, i.index, 4) && j.up(indicies[n].chip, indicies[n].index, 4) {
				j.incCost()
				k = append(k, j)
			}
			j = s.copy()
			if j.elevator_down(0) && j.down(i.chip, i.index, 0) && j.down(indicies[n].chip, indicies[n].index, 0) {
				j.incCost()
				k = append(k, j)
			}
		}
	}

	return k
}

func NewSearch(init_state, target_state State) *Search {
	return &Search{
		target: target_state,
		open:   []State{init_state},
		closed: []State{},
	}
}

func (s *Search) addClosed(states []State) {
state_loop:
	for _, i := range states {
		for _, j := range s.closed {
			if EqualStates(i, j) {
				continue state_loop
			}
		}
		s.closed = append(s.closed, i)
	}
}

func (s *Search) addOpen(states []State) {
state_loop:
	for _, i := range states {
		for _, j := range s.closed {
			if EqualStates(i, j) {
				continue state_loop
			}
		}
		for _, j := range s.open {
			if EqualStates(i, j) {
				continue state_loop
			}
		}
		s.open = append(s.open, i)
	}
}

func (s *Search) search() bool {
	fmt.Println("\n\nrun")
	fmt.Println("\n\n", s.closed)
	fmt.Println("\n\n", s.open)
	if len(s.open) < 1 {
		return false
	}
	current := s.open[0]
	s.open = s.open[1:]
	if EqualStates(s.target, current) {
		s.cost = current.cost
		return false
	} else {
		s.addClosed([]State{current})
		s.addOpen(current.nextStates())
		return true
	}
}

func main() {
	start := time.Now()

	k := NewSearch(NewState(), NewTargetState())

	for k.search() {
	}

	fmt.Println("Cost: ", k.cost)

	fmt.Println(fmt.Sprintf("time elapsed: %s", time.Since(start)))
}
<commit_msg>change day 11<commit_after>package main

import (
	"fmt"
	"time"
)

type (
	Pair struct {
		gen  int
		chip int
	}

	State struct {
		cost      int
		heuristic int
		elevator  int
		pairs     []Pair
	}
)

func NewPair(gen, chip int) Pair {
	return Pair{
		gen:  gen,
		chip: chip,
	}
}

func (p *Pair) up(limit int, chip bool) (Pair, bool) {
	if chip {
		k := p.chip + 1
		if k >= limit {
			return NewPair(0, 0), false
		}
		return NewPair(p.gen, k), true
	} else {
		k := p.gen + 1
		if k >= limit {
			return NewPair(0, 0), false
		}
		return NewPair(k, p.chip), true
	}
}

func (p *Pair) down(limit int, chip bool) (Pair, bool) {
	if chip {
		k := p.chip - 1
		if k < limit {
			return NewPair(0, 0), false
		}
		return NewPair(p.gen, k), true
	} else {
		k := p.gen - 1
		if k < limit {
			return NewPair(0, 0), false
		}
		return NewPair(k, p.chip), true
	}
}

func (s *State) up(limit, pairId int, chip bool) (State, bool) {
	newState := *s
	pair, success := s.pairs[pairId].up(limit, chip)
	if success {
		newState.pairs[pairId] = pair
		return newState, true
	} else {
		return State{}, false
	}
}

func (s *State) down(limit, pairId int, chip bool) (State, bool) {
	newState := *s
	pair, success := s.pairs[pairId].down(limit, chip)
	if success {
		newState.pairs[pairId] = pair
		return newState, true
	} else {
		return State{}, false
	}
}

func main() {
	start := time.Now()

	init_state := State{
		cost:      0,
		heuristic: 0,
		elevator:  0,
		pairs: []Pair{
			NewPair(0, 0),
			NewPair(0, 0),
			NewPair(1, 1),
			NewPair(1, 1),
			NewPair(1, 2),
		},
	}

	target_state := State{
		cost:      0,
		heuristic: 0,
		elevator:  3,
		pairs: []Pair{
			NewPair(3, 3),
			NewPair(3, 3),
			NewPair(3, 3),
			NewPair(3, 3),
			NewPair(3, 3),
		},
	}

	fmt.Println(fmt.Sprintf("time elapsed: %s", time.Since(start)))
}
<|endoftext|>


<commit_before>package sde

import (
	"fmt"
	"github.com/THUNDERGROOVE/SDETool2/log"
	"reflect"
	"strings"
	"time"
)

var WorthyAttributes map[string]AtterSet

type AtterSet struct {
	SetName       string
	AttributeName string
	DoRangeFilter bool
	ValueFunc     func(t SDEType, val interface{}) interface{}
}

func init() {
	defer Debug(time.Now())
	WorthyAttributes = make(map[string]AtterSet, 0)

	// Biotic stuff
	WorthyAttributes["mCharProp.meleeDamage"] = AtterSet{SetName: "Biotics", AttributeName: "melee damage"}
	WorthyAttributes["mCharProp.maxStamina"] = AtterSet{SetName: "Biotics", AttributeName: "stamina"}
	WorthyAttributes["mCharProp.staminaRecoveryPerSecond"] = AtterSet{SetName: "Biotics", AttributeName: "stamina recovery"}
	WorthyAttributes["mVICProp.groundSpeed"] = AtterSet{SetName: "Biotics", AttributeName: "speed", DoRangeFilter: true}
	WorthyAttributes["mCharProp.movementSprint.groundSpeedScale"] = AtterSet{SetName: "Biotics", AttributeName: "sprint speed", DoRangeFilter: true,
		ValueFunc: func(t SDEType, val interface{}) interface{} {
			if v, ok := t.Attributes["mVICProp.groundSpeed"]; ok {
				if speed, kk := val.(float64); kk {
					if scale, kkk := v.(float64); kkk {
						log.Info("Speed:", speed, "scale:", scale)
						return interface{}(float64(speed * scale))
					}
				}
			} else {
				log.LogError("Type assertion error. speed:", reflect.TypeOf(v), "val:", reflect.TypeOf(val))
			}

			return interface{}(float64(-1))
		}}
	WorthyAttributes["mCharProp.movementRun.strafeSpeedScale"] = AtterSet{SetName: "Biotics", AttributeName: "strafe speed", DoRangeFilter: true,
		ValueFunc: func(t SDEType, val interface{}) interface{} {
			if v, ok := t.Attributes["mVICProp.groundSpeed"]; ok {
				if speed, kk := val.(float64); kk {
					if scale, kkk := v.(float64); kkk {
						log.Info("Speed:", speed, "scale:", scale)
						return interface{}(float64(speed * scale))
					}
				}
			} else {
				log.LogError("Type assertion error.  speed:", reflect.TypeOf(v), "val:", reflect.TypeOf(val))
			}
			return interface{}(float64(-1))
		}}

	// Regen
	WorthyAttributes["mVICProp.healArmorRate"] = AtterSet{SetName: "Regeneration", AttributeName: "armor repair rate"}
	WorthyAttributes["mVICProp.healShieldRate"] = AtterSet{SetName: "Regeneration", AttributeName: "shield recharge rate"}
	WorthyAttributes["mVICProp.shieldRechargeDelay"] = AtterSet{SetName: "Regeneration", AttributeName: "shield recharge delay"}
	WorthyAttributes["mVICProp.shieldRechargePauseOnShieldDepleted"] = AtterSet{SetName: "Regeneration", AttributeName: "shield depleted delay"}

	//HP
	WorthyAttributes["mVICProp.maxArmor"] = AtterSet{SetName: "HP", AttributeName: "armor"}
	WorthyAttributes["mVICProp.maxShield"] = AtterSet{SetName: "HP", AttributeName: "shield"}

	//Fitting
	WorthyAttributes["mVICProp.maxPowerReserve"] = AtterSet{SetName: "Fitting", AttributeName: "PG"}
	WorthyAttributes["mVICProp.maxCpuReserve"] = AtterSet{SetName: "Fitting", AttributeName: "CPU"}
	WorthyAttributes["mVICProp.amountCpuUsage"] = AtterSet{SetName: "Fitting", AttributeName: "CPU usage"}
	WorthyAttributes["mVICProp.amountPowerUsage"] = AtterSet{SetName: "Fitting", AttributeName: "PG usage"}

	//EWAR
	WorthyAttributes["mVICProp.signatureScanPrecision"] = AtterSet{SetName: "EWAR", AttributeName: "scan precision"}
	WorthyAttributes["mVICProp.signatureScanProfile"] = AtterSet{SetName: "EWAR", AttributeName: "scan profile"}
	WorthyAttributes["mVICProp.signatureScanRadius"] = AtterSet{SetName: "EWAR", AttributeName: "scan radius", DoRangeFilter: true}

	//Misc
	WorthyAttributes["metaLevel"] = AtterSet{SetName: "Misc", AttributeName: "meta level"}
}

func PrintWorthyStats(t SDEType) {
	defer Debug(time.Now())
	p := make(map[string][]string)
	// Iterate attributes for matches
	for k, v := range WorthyAttributes {
		if val, ok := t.Attributes[k]; ok {
			if _, kk := p[v.SetName]; !kk {
				p[v.SetName] = make([]string, 0)
			}
			if v.DoRangeFilter && v.ValueFunc == nil {
				log.Info("value", v.AttributeName, "has range filter but no value func")
				p[v.SetName] = append(p[v.SetName], fmt.Sprintf("%v: %v", v.AttributeName, DoRangeFilter(val)))
			} else if v.ValueFunc != nil && v.DoRangeFilter == false {
				log.Info("value", v.AttributeName, "has value func but no range filter")
				p[v.SetName] = append(p[v.SetName], fmt.Sprintf("%v: %v", v.AttributeName, v.ValueFunc(t, val)))
			} else if v.DoRangeFilter && v.ValueFunc != nil {
				log.Info("value", v.AttributeName, "has range filter and a value func")
				p[v.SetName] = append(p[v.SetName], fmt.Sprintf("%v: %v", v.AttributeName, DoRangeFilter(v.ValueFunc(t, val))))
			} else {
				p[v.SetName] = append(p[v.SetName], fmt.Sprintf("%v: %v", v.AttributeName, val))
			}
		}
	}
	// Check modifiers.
	for k, v := range t.Attributes {
		if strings.Contains(k, ".attributeName") {
			index := strings.Split(strings.Split(k, "modifier.")[1], ".")[0]
			for kk, vv := range WorthyAttributes {
				if vstr, ok := v.(string); ok {
					log.Info("Attribute", k, "is of index", index, "?")
					if kk == vstr {
						log.Info("Holy tits found a match")
						val := t.Attributes[fmt.Sprintf("modifier.%v.modifierValue", index)]
						mod := t.Attributes[fmt.Sprintf("modifier.%v.modifierType", index)]
						if vv.DoRangeFilter {
							p[vv.SetName] = append(p[vv.SetName], fmt.Sprintf("modifies: %v by %v using %v", vv.AttributeName, DoRangeFilter(val), mod))
						} else {
							p[vv.SetName] = append(p[vv.SetName], fmt.Sprintf("modifies: %v by %v using %v", vv.AttributeName, val, mod))
						}
					}
				} else {
					log.LogError("Attribute name wasn't a stirng? o:")
				}
			}
		}
	}

	for k, v := range p {
		fmt.Printf("=== %v ===\n", k)
		for _, vv := range v {
			fmt.Println("  ", vv)
		}
	}
}

func DoRangeFilter(i interface{}) float64 {
	if v, ok := i.(float64); ok {
		return float64(v / 100)
	}

	log.Info("Do range filter had no int in interface :/ got", reflect.TypeOf(i))

	return float64(0)
}
<commit_msg>Alphabetical -s group printing<commit_after>package sde

import (
	"fmt"
	"github.com/THUNDERGROOVE/SDETool2/log"
	"reflect"
	"sort"
	"strings"
	"time"
)

var WorthyAttributes map[string]AtterSet

type AtterSet struct {
	SetName       string
	AttributeName string
	DoRangeFilter bool
	ValueFunc     func(t SDEType, val interface{}) interface{}
}

func init() {
	defer Debug(time.Now())
	WorthyAttributes = make(map[string]AtterSet, 0)

	// Biotic stuff
	WorthyAttributes["mCharProp.meleeDamage"] = AtterSet{SetName: "Biotics", AttributeName: "melee damage"}
	WorthyAttributes["mCharProp.maxStamina"] = AtterSet{SetName: "Biotics", AttributeName: "stamina"}
	WorthyAttributes["mCharProp.staminaRecoveryPerSecond"] = AtterSet{SetName: "Biotics", AttributeName: "stamina recovery"}
	WorthyAttributes["mVICProp.groundSpeed"] = AtterSet{SetName: "Biotics", AttributeName: "speed", DoRangeFilter: true}
	WorthyAttributes["mCharProp.movementSprint.groundSpeedScale"] = AtterSet{SetName: "Biotics", AttributeName: "sprint speed", DoRangeFilter: true,
		ValueFunc: func(t SDEType, val interface{}) interface{} {
			if v, ok := t.Attributes["mVICProp.groundSpeed"]; ok {
				if speed, kk := val.(float64); kk {
					if scale, kkk := v.(float64); kkk {
						log.Info("Speed:", speed, "scale:", scale)
						return interface{}(float64(speed * scale))
					}
				}
			} else {
				log.LogError("Type assertion error. speed:", reflect.TypeOf(v), "val:", reflect.TypeOf(val))
			}

			return interface{}(float64(-1))
		}}
	WorthyAttributes["mCharProp.movementRun.strafeSpeedScale"] = AtterSet{SetName: "Biotics", AttributeName: "strafe speed", DoRangeFilter: true,
		ValueFunc: func(t SDEType, val interface{}) interface{} {
			if v, ok := t.Attributes["mVICProp.groundSpeed"]; ok {
				if speed, kk := val.(float64); kk {
					if scale, kkk := v.(float64); kkk {
						log.Info("Speed:", speed, "scale:", scale)
						return interface{}(float64(speed * scale))
					}
				}
			} else {
				log.LogError("Type assertion error.  speed:", reflect.TypeOf(v), "val:", reflect.TypeOf(val))
			}
			return interface{}(float64(-1))
		}}

	// Regen
	WorthyAttributes["mVICProp.healArmorRate"] = AtterSet{SetName: "Regeneration", AttributeName: "armor repair rate"}
	WorthyAttributes["mVICProp.healShieldRate"] = AtterSet{SetName: "Regeneration", AttributeName: "shield recharge rate"}
	WorthyAttributes["mVICProp.shieldRechargeDelay"] = AtterSet{SetName: "Regeneration", AttributeName: "shield recharge delay"}
	WorthyAttributes["mVICProp.shieldRechargePauseOnShieldDepleted"] = AtterSet{SetName: "Regeneration", AttributeName: "shield depleted delay"}

	//HP
	WorthyAttributes["mVICProp.maxArmor"] = AtterSet{SetName: "HP", AttributeName: "armor"}
	WorthyAttributes["mVICProp.maxShield"] = AtterSet{SetName: "HP", AttributeName: "shield"}

	//Fitting
	WorthyAttributes["mVICProp.maxPowerReserve"] = AtterSet{SetName: "Fitting", AttributeName: "PG"}
	WorthyAttributes["mVICProp.maxCpuReserve"] = AtterSet{SetName: "Fitting", AttributeName: "CPU"}
	WorthyAttributes["mVICProp.amountCpuUsage"] = AtterSet{SetName: "Fitting", AttributeName: "CPU usage"}
	WorthyAttributes["mVICProp.amountPowerUsage"] = AtterSet{SetName: "Fitting", AttributeName: "PG usage"}

	//EWAR
	WorthyAttributes["mVICProp.signatureScanPrecision"] = AtterSet{SetName: "EWAR", AttributeName: "scan precision"}
	WorthyAttributes["mVICProp.signatureScanProfile"] = AtterSet{SetName: "EWAR", AttributeName: "scan profile"}
	WorthyAttributes["mVICProp.signatureScanRadius"] = AtterSet{SetName: "EWAR", AttributeName: "scan radius", DoRangeFilter: true}

	//Misc
	WorthyAttributes["metaLevel"] = AtterSet{SetName: "Misc", AttributeName: "meta level"}
}

func PrintWorthyStats(t SDEType) {
	defer Debug(time.Now())
	p := make(map[string][]string)
	// Iterate attributes for matches
	for k, v := range WorthyAttributes {
		if val, ok := t.Attributes[k]; ok {
			if _, kk := p[v.SetName]; !kk {
				p[v.SetName] = make([]string, 0)
			}
			if v.DoRangeFilter && v.ValueFunc == nil {
				log.Info("value", v.AttributeName, "has range filter but no value func")
				p[v.SetName] = append(p[v.SetName], fmt.Sprintf("%v: %v", v.AttributeName, DoRangeFilter(val)))
			} else if v.ValueFunc != nil && v.DoRangeFilter == false {
				log.Info("value", v.AttributeName, "has value func but no range filter")
				p[v.SetName] = append(p[v.SetName], fmt.Sprintf("%v: %v", v.AttributeName, v.ValueFunc(t, val)))
			} else if v.DoRangeFilter && v.ValueFunc != nil {
				log.Info("value", v.AttributeName, "has range filter and a value func")
				p[v.SetName] = append(p[v.SetName], fmt.Sprintf("%v: %v", v.AttributeName, DoRangeFilter(v.ValueFunc(t, val))))
			} else {
				p[v.SetName] = append(p[v.SetName], fmt.Sprintf("%v: %v", v.AttributeName, val))
			}
		}
	}
	// Check modifiers.
	for k, v := range t.Attributes {
		if strings.Contains(k, ".attributeName") {
			index := strings.Split(strings.Split(k, "modifier.")[1], ".")[0]
			for kk, vv := range WorthyAttributes {
				if vstr, ok := v.(string); ok {
					log.Info("Attribute", k, "is of index", index, "?")
					if kk == vstr {
						log.Info("Holy tits found a match")
						val := t.Attributes[fmt.Sprintf("modifier.%v.modifierValue", index)]
						mod := t.Attributes[fmt.Sprintf("modifier.%v.modifierType", index)]
						if vv.DoRangeFilter {
							p[vv.SetName] = append(p[vv.SetName], fmt.Sprintf("modifies: %v by %v using %v", vv.AttributeName, DoRangeFilter(val), mod))
						} else {
							p[vv.SetName] = append(p[vv.SetName], fmt.Sprintf("modifies: %v by %v using %v", vv.AttributeName, val, mod))
						}
					}
				} else {
					log.LogError("Attribute name wasn't a stirng? o:")
				}
			}
		}
	}

	keys := make([]string, 0)

	for k, _ := range p {
		keys = append(keys, k)
	}

	sort.Strings(keys)

	for _, key := range keys {
		fmt.Printf("=== %v ===\n", key)
		for _, vv := range p[key] {
			fmt.Printf("  %v\n", vv)
		}
	}
}

func DoRangeFilter(i interface{}) float64 {
	if v, ok := i.(float64); ok {
		return float64(v / 100)
	}

	log.Info("Do range filter had no int in interface :/ got", reflect.TypeOf(i))

	return float64(0)
}
<|endoftext|>


<commit_before>package collector

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	simpleJson "github.com/bitly/go-simplejson"
	"github.com/matsumana/flink_exporter/util"
	"strconv"
	"strings"
)

type ReadWriteMertics struct {
	JobName      string
	ReadBytes    int64
	WriteBytes   int64
	ReadRecords  int64
	WriteRecords int64
}

type ReadWriteTotalMertics struct {
	ReadBytesTotal    int64
	WriteBytesTotal   int64
	ReadRecordsTotal  int64
	WriteRecordsTotal int64
	Details           []ReadWriteMertics
}

type CheckpointMetrics struct {
	JobName  string
	Count    int64
	Duration int
	Size     int64
}

type ExceptionMetrics struct {
	JobName string
	Count   int
}

// see https://github.com/apache/flink/blob/release-1.0.3/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobStatus.java
// TODO Must modify, After Flink version up.
type JobStatusMetrics struct {
	JobName    string
	Created    int
	Running    int
	Failing    int
	Failed     int
	Cancelling int
	Canceled   int
	Finished   int
	Restarting int
}

type jobDetail struct {
	id          string
	name        string
	detail      *simpleJson.Json
	checkPoints *simpleJson.Json
	exceptions  *simpleJson.Json
}

type Job struct{}

func (j *Job) GetMetrics(flinkJobManagerUrl string) ([]JobStatusMetrics, ReadWriteTotalMertics, []CheckpointMetrics, []ExceptionMetrics) {
	jobs := j.getJobs(flinkJobManagerUrl)
	jobDetails := j.getJobDetails(flinkJobManagerUrl, jobs)
	jobStatuses := j.getJobStatus(jobDetails)
	readWrites := j.getReadWrite(jobDetails)
	checkpoints := j.getCheckpoints(jobDetails)
	exceptions := j.getExceptions(jobDetails)
	return jobStatuses, readWrites, checkpoints, exceptions
}

func (j *Job) getJobs(flinkJobManagerUrl string) []string {
	url := strings.Trim(flinkJobManagerUrl, "/") + "/jobs"
	httpClient := util.HttpClient{}
	jsonStr, err := httpClient.Get(url)
	if err != nil {
		log.Errorf("HttpClient.Get = %v", err)
		return []string{}
	}

	// parse
	js, err := simpleJson.NewJson([]byte(jsonStr))
	if err != nil {
		log.Errorf("simpleJson.NewJson = %v", err)
		return []string{}
	}

	// jobs
	var jobs []string
	jobs, err = js.Get("jobs-running").StringArray()
	if err != nil {
		log.Errorf("js.Get 'jobs-running' = %v", err)
		return []string{}
	}
	log.Debugf("jobs = %v", jobs)

	return jobs
}

func (j *Job) getJobDetails(flinkJobManagerUrl string, jobs []string) map[string]jobDetail {
	httpClient := util.HttpClient{}
	details := map[string]jobDetail{}
	details = make(map[string]jobDetail)

	// collect all metrics
	for _, job := range jobs {
		// --- detail ---------------------
		url := strings.Trim(flinkJobManagerUrl, "/") + "/jobs/" + job
		jsonStr, err := httpClient.Get(url)
		if err != nil {
			log.Errorf("HttpClient.Get = %v", err)
			continue
		}

		// parse
		js, err := simpleJson.NewJson([]byte(jsonStr))
		if err != nil {
			log.Errorf("simpleJson.NewJson = %v", err)
			continue
		}

		// job name
		var jobName string
		jobName, err = js.Get("name").String()
		if err != nil {
			log.Errorf("js.Get 'name' = %v", err)
			continue
		}
		log.Debugf("jobName = %v", jobName)

		detail := jobDetail{}
		detail.id = job
		detail.name = jobName
		detail.detail = js

		// --- checkpoints ---------------------
		url = strings.Trim(flinkJobManagerUrl, "/") + "/jobs/" + job + "/checkpoints"
		jsonStr, err = httpClient.Get(url)
		if err != nil {
			log.Errorf("HttpClient.Get = %v", err)
			continue
		}

		// parse when exists checkpoints
		if jsonStr != "{}" {
			js, err = simpleJson.NewJson([]byte(jsonStr))
			if err != nil {
				log.Errorf("simpleJson.NewJson = %v", err)
				continue
			}
			detail.checkPoints = js
		}

		// --- exceptions ---------------------
		url = strings.Trim(flinkJobManagerUrl, "/") + "/jobs/" + job + "/exceptions"
		jsonStr, err = httpClient.Get(url)
		if err != nil {
			log.Errorf("HttpClient.Get = %v", err)
			continue
		}

		// parse
		js, err = simpleJson.NewJson([]byte(jsonStr))
		if err != nil {
			log.Errorf("simpleJson.NewJson = %v", err)
			continue
		}
		detail.exceptions = js

		details[detail.id] = detail
	}
	log.Debugf("jobDetails = %v", details)

	return details
}

func (j *Job) getJobStatus(jobDetails map[string]jobDetail) []JobStatusMetrics {
	jobStatuses := []JobStatusMetrics{}
	for _, jobDetail := range jobDetails {
		// state
		var state string
		state, err := jobDetail.detail.Get("state").String()
		if err != nil {
			log.Errorf("js.Get 'state' = %v", err)
			return []JobStatusMetrics{}
		}
		log.Debugf("state = %v", state)

		jobStatus := JobStatusMetrics{}
		jobStatus.JobName = jobDetail.name

		switch state {
		case "CREATED":
			jobStatus.Created += 1
		case "RUNNING":
			jobStatus.Running += 1
		case "FAILING":
			jobStatus.Failing += 1
		case "FAILED":
			jobStatus.Failed += 1
		case "CANCELLING":
			jobStatus.Cancelling += 1
		case "CANCELED":
			jobStatus.Canceled += 1
		case "FINISHED":
			jobStatus.Finished += 1
		case "RESTARTING":
			jobStatus.Restarting += 1
		}

		jobStatuses = append(jobStatuses, jobStatus)
	}

	log.Debugf("jobStatuses = %v", jobStatuses)

	return jobStatuses
}

func (j *Job) getReadWrite(jobDetails map[string]jobDetail) ReadWriteTotalMertics {
	total := ReadWriteTotalMertics{}
	readWrites := []ReadWriteMertics{}
	for _, jobDetail := range jobDetails {
		// vertices
		var vertices []interface{}
		vertices, err := jobDetail.detail.Get("vertices").Array()
		if err != nil {
			log.Errorf("js.Get 'vertices' = %v", err)
			return ReadWriteTotalMertics{}
		}
		log.Debugf("vertices = %v", vertices)

		readWrite := ReadWriteMertics{}
		readWrite.JobName = jobDetail.name

		for _, vertice := range vertices {
			v, _ := vertice.(map[string]interface{})
			log.Debugf("metrics = %v", v["metrics"])

			metrics, _ := v["metrics"].(map[string]interface{})
			record := ReadWriteMertics{}
			if strings.HasPrefix(fmt.Sprint(v["name"]), "Source") {
				record.WriteBytes, _ = strconv.ParseInt(fmt.Sprint(metrics["write-bytes"]), 10, 64)
				record.WriteRecords, _ = strconv.ParseInt(fmt.Sprint(metrics["write-records"]), 10, 64)
				readWrite.WriteBytes += record.WriteBytes
				readWrite.WriteRecords += record.WriteRecords
			} else {
				record.ReadBytes, _ = strconv.ParseInt(fmt.Sprint(metrics["read-bytes"]), 10, 64)
				record.ReadRecords, _ = strconv.ParseInt(fmt.Sprint(metrics["read-records"]), 10, 64)
				readWrite.ReadBytes += record.ReadBytes
				readWrite.ReadRecords += record.ReadRecords
			}
		}

		total.ReadBytesTotal += readWrite.ReadBytes
		total.ReadRecordsTotal += readWrite.ReadRecords
		total.WriteBytesTotal += readWrite.WriteBytes
		total.WriteRecordsTotal += readWrite.WriteRecords

		readWrites = append(readWrites, readWrite)
	}

	log.Debugf("readWrites = %v", readWrites)

	total.Details = readWrites

	return total
}

func (j *Job) getCheckpoints(jobDetails map[string]jobDetail) []CheckpointMetrics {
	checkpoints := []CheckpointMetrics{}
	for _, jobDetail := range jobDetails {
		checkpoint := CheckpointMetrics{}
		checkpoint.JobName = jobDetail.name
		if jobDetail.checkPoints != nil {
			// count
			var count int64
			count, err := jobDetail.checkPoints.Get("count").Int64()
			if err != nil {
				log.Errorf("js.Get 'count' = %v", err)
				return []CheckpointMetrics{}
			}
			log.Debugf("count = %v", count)

			checkpoint.Count = count

			// history
			var histories []interface{}
			histories, err = jobDetail.checkPoints.Get("history").Array()
			if err != nil {
				log.Errorf("js.Get 'history' = %v", err)
				return []CheckpointMetrics{}
			}
			log.Debugf("history = %v", histories)

			if len(histories) > 0 {
				latest, _ := histories[len(histories)-1].(map[string]interface{})
				checkpoint.Duration, _ = strconv.Atoi(fmt.Sprint(latest["duration"]))
				checkpoint.Size, _ = strconv.ParseInt(fmt.Sprint(latest["size"]), 10, 64)
			}
		}

		checkpoints = append(checkpoints, checkpoint)
	}

	log.Debugf("checkpoints = %v", checkpoints)

	return checkpoints
}

func (j *Job) getExceptions(jobDetails map[string]jobDetail) []ExceptionMetrics {
	exceptions := []ExceptionMetrics{}
	for _, jobDetail := range jobDetails {
		// exceptions
		var allExceptions []interface{}
		allExceptions, err := jobDetail.exceptions.Get("all-exceptions").Array()
		if err != nil {
			log.Errorf("js.Get 'all-exceptions' = %v", err)
			return []ExceptionMetrics{}
		}
		log.Debugf("allExceptions = %v", allExceptions)

		exceptions = append(exceptions,
			ExceptionMetrics{
				JobName: jobDetail.name,
				Count:   len(allExceptions),
			})
	}

	log.Debugf("exceptions = %v", exceptions)

	return exceptions
}
<commit_msg>refactor error processing<commit_after>package collector

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	simpleJson "github.com/bitly/go-simplejson"
	"github.com/matsumana/flink_exporter/util"
	"strconv"
	"strings"
)

type ReadWriteMertics struct {
	JobName      string
	ReadBytes    int64
	WriteBytes   int64
	ReadRecords  int64
	WriteRecords int64
}

type ReadWriteTotalMertics struct {
	ReadBytesTotal    int64
	WriteBytesTotal   int64
	ReadRecordsTotal  int64
	WriteRecordsTotal int64
	Details           []ReadWriteMertics
}

type CheckpointMetrics struct {
	JobName  string
	Count    int64
	Duration int
	Size     int64
}

type ExceptionMetrics struct {
	JobName string
	Count   int
}

// see https://github.com/apache/flink/blob/release-1.0.3/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobStatus.java
// TODO Must modify, After Flink version up.
type JobStatusMetrics struct {
	JobName    string
	Created    int
	Running    int
	Failing    int
	Failed     int
	Cancelling int
	Canceled   int
	Finished   int
	Restarting int
}

type jobDetail struct {
	id          string
	name        string
	detail      *simpleJson.Json
	checkPoints *simpleJson.Json
	exceptions  *simpleJson.Json
}

type Job struct{}

func (j *Job) GetMetrics(flinkJobManagerUrl string) ([]JobStatusMetrics, ReadWriteTotalMertics, []CheckpointMetrics, []ExceptionMetrics) {
	jobs := j.getJobs(flinkJobManagerUrl)
	jobDetails := j.getJobDetails(flinkJobManagerUrl, jobs)
	jobStatuses := j.getJobStatus(jobDetails)
	readWrites := j.getReadWrite(jobDetails)
	checkpoints := j.getCheckpoints(jobDetails)
	exceptions := j.getExceptions(jobDetails)
	return jobStatuses, readWrites, checkpoints, exceptions
}

func (j *Job) getJobs(flinkJobManagerUrl string) []string {
	url := strings.Trim(flinkJobManagerUrl, "/") + "/jobs"
	httpClient := util.HttpClient{}
	jsonStr, err := httpClient.Get(url)
	if err != nil {
		log.Errorf("HttpClient.Get = %v", err)
		return []string{}
	}

	// parse
	js, err := simpleJson.NewJson([]byte(jsonStr))
	if err != nil {
		log.Errorf("simpleJson.NewJson = %v", err)
		return []string{}
	}

	// jobs
	var jobs []string
	jobs, err = js.Get("jobs-running").StringArray()
	if err != nil {
		log.Errorf("js.Get 'jobs-running' = %v", err)
		return []string{}
	}
	log.Debugf("jobs = %v", jobs)

	return jobs
}

func (j *Job) getJobDetails(flinkJobManagerUrl string, jobs []string) map[string]jobDetail {
	httpClient := util.HttpClient{}
	details := map[string]jobDetail{}
	details = make(map[string]jobDetail)

	// collect all metrics
	for _, job := range jobs {
		// --- detail ---------------------
		url := strings.Trim(flinkJobManagerUrl, "/") + "/jobs/" + job
		jsonStr, err := httpClient.Get(url)
		if err != nil {
			log.Errorf("HttpClient.Get = %v", err)
			continue
		}

		// parse
		js, err := simpleJson.NewJson([]byte(jsonStr))
		if err != nil {
			log.Errorf("simpleJson.NewJson = %v", err)
			continue
		}

		// job name
		var jobName string
		jobName, err = js.Get("name").String()
		if err != nil {
			log.Errorf("js.Get 'name' = %v", err)
			continue
		}
		log.Debugf("jobName = %v", jobName)

		detail := jobDetail{}
		detail.id = job
		detail.name = jobName
		detail.detail = js

		// --- checkpoints ---------------------
		url = strings.Trim(flinkJobManagerUrl, "/") + "/jobs/" + job + "/checkpoints"
		jsonStr, err = httpClient.Get(url)
		if err != nil {
			log.Errorf("HttpClient.Get = %v", err)
			continue
		}

		// parse when exists checkpoints
		if jsonStr != "{}" {
			js, err = simpleJson.NewJson([]byte(jsonStr))
			if err != nil {
				log.Errorf("simpleJson.NewJson = %v", err)
				continue
			}
			detail.checkPoints = js
		}

		// --- exceptions ---------------------
		url = strings.Trim(flinkJobManagerUrl, "/") + "/jobs/" + job + "/exceptions"
		jsonStr, err = httpClient.Get(url)
		if err != nil {
			log.Errorf("HttpClient.Get = %v", err)
			continue
		}

		// parse
		js, err = simpleJson.NewJson([]byte(jsonStr))
		if err != nil {
			log.Errorf("simpleJson.NewJson = %v", err)
			continue
		}
		detail.exceptions = js

		details[detail.id] = detail
	}
	log.Debugf("jobDetails = %v", details)

	return details
}

func (j *Job) getJobStatus(jobDetails map[string]jobDetail) []JobStatusMetrics {
	jobStatuses := []JobStatusMetrics{}
	for _, jobDetail := range jobDetails {
		// state
		var state string
		state, err := jobDetail.detail.Get("state").String()
		if err != nil {
			log.Errorf("js.Get 'state' = %v", err)
			return []JobStatusMetrics{}
		}
		log.Debugf("state = %v", state)

		jobStatus := JobStatusMetrics{}
		jobStatus.JobName = jobDetail.name

		switch state {
		case "CREATED":
			jobStatus.Created += 1
		case "RUNNING":
			jobStatus.Running += 1
		case "FAILING":
			jobStatus.Failing += 1
		case "FAILED":
			jobStatus.Failed += 1
		case "CANCELLING":
			jobStatus.Cancelling += 1
		case "CANCELED":
			jobStatus.Canceled += 1
		case "FINISHED":
			jobStatus.Finished += 1
		case "RESTARTING":
			jobStatus.Restarting += 1
		}

		jobStatuses = append(jobStatuses, jobStatus)
	}

	log.Debugf("jobStatuses = %v", jobStatuses)

	return jobStatuses
}

func (j *Job) getReadWrite(jobDetails map[string]jobDetail) ReadWriteTotalMertics {
	total := ReadWriteTotalMertics{}
	readWrites := []ReadWriteMertics{}
	for _, jobDetail := range jobDetails {
		// vertices
		var vertices []interface{}
		vertices, err := jobDetail.detail.Get("vertices").Array()
		if err != nil {
			log.Errorf("js.Get 'vertices' = %v", err)
			return ReadWriteTotalMertics{}
		}
		log.Debugf("vertices = %v", vertices)

		readWrite := ReadWriteMertics{}
		readWrite.JobName = jobDetail.name

		for _, verticeTmp := range vertices {
			if vertice, okVertice := verticeTmp.(map[string]interface{}); okVertice {
				if metricsTmp, foundMetrics := vertice["metrics"]; foundMetrics {
					if metrics, okMetrics := metricsTmp.(map[string]interface{}); okMetrics {
						record := ReadWriteMertics{}
						if name, foundName := vertice["name"]; foundName {
							if strings.HasPrefix(fmt.Sprint(name), "Source") {
								record.WriteBytes = j.getValueAsInt64(metrics, "write-bytes")
								record.WriteRecords = j.getValueAsInt64(metrics, "write-records")
								readWrite.WriteBytes += record.WriteBytes
								readWrite.WriteRecords += record.WriteRecords
							} else {
								record.ReadBytes = j.getValueAsInt64(metrics, "read-bytes")
								record.ReadRecords = j.getValueAsInt64(metrics, "read-records")
								readWrite.ReadBytes += record.ReadBytes
								readWrite.ReadRecords += record.ReadRecords
							}
						}
					}
				}
			}
		}

		total.ReadBytesTotal += readWrite.ReadBytes
		total.ReadRecordsTotal += readWrite.ReadRecords
		total.WriteBytesTotal += readWrite.WriteBytes
		total.WriteRecordsTotal += readWrite.WriteRecords

		readWrites = append(readWrites, readWrite)
	}

	log.Debugf("readWrites = %v", readWrites)

	total.Details = readWrites

	return total
}

func (j *Job) getValueAsInt64(metrics map[string]interface{}, key string) int64 {
	if value, found := metrics[key]; found {
		converted, err := strconv.ParseInt(fmt.Sprint(value), 10, 64)
		if err != nil {
			return 0
		}
		return converted
	} else {
		return 0
	}
}

func (j *Job) getCheckpoints(jobDetails map[string]jobDetail) []CheckpointMetrics {
	checkpoints := []CheckpointMetrics{}
	for _, jobDetail := range jobDetails {
		checkpoint := CheckpointMetrics{}
		checkpoint.JobName = jobDetail.name
		if jobDetail.checkPoints != nil {
			// count
			var count int64
			count, err := jobDetail.checkPoints.Get("count").Int64()
			if err != nil {
				log.Errorf("js.Get 'count' = %v", err)
				return []CheckpointMetrics{}
			}
			log.Debugf("count = %v", count)

			checkpoint.Count = count

			// history
			var histories []interface{}
			histories, err = jobDetail.checkPoints.Get("history").Array()
			if err != nil {
				log.Errorf("js.Get 'history' = %v", err)
				return []CheckpointMetrics{}
			}
			log.Debugf("history = %v", histories)

			if len(histories) > 0 {
				if latest, ok := histories[len(histories)-1].(map[string]interface{}); ok {
					checkpoint.Duration = int(j.getValueAsInt64(latest, "duration"))
					checkpoint.Size = j.getValueAsInt64(latest, "size")
				} else {
					checkpoint.Duration = 0
					checkpoint.Size = 0
				}
			}
		}

		checkpoints = append(checkpoints, checkpoint)
	}

	log.Debugf("checkpoints = %v", checkpoints)

	return checkpoints
}

func (j *Job) getExceptions(jobDetails map[string]jobDetail) []ExceptionMetrics {
	exceptions := []ExceptionMetrics{}
	for _, jobDetail := range jobDetails {
		// exceptions
		var allExceptions []interface{}
		allExceptions, err := jobDetail.exceptions.Get("all-exceptions").Array()
		if err != nil {
			log.Errorf("js.Get 'all-exceptions' = %v", err)
			return []ExceptionMetrics{}
		}
		log.Debugf("allExceptions = %v", allExceptions)

		exceptions = append(exceptions,
			ExceptionMetrics{
				JobName: jobDetail.name,
				Count:   len(allExceptions),
			})
	}

	log.Debugf("exceptions = %v", exceptions)

	return exceptions
}
<|endoftext|>


<commit_before>package send

import (
	"encoding/json"
	"fmt"
	"path/filepath"
	"runtime"

	"github.com/mongodb/grip/message"
)

const (
	defaultFormatTmpl  = "[p=%s]: %s"
	callSiteTmpl       = "[p=%s] [%s:%d]: %s"
	completeFormatTmpl = "[%s] (p=%s) %s"
)

// MessageFormatter is a function type used by senders to construct the
// entire string returned as part of the output. This makes it
// possible to modify the logging format without needing to implement
// new Sender interfaces.
type MessageFormatter func(message.Composer) (string, error)

// MakeJSONFormtter returns a MessageFormatter, that returns messages
// as the string form of a JSON document built using the Raw method of
// the Composer. Returns an error if there was a problem marshalling JSON.
func MakeJSONFormatter() MessageFormatter {
	return func(m message.Composer) (string, error) {
		out, err := json.Marshal(m.Raw())
		if err != nil {
			return "", err
		}

		return string(out), nil
	}
}

// MakeDefaultFormatter returns a MessageFormatter that will produce a
// message in the following format:
//
//     [p=<level>]: <message>
//
// It can never error.
func MakeDefaultFormatter() MessageFormatter {
	return func(m message.Composer) (string, error) {
		return fmt.Sprintf(defaultFormatTmpl, m.Priority(), m.String()), nil
	}
}

// MakePlainFormatter returns a MessageFormatter that simply returns the
// string format of the log message.
func MakePlainFormatter() MessageFormatter {
	return func(m message.Composer) (string, error) {
		return m.String(), nil
	}
}

// MakeCallSiteFormatter returns a MessageFormater that formats
// messages with the following format:
//
//     [p=<levvel>] [<fileName>:<lineNumber>]: <message>
//
// It can never error.
func MakeCallSiteFormatter(depth int) MessageFormatter {
	depth++
	return func(m message.Composer) (string, error) {
		file, line := callerInfo(depth)
		return fmt.Sprintf(callSiteTmpl, m.Priority(), file, line, m), nil
	}
}

//MakeXMPPFormatter returns a MessageFormatter that will produce
// messages in the following format, used primarily by the xmpp logger:
//
//     [<name>] (p=<priority>) <message>
//
// It can never error.
func MakeXMPPFormatter(name string) MessageFormatter {
	return func(m message.Composer) (string, error) {
		return fmt.Sprintf(completeFormatTmpl, name, m.Priority(), m.String()), nil
	}
}

func callerInfo(depth int) (string, int) {
	// increase depth to account for callerInfo itself.
	depth++

	// get caller info.
	_, file, line, _ := runtime.Caller(depth)

	// get the directory and filename
	dir, fileName := filepath.Split(file)
	file = filepath.Join(filepath.Base(dir), fileName)

	return file, line
}
<commit_msg>MAKE-85: fix typo<commit_after>package send

import (
	"encoding/json"
	"fmt"
	"path/filepath"
	"runtime"

	"github.com/mongodb/grip/message"
)

const (
	defaultFormatTmpl  = "[p=%s]: %s"
	callSiteTmpl       = "[p=%s] [%s:%d]: %s"
	completeFormatTmpl = "[%s] (p=%s) %s"
)

// MessageFormatter is a function type used by senders to construct the
// entire string returned as part of the output. This makes it
// possible to modify the logging format without needing to implement
// new Sender interfaces.
type MessageFormatter func(message.Composer) (string, error)

// MakeJSONFormatter returns a MessageFormatter, that returns messages
// as the string form of a JSON document built using the Raw method of
// the Composer. Returns an error if there was a problem marshalling JSON.
func MakeJSONFormatter() MessageFormatter {
	return func(m message.Composer) (string, error) {
		out, err := json.Marshal(m.Raw())
		if err != nil {
			return "", err
		}

		return string(out), nil
	}
}

// MakeDefaultFormatter returns a MessageFormatter that will produce a
// message in the following format:
//
//     [p=<level>]: <message>
//
// It can never error.
func MakeDefaultFormatter() MessageFormatter {
	return func(m message.Composer) (string, error) {
		return fmt.Sprintf(defaultFormatTmpl, m.Priority(), m.String()), nil
	}
}

// MakePlainFormatter returns a MessageFormatter that simply returns the
// string format of the log message.
func MakePlainFormatter() MessageFormatter {
	return func(m message.Composer) (string, error) {
		return m.String(), nil
	}
}

// MakeCallSiteFormatter returns a MessageFormater that formats
// messages with the following format:
//
//     [p=<levvel>] [<fileName>:<lineNumber>]: <message>
//
// It can never error.
func MakeCallSiteFormatter(depth int) MessageFormatter {
	depth++
	return func(m message.Composer) (string, error) {
		file, line := callerInfo(depth)
		return fmt.Sprintf(callSiteTmpl, m.Priority(), file, line, m), nil
	}
}

//MakeXMPPFormatter returns a MessageFormatter that will produce
// messages in the following format, used primarily by the xmpp logger:
//
//     [<name>] (p=<priority>) <message>
//
// It can never error.
func MakeXMPPFormatter(name string) MessageFormatter {
	return func(m message.Composer) (string, error) {
		return fmt.Sprintf(completeFormatTmpl, name, m.Priority(), m.String()), nil
	}
}

func callerInfo(depth int) (string, int) {
	// increase depth to account for callerInfo itself.
	depth++

	// get caller info.
	_, file, line, _ := runtime.Caller(depth)

	// get the directory and filename
	dir, fileName := filepath.Split(file)
	file = filepath.Join(filepath.Base(dir), fileName)

	return file, line
}
<|endoftext|>


<commit_before>package db

import (
    "database/sql"
    "fmt"
    _ "github.com/lib/pq"
    "github.com/orc/utils"
    "log"
    "reflect"
    "strconv"
    "strings"
    "time"
    "errors"
)

var DB *sql.DB = nil

func Exec(query string, params []interface{}) sql.Result {
    log.Println(query)
    stmt, err := DB.Prepare(query)
    utils.HandleErr("[queries.Exec] Prepare: ", err, nil)
    defer stmt.Close()
    result, err := stmt.Exec(params...)
    utils.HandleErr("[queries.Exec] Exec: ", err, nil)
    return result
}

func Query(query string, params []interface{}) []interface{} {
    log.Println(query)

    stmt, err := DB.Prepare(query)
    utils.HandleErr("[queries.Query] Prepare: ", err, nil)
    defer stmt.Close()
    rows, err := stmt.Query(params...)
    utils.HandleErr("[queries.Query] Query: ", err, nil)
    defer rows.Close()

    rowsInf := Exec(query, params)
    columns, err := rows.Columns()
    utils.HandleErr("[queries.Query] Columns: ", err, nil)
    size, err := rowsInf.RowsAffected()
    utils.HandleErr("[queries.Query] RowsAffected: ", err, nil)

    return ConvertData(columns, size, rows)
}

func QueryRow(query string, params []interface{}) *sql.Row {
    log.Println(query)
    stmt, err := DB.Prepare(query)
    utils.HandleErr("[queries.QueryRow] Prepare: ", err, nil)
    defer stmt.Close()
    result := stmt.QueryRow(params...)
    utils.HandleErr("[queries.QueryRow] Query: ", err, nil)
    return result
}

func QueryCreateSecuence(tableName string) {
    Exec("CREATE SEQUENCE "+tableName+"_id_seq;", nil)
}

func QueryCreateTable(m interface{}) {
    model := reflect.ValueOf(m)
    tableName := model.Elem().FieldByName("TableName").String()

    QueryCreateSecuence(tableName)
    query := "CREATE TABLE IF NOT EXISTS %s ("
    mF := model.Elem().FieldByName("Fields").Elem().Type()
    for i := 0; i < mF.Elem().NumField(); i++ {
        query += mF.Elem().Field(i).Tag.Get("name") + " "
        query += mF.Elem().Field(i).Tag.Get("type") + " "
        query += mF.Elem().Field(i).Tag.Get("null") + " "
        switch mF.Elem().Field(i).Tag.Get("extra") {
        case "PRIMARY":
            query += "PRIMARY KEY DEFAULT NEXTVAL('"
            query += tableName + "_id_seq'), "
            break
        case "REFERENCES":
            query += "REFERENCES " + mF.Elem().Field(i).Tag.Get("refTable") + "(" + mF.Elem().Field(i).Tag.Get("refField") + ") ON DELETE CASCADE, "
            break
        case "UNIQUE":
            query += "UNIQUE, "
            break
        default:
            query += ", "
        }
    }
    query = query[0 : len(query)-2]
    query += ");"
    Exec(fmt.Sprintf(query, tableName), nil)
}

func QueryInsert(m interface{}, extra string) *sql.Row {
    var i int

    query := "INSERT INTO %s ("
    tableName := reflect.ValueOf(m).Elem().FieldByName("TableName").String()

    tFields := reflect.ValueOf(m).Elem().FieldByName("Fields").Elem().Type().Elem()
    vFields := reflect.ValueOf(m).Elem().FieldByName("Fields").Elem().Elem()

    n := tFields.NumField()
    p := make([]interface{}, n-1)

    for i = 1; i < n; i++ {
        query += tFields.Field(i).Tag.Get("name") + ", "
        v, ok := utils.ConvertTypeModel(tFields.Field(i).Tag.Get("type"), vFields.Field(i))
        if !ok && tFields.Field(i).Tag.Get("null") == "NULL" {
            continue
        }
        p[i-1] = v
    }
    query = query[0 : len(query)-2]
    query += ") VALUES (%s) %s;"

    // if i < 2 {
    //     return
    // }

    return QueryRow(fmt.Sprintf(query, tableName, strings.Join(MakeParams(n-1), ", "), extra), p)
}

func QueryUpdate(m interface{}) *sql.Row {
    model := reflect.ValueOf(m).Elem()
    tableName := model.FieldByName("TableName").String()
    i, j := 1, 1

    query := "UPDATE %s SET "

    tFields := model.FieldByName("Fields").Elem().Type().Elem()
    vFields := model.FieldByName("Fields").Elem().Elem()

    p := make([]interface{}, 0)

    for ; j < tFields.NumField(); j++ {
        v, ok := utils.ConvertTypeModel(tFields.Field(j).Tag.Get("type"), vFields.Field(j))
        if ok == false {
            continue
        }
        query += tFields.Field(j).Tag.Get("name") + "=$" + strconv.Itoa(i) + ", "
        p = append(p, v)
        i++
    }
    query = query[0 : len(query)-2]

    if i < 2 {
        return nil
    }

    if model.FieldByName("WherePart").Len() != 0 {
        query += " WHERE %s;"
        v := model.MethodByName("GenerateWherePart").Call([]reflect.Value{reflect.ValueOf(i)})
        return QueryRow(fmt.Sprintf(query, tableName, v[0]), append(p, v[1].Interface().([]interface{})...))
    } else {
        query += ";"
        return QueryRow(fmt.Sprintf(query, tableName), p)
    }
}

func QueryDeleteByIds(tableName, ids string) {
    query := "DELETE FROM %s WHERE id IN (%s)"
    Exec(fmt.Sprintf(query, tableName, ids), nil)
}

func IsExists_(tableName string, fields []string, params []interface{}) bool {
    query := "SELECT %s FROM %s WHERE %s;"
    f := strings.Join(fields, ", ")
    p := strings.Join(MakePairs(fields), " AND ")

    var result string
    row := QueryRow(fmt.Sprintf(query, f, tableName, p), params)
    err := row.Scan(&result)

    return err != sql.ErrNoRows
}

func MakeParams(n int) []string {
    var result = make([]string, n)
    for i := 0; i < n; i++ {
        result[i] = "$" + strconv.Itoa(i+1)
    }
    return result
}

func MakePairs(fields []string) []string {
    var result = make([]string, len(fields))
    for i := 0; i < len(fields); i++ {
        result[i] = fields[i] + "=$" + strconv.Itoa(i+1)
    }
    return result
}

func Select(m interface{}, fields []string) []interface{} {
    model := reflect.ValueOf(m).Elem()
    tableName := model.FieldByName("TableName").String()

    orderBy := " ORDER BY " + model.FieldByName("OrderBy").Interface().(string)

    var limit string
    switch model.FieldByName("Limit").Interface().(type) {
    case string:
        limit = " LIMIT " + model.FieldByName("Limit").Interface().(string)
        break
    case int:
        limit = " LIMIT " + strconv.Itoa(model.FieldByName("Limit").Interface().(int))
        break
    }

    offset := " OFFSET " + strconv.Itoa(model.FieldByName("Offset").Interface().(int))
    extra := orderBy + limit + offset

    query := "SELECT %s FROM %s"

    if model.FieldByName("WherePart").Len() != 0 {
        query += " WHERE %s" + extra + ";"
        v := model.MethodByName("GenerateWherePart").Call([]reflect.Value{reflect.ValueOf(1)})
        return Query(fmt.Sprintf(query, strings.Join(fields, ", "), tableName, v[0]), v[1].Interface().([]interface{}))
    } else {
        query += extra + ";"
        return Query(fmt.Sprintf(query, strings.Join(fields, ", "), tableName), nil)
    }
}

func SelectRow(m interface{}, fields []string) *sql.Row {
    model := reflect.ValueOf(m).Elem()
    tableName := model.FieldByName("TableName").String()

    query := "SELECT %s FROM %s"

    if model.FieldByName("WherePart").Len() != 0 {
        query += " WHERE %s;"
        v := model.MethodByName("GenerateWherePart").Call([]reflect.Value{reflect.ValueOf(1)})
        return QueryRow(fmt.Sprintf(query, strings.Join(fields, ", "), tableName, v[0]), v[1].Interface().([]interface{}))
    } else {
        query += ";"
        return QueryRow(fmt.Sprintf(query, strings.Join(fields, ", "), tableName), nil)
    }
}

func SelectCount(tableName string) int {
    return int(Query("SELECT COUNT(*) FROM "+tableName+";", nil)[0].(map[string]interface{})["count"].(int))
}

func ConvertData(columns []string, size int64, rows *sql.Rows) []interface{} {
    row := make([]interface{}, len(columns))
    values := make([]interface{}, len(columns))
    answer := make([]interface{}, size)

    for i, _ := range row {
        row[i] = &values[i]
    }

    j := 0
    for rows.Next() {
        rows.Scan(row...)
        record := make(map[string]interface{}, len(values))
        for i, col := range values {
            if col != nil {
                //fmt.Printf("\n%s: type= %s\n", columns[i], reflect.TypeOf(col))
                switch col.(type) {
                case bool:
                    record[columns[i]] = col.(bool)
                    break
                case int:
                    record[columns[i]] = col.(int)
                    break
                case int64:
                    record[columns[i]] = int(col.(int64))
                    break
                case float64:
                    record[columns[i]] = col.(float64)
                    break
                case string:
                    record[columns[i]] = col.(string)
                    break
                // case []byte:
                //     record[columns[i]] = string(col.([]byte))
                //     break
                case []int8:
                    record[columns[i]] = col.([]string)
                    break
                case time.Time:
                    record[columns[i]] = col
                    break
                case []uint8:
                    data := strings.Split(strings.Trim(string(col.([]uint8)), "{}"), ",")
                    if len(data) == 1 {
                        record[columns[i]] = data[0]
                    } else {
                        record[columns[i]] = data
                    }
                    break
                default:
                    utils.HandleErr("ConvertData: ", errors.New("Unexpected type."), nil)
                }
            }
            answer[j] = record
        }
        j++
    }
    rows.Close()
    return answer
}
<commit_msg>queries.go: delete unused function - SelectCount<commit_after>package db

import (
    "database/sql"
    "fmt"
    _ "github.com/lib/pq"
    "github.com/orc/utils"
    "log"
    "reflect"
    "strconv"
    "strings"
    "time"
    "errors"
)

var DB *sql.DB = nil

func Exec(query string, params []interface{}) sql.Result {
    log.Println(query)
    stmt, err := DB.Prepare(query)
    utils.HandleErr("[queries.Exec] Prepare: ", err, nil)
    defer stmt.Close()
    result, err := stmt.Exec(params...)
    utils.HandleErr("[queries.Exec] Exec: ", err, nil)
    return result
}

func Query(query string, params []interface{}) []interface{} {
    log.Println(query)

    stmt, err := DB.Prepare(query)
    utils.HandleErr("[queries.Query] Prepare: ", err, nil)
    defer stmt.Close()
    rows, err := stmt.Query(params...)
    utils.HandleErr("[queries.Query] Query: ", err, nil)
    defer rows.Close()

    rowsInf := Exec(query, params)
    columns, err := rows.Columns()
    utils.HandleErr("[queries.Query] Columns: ", err, nil)
    size, err := rowsInf.RowsAffected()
    utils.HandleErr("[queries.Query] RowsAffected: ", err, nil)

    return ConvertData(columns, size, rows)
}

func QueryRow(query string, params []interface{}) *sql.Row {
    log.Println(query)
    stmt, err := DB.Prepare(query)
    utils.HandleErr("[queries.QueryRow] Prepare: ", err, nil)
    defer stmt.Close()
    result := stmt.QueryRow(params...)
    utils.HandleErr("[queries.QueryRow] Query: ", err, nil)
    return result
}

func QueryCreateSecuence(tableName string) {
    Exec("CREATE SEQUENCE "+tableName+"_id_seq;", nil)
}

func QueryCreateTable(m interface{}) {
    model := reflect.ValueOf(m)
    tableName := model.Elem().FieldByName("TableName").String()

    QueryCreateSecuence(tableName)
    query := "CREATE TABLE IF NOT EXISTS %s ("
    mF := model.Elem().FieldByName("Fields").Elem().Type()
    for i := 0; i < mF.Elem().NumField(); i++ {
        query += mF.Elem().Field(i).Tag.Get("name") + " "
        query += mF.Elem().Field(i).Tag.Get("type") + " "
        query += mF.Elem().Field(i).Tag.Get("null") + " "
        switch mF.Elem().Field(i).Tag.Get("extra") {
        case "PRIMARY":
            query += "PRIMARY KEY DEFAULT NEXTVAL('"
            query += tableName + "_id_seq'), "
            break
        case "REFERENCES":
            query += "REFERENCES " + mF.Elem().Field(i).Tag.Get("refTable") + "(" + mF.Elem().Field(i).Tag.Get("refField") + ") ON DELETE CASCADE, "
            break
        case "UNIQUE":
            query += "UNIQUE, "
            break
        default:
            query += ", "
        }
    }
    query = query[0 : len(query)-2]
    query += ");"
    Exec(fmt.Sprintf(query, tableName), nil)
}

func QueryInsert(m interface{}, extra string) *sql.Row {
    var i int

    query := "INSERT INTO %s ("
    tableName := reflect.ValueOf(m).Elem().FieldByName("TableName").String()

    tFields := reflect.ValueOf(m).Elem().FieldByName("Fields").Elem().Type().Elem()
    vFields := reflect.ValueOf(m).Elem().FieldByName("Fields").Elem().Elem()

    n := tFields.NumField()
    p := make([]interface{}, n-1)

    for i = 1; i < n; i++ {
        query += tFields.Field(i).Tag.Get("name") + ", "
        v, ok := utils.ConvertTypeModel(tFields.Field(i).Tag.Get("type"), vFields.Field(i))
        if !ok && tFields.Field(i).Tag.Get("null") == "NULL" {
            continue
        }
        p[i-1] = v
    }
    query = query[0 : len(query)-2]
    query += ") VALUES (%s) %s;"

    // if i < 2 {
    //     return
    // }

    return QueryRow(fmt.Sprintf(query, tableName, strings.Join(MakeParams(n-1), ", "), extra), p)
}

func QueryUpdate(m interface{}) *sql.Row {
    model := reflect.ValueOf(m).Elem()
    tableName := model.FieldByName("TableName").String()
    i, j := 1, 1

    query := "UPDATE %s SET "

    tFields := model.FieldByName("Fields").Elem().Type().Elem()
    vFields := model.FieldByName("Fields").Elem().Elem()

    p := make([]interface{}, 0)

    for ; j < tFields.NumField(); j++ {
        v, ok := utils.ConvertTypeModel(tFields.Field(j).Tag.Get("type"), vFields.Field(j))
        if ok == false {
            continue
        }
        query += tFields.Field(j).Tag.Get("name") + "=$" + strconv.Itoa(i) + ", "
        p = append(p, v)
        i++
    }
    query = query[0 : len(query)-2]

    if i < 2 {
        return nil
    }

    if model.FieldByName("WherePart").Len() != 0 {
        query += " WHERE %s;"
        v := model.MethodByName("GenerateWherePart").Call([]reflect.Value{reflect.ValueOf(i)})
        return QueryRow(fmt.Sprintf(query, tableName, v[0]), append(p, v[1].Interface().([]interface{})...))
    } else {
        query += ";"
        return QueryRow(fmt.Sprintf(query, tableName), p)
    }
}

func QueryDeleteByIds(tableName, ids string) {
    query := "DELETE FROM %s WHERE id IN (%s)"
    Exec(fmt.Sprintf(query, tableName, ids), nil)
}

func IsExists_(tableName string, fields []string, params []interface{}) bool {
    query := "SELECT %s FROM %s WHERE %s;"
    f := strings.Join(fields, ", ")
    p := strings.Join(MakePairs(fields), " AND ")

    var result string
    row := QueryRow(fmt.Sprintf(query, f, tableName, p), params)
    err := row.Scan(&result)

    return err != sql.ErrNoRows
}

func MakeParams(n int) []string {
    var result = make([]string, n)
    for i := 0; i < n; i++ {
        result[i] = "$" + strconv.Itoa(i+1)
    }
    return result
}

func MakePairs(fields []string) []string {
    var result = make([]string, len(fields))
    for i := 0; i < len(fields); i++ {
        result[i] = fields[i] + "=$" + strconv.Itoa(i+1)
    }
    return result
}

func Select(m interface{}, fields []string) []interface{} {
    model := reflect.ValueOf(m).Elem()
    tableName := model.FieldByName("TableName").String()

    orderBy := " ORDER BY " + model.FieldByName("OrderBy").Interface().(string)

    var limit string
    switch model.FieldByName("Limit").Interface().(type) {
    case string:
        limit = " LIMIT " + model.FieldByName("Limit").Interface().(string)
        break
    case int:
        limit = " LIMIT " + strconv.Itoa(model.FieldByName("Limit").Interface().(int))
        break
    }

    offset := " OFFSET " + strconv.Itoa(model.FieldByName("Offset").Interface().(int))
    extra := orderBy + limit + offset

    query := "SELECT %s FROM %s"

    if model.FieldByName("WherePart").Len() != 0 {
        query += " WHERE %s" + extra + ";"
        v := model.MethodByName("GenerateWherePart").Call([]reflect.Value{reflect.ValueOf(1)})
        return Query(fmt.Sprintf(query, strings.Join(fields, ", "), tableName, v[0]), v[1].Interface().([]interface{}))
    } else {
        query += extra + ";"
        return Query(fmt.Sprintf(query, strings.Join(fields, ", "), tableName), nil)
    }
}

func SelectRow(m interface{}, fields []string) *sql.Row {
    model := reflect.ValueOf(m).Elem()
    tableName := model.FieldByName("TableName").String()

    query := "SELECT %s FROM %s"

    if model.FieldByName("WherePart").Len() != 0 {
        query += " WHERE %s;"
        v := model.MethodByName("GenerateWherePart").Call([]reflect.Value{reflect.ValueOf(1)})
        return QueryRow(fmt.Sprintf(query, strings.Join(fields, ", "), tableName, v[0]), v[1].Interface().([]interface{}))
    } else {
        query += ";"
        return QueryRow(fmt.Sprintf(query, strings.Join(fields, ", "), tableName), nil)
    }
}

func ConvertData(columns []string, size int64, rows *sql.Rows) []interface{} {
    row := make([]interface{}, len(columns))
    values := make([]interface{}, len(columns))
    answer := make([]interface{}, size)

    for i, _ := range row {
        row[i] = &values[i]
    }

    j := 0
    for rows.Next() {
        rows.Scan(row...)
        record := make(map[string]interface{}, len(values))
        for i, col := range values {
            if col != nil {
                //fmt.Printf("\n%s: type= %s\n", columns[i], reflect.TypeOf(col))
                switch col.(type) {
                case bool:
                    record[columns[i]] = col.(bool)
                    break
                case int:
                    record[columns[i]] = col.(int)
                    break
                case int64:
                    record[columns[i]] = int(col.(int64))
                    break
                case float64:
                    record[columns[i]] = col.(float64)
                    break
                case string:
                    record[columns[i]] = col.(string)
                    break
                // case []byte:
                //     record[columns[i]] = string(col.([]byte))
                //     break
                case []int8:
                    record[columns[i]] = col.([]string)
                    break
                case time.Time:
                    record[columns[i]] = col
                    break
                case []uint8:
                    data := strings.Split(strings.Trim(string(col.([]uint8)), "{}"), ",")
                    if len(data) == 1 {
                        record[columns[i]] = data[0]
                    } else {
                        record[columns[i]] = data
                    }
                    break
                default:
                    utils.HandleErr("ConvertData: ", errors.New("Unexpected type."), nil)
                }
            }
            answer[j] = record
        }
        j++
    }
    rows.Close()
    return answer
}
<|endoftext|>