<commit_before>package gitschemalex

import (
	"database/sql"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"testing"

	_ "github.com/go-sql-driver/mysql"
	"github.com/lestrrat/go-test-mysqld"
)

func TestRunner(t *testing.T) {
	var dsn = "root:@tcp(127.0.0.1:3306)/mysql"
	if ok, _ := strconv.ParseBool(os.Getenv("TRAVIS")); !ok {
		mysqld, err := mysqltest.NewMysqld(nil)
		if err != nil {
			t.Fatal(err)
		}
		defer mysqld.Stop()
		dsn = mysqld.DSN()
	}

	db, err := sql.Open("mysql", dsn)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()

	if _, err := db.Exec("CREATE DATABASE IF NOT EXISTS `test`"); err != nil {
		t.Fatal(err)
	}

	if _, err := db.Exec("USE `test`"); err != nil {
		t.Fatal(err)
	}

	dir, err := ioutil.TempDir("", "gitschemalex")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	if err := os.Chdir(dir); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "init").Run(); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "config", "user.email", "hoge@example.com").Run(); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "config", "user.name", "hoge").Run(); err != nil {
		t.Fatal(err)
	}

	schema, err := os.Create(filepath.Join(dir, "schema.sql"))
	if err != nil {
		t.Fatal(err)
	}

	// first table

	if _, err := schema.WriteString("CREATE TABLE hoge ( `id` INTEGER NOT NULL, `c` VARCHAR(20) );\n"); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "add", "schema.sql").Run(); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "commit", "-m", "initial commit").Run(); err != nil {
		t.Fatal(err)
	}

	// This is a silly hack, but we need to change the DSN from "mysql" or
	// whatever to "test"
	re := regexp.MustCompile(`/[^/]+$`)
	dsn = re.ReplaceAllString(dsn, `/test`)
	r := &Runner{
		Workspace: dir,
		Deploy:    true,
		DSN:       dsn,
		Table:     "git_schemalex_version",
		Schema:    "schema.sql",
	}
	if err := r.Run(); err != nil {
		t.Fatal(err)
	}

	// deployed
	if _, err := db.Exec("INSERT INTO `hoge` (`id`, `c`) VALUES (1, '2')"); err != nil {
		t.Fatal(err)
	}

	// second table

	if _, err := schema.WriteString("CREATE TABLE fuga ( `id` INTEGER NOT NULL, `c` VARCHAR(20) );\n"); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "add", "schema.sql").Run(); err != nil {
		t.Fatal(err)
	}
	if err := exec.Command("git", "commit", "--author", "hoge <hoge@example.com>", "-m", "second commit").Run(); err != nil {
		t.Fatal(err)
	}

	if err := r.Run(); err != nil {
		t.Fatal(err)
	}

	if _, err := db.Exec("INSERT INTO `fuga` (`id`, `c`) VALUES (1, '2')"); err != nil {
		t.Fatal(err)
	}

	// equal version

	if e, g := ErrEqualVersion, r.Run(); e != g {
		t.Fatal("should %v got %v", e, g)
	}
}
<commit_msg>I'm so stupid<commit_after>package gitschemalex

import (
	"database/sql"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"testing"

	_ "github.com/go-sql-driver/mysql"
	"github.com/lestrrat/go-test-mysqld"
)

func TestRunner(t *testing.T) {
	var dsn = "root:@tcp(127.0.0.1:3306)/mysql"
	if ok, _ := strconv.ParseBool(os.Getenv("TRAVIS")); !ok {
		mysqld, err := mysqltest.NewMysqld(nil)
		if err != nil {
			t.Fatal(err)
		}
		defer mysqld.Stop()
		dsn = mysqld.DSN()
	}

	db, err := sql.Open("mysql", dsn)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()

	if _, err := db.Exec("CREATE DATABASE IF NOT EXISTS `test`"); err != nil {
		t.Fatal(err)
	}

	if _, err := db.Exec("USE `test`"); err != nil {
		t.Fatal(err)
	}

	dir, err := ioutil.TempDir("", "gitschemalex")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	if err := os.Chdir(dir); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "init").Run(); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "config", "user.email", "hoge@example.com").Run(); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "config", "user.name", "hoge").Run(); err != nil {
		t.Fatal(err)
	}

	schema, err := os.Create(filepath.Join(dir, "schema.sql"))
	if err != nil {
		t.Fatal(err)
	}

	// first table

	if _, err := schema.WriteString("CREATE TABLE hoge ( `id` INTEGER NOT NULL, `c` VARCHAR(20) );\n"); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "add", "schema.sql").Run(); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "commit", "-m", "initial commit").Run(); err != nil {
		t.Fatal(err)
	}

	// This is a silly hack, but we need to change the DSN from "mysql" or
	// whatever to "test"
	re := regexp.MustCompile(`/[^/]+$`)
	dsn = re.ReplaceAllString(dsn, `/test`)
	r := &Runner{
		Workspace: dir,
		Deploy:    true,
		DSN:       dsn,
		Table:     "git_schemalex_version",
		Schema:    "schema.sql",
	}
	if err := r.Run(); err != nil {
		t.Fatal(err)
	}

	// deployed
	if _, err := db.Exec("INSERT INTO `hoge` (`id`, `c`) VALUES (1, '2')"); err != nil {
		t.Fatal(err)
	}

	// second table

	if _, err := schema.WriteString("CREATE TABLE fuga ( `id` INTEGER NOT NULL, `c` VARCHAR(20) );\n"); err != nil {
		t.Fatal(err)
	}

	if err := exec.Command("git", "add", "schema.sql").Run(); err != nil {
		t.Fatal(err)
	}
	if err := exec.Command("git", "commit", "--author", "hoge <hoge@example.com>", "-m", "second commit").Run(); err != nil {
		t.Fatal(err)
	}

	if err := r.Run(); err != nil {
		t.Fatal(err)
	}

	if _, err := db.Exec("INSERT INTO `fuga` (`id`, `c`) VALUES (1, '2')"); err != nil {
		t.Fatal(err)
	}

	// equal version

	if e, g := ErrEqualVersion, r.Run(); e != g {
		t.Fatal("should %v got %v", e, g)
	}
}
<|endoftext|>


<commit_before>package ParquetReader

import (
	"encoding/binary"
	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/xitongsys/parquet-go/Common"
	"github.com/xitongsys/parquet-go/Layout"
	"github.com/xitongsys/parquet-go/Marshal"
	"github.com/xitongsys/parquet-go/ParquetFile"
	"github.com/xitongsys/parquet-go/SchemaHandler"
	"github.com/xitongsys/parquet-go/parquet"
	"reflect"
	"strings"
	"sync"
)

type ParquetReader struct {
	SchemaHandler *SchemaHandler.SchemaHandler
	NP            int64 //parallel number
	Footer        *parquet.FileMetaData
	PFile         ParquetFile.ParquetFile

	ColumnBuffers map[string]*ColumnBufferType
}

//Create a parquet column reader
func NewParquetColumnReader(pFile ParquetFile.ParquetFile, np int64) (*ParquetReader, error) {
	var err error
	res := new(ParquetReader)
	res.NP = np
	res.PFile = pFile
	res.ReadFooter()
	res.ColumnBuffers = make(map[string]*ColumnBufferType)
	if res.SchemaHandler, err = SchemaHandler.NewSchemaHandlerFromSchemaList(res.Footer.GetSchema()); err != nil {
		return nil, err
	}

	for i := 0; i < len(res.SchemaHandler.SchemaElements); i++ {
		schema := res.SchemaHandler.SchemaElements[i]
		pathStr := res.SchemaHandler.IndexMap[int32(i)]
		numChildren := schema.GetNumChildren()
		if numChildren == 0 {
			res.ColumnBuffers[pathStr], err = NewColumnBuffer(pFile, res.Footer, res.SchemaHandler, pathStr)
			if err != nil {
				return res, err
			}
		}
	}
	return res, err
}

//Create a parquet reader
func NewParquetReader(pFile ParquetFile.ParquetFile, obj interface{}, np int64) (*ParquetReader, error) {
	var err error
	res := new(ParquetReader)
	res.NP = np
	res.PFile = pFile
	res.ReadFooter()
	res.ColumnBuffers = make(map[string]*ColumnBufferType)
	//res.SchemaHandler = SchemaHandler.NewSchemaHandlerFromSchemaList(res.Footer.GetSchema())
	if res.SchemaHandler, err = SchemaHandler.NewSchemaHandlerFromStruct(obj); err != nil {
		return nil, err
	}
	res.RenameSchema()

	for i := 0; i < len(res.SchemaHandler.SchemaElements); i++ {
		schema := res.SchemaHandler.SchemaElements[i]
		pathStr := res.SchemaHandler.IndexMap[int32(i)]
		numChildren := schema.GetNumChildren()
		if numChildren == 0 {
			res.ColumnBuffers[pathStr], err = NewColumnBuffer(pFile, res.Footer, res.SchemaHandler, pathStr)
			if err != nil {
				return res, err
			}
		}
	}
	return res, err
}

//Rename schema name to inname
func (self *ParquetReader) RenameSchema() {
	for i := 0; i < len(self.Footer.Schema); i++ {
		self.Footer.Schema[i].Name = self.SchemaHandler.Infos[i]["inname"].(string)
	}
	for _, rowGroup := range self.Footer.RowGroups {
		for _, chunk := range rowGroup.Columns {
			exPath := make([]string, 0)
			exPath = append(exPath, self.SchemaHandler.GetRootName())
			exPath = append(exPath, chunk.MetaData.GetPathInSchema()...)
			exPathStr := Common.PathToStr(exPath)

			inPathStr := self.SchemaHandler.ExPathToInPath[exPathStr]
			inPath := Common.StrToPath(inPathStr)[1:]
			chunk.MetaData.PathInSchema = inPath
		}
	}
}

func (self *ParquetReader) GetNumRows() int64 {
	return self.Footer.GetNumRows()
}

//Get the footer size
func (self *ParquetReader) GetFooterSize() (uint32, error) {
	buf := make([]byte, 4)
	if _, err := self.PFile.Seek(-8, 2); err != nil {
		return 0, err
	}
	if _, err := self.PFile.Read(buf); err != nil {
		return 0, err
	}
	size := binary.LittleEndian.Uint32(buf)
	return size, nil
}

//Read footer from parquet file
func (self *ParquetReader) ReadFooter() error {
	var (
		size uint32
		err  error
	)
	if size, err = self.GetFooterSize(); err != nil {
		return err
	}
	if _, err = self.PFile.Seek(int(-(int64)(8+size)), 2); err != nil {
		return err
	}
	self.Footer = parquet.NewFileMetaData()
	pf := thrift.NewTCompactProtocolFactory()
	protocol := pf.GetProtocol(thrift.NewStreamTransportR(self.PFile))
	if err = self.Footer.Read(protocol); err != nil {
		return err
	}
	return nil
}

//Read rows of parquet file
func (self *ParquetReader) Read(dstInterface interface{}) (err error) {
	defer func() {
		if r := recover(); r != nil {
			err = r.(error)
		}
	}()

	tmap := make(map[string]*Layout.Table)
	locker := new(sync.Mutex)
	ot := reflect.TypeOf(dstInterface).Elem().Elem()
	num := reflect.ValueOf(dstInterface).Elem().Len()
	if num <= 0 {
		return
	}

	doneChan := make(chan int, self.NP)
	taskChan := make(chan string, len(self.ColumnBuffers))
	stopChan := make(chan int)

	for i := int64(0); i < self.NP; i++ {
		go func() {
			for {
				select {
				case <-stopChan:
					return
				case pathStr := <-taskChan:
					cb := self.ColumnBuffers[pathStr]
					table, _, _ := cb.ReadRows(int64(num))
					locker.Lock()
					if _, ok := tmap[pathStr]; ok {
						tmap[pathStr].Merge(table)
					} else {
						tmap[pathStr] = Layout.NewTableFromTable(table)
						tmap[pathStr].Merge(table)
					}
					locker.Unlock()
					doneChan <- 0
				}
			}
		}()
	}
	for key, _ := range self.ColumnBuffers {
		taskChan <- key
	}
	for i := 0; i < len(self.ColumnBuffers); i++ {
		<-doneChan
	}
	for i := int64(0); i < self.NP; i++ {
		stopChan <- 0
	}

	dstList := make([]interface{}, self.NP)
	delta := (int64(num) + self.NP - 1) / self.NP

	doneChan = make(chan int)
	for c := int64(0); c < self.NP; c++ {
		bgn := c * delta
		end := bgn + delta
		if end > int64(num) {
			end = int64(num)
		}
		if bgn >= int64(num) {
			bgn, end = int64(num), int64(num)
		}
		go func(b, e, index int) {
			dstList[index] = reflect.New(reflect.SliceOf(ot)).Interface()
			Marshal.Unmarshal(&tmap, b, e, dstList[index], self.SchemaHandler)
			doneChan <- 0
		}(int(bgn), int(end), int(c))
	}
	for c := int64(0); c < self.NP; c++ {
		<-doneChan
	}

	resTmp := reflect.MakeSlice(reflect.SliceOf(ot), 0, num)
	for _, dst := range dstList {
		resTmp = reflect.AppendSlice(resTmp, reflect.ValueOf(dst).Elem())
	}
	reflect.ValueOf(dstInterface).Elem().Set(resTmp)

	return nil
}

//Read column by path in schema.
func (self *ParquetReader) ReadColumnByPath(pathStr string, dstInterface *[]interface{}) (err error) {
	defer func() {
		if r := recover(); r != nil {
			err = r.(error)
		}
	}()

	num := reflect.ValueOf(dstInterface).Elem().Len()
	if num <= 0 {
		return
	}
	rootName := self.SchemaHandler.GetRootName()

	if len(pathStr) <= 0 {
		return
	} else if !strings.HasPrefix(pathStr, rootName) {
		pathStr = rootName + "." + pathStr
	}

	if cb, ok := self.ColumnBuffers[pathStr]; ok {
		table, _, _ := cb.ReadRows(int64(num))
		i, j, ln := 0, 0, len(table.Values)

		numi := 0
		for i < ln {
			rec := [][]interface{}{}
			j = i
			for j < ln {
				rec = append(rec, []interface{}{table.Values[j], table.RepetitionLevels[j], table.DefinitionLevels[j]})
				j++
				if j < ln && table.RepetitionLevels[j] == 0 {
					break
				}
			}
			(*dstInterface)[numi] = rec
			numi++
			i = j
		}
	}
	return nil
}

//Read column by index. The index of first column is 0.
func (self *ParquetReader) ReadColumnByIndex(index int, dstInterface *[]interface{}) error {
	if index >= len(self.SchemaHandler.ValueColumns) {
		return nil
	}
	pathStr := self.SchemaHandler.ValueColumns[index]
	err := self.ReadColumnByPath(pathStr, dstInterface)
	return err
}

//Stop Read
func (self *ParquetReader) ReadStop() {
	for _, cb := range self.ColumnBuffers {
		if cb != nil {
			cb.PFile.Close()
		}
	}
}
<commit_msg>del recover in parquetreader<commit_after>package ParquetReader

import (
	"encoding/binary"
	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/xitongsys/parquet-go/Common"
	"github.com/xitongsys/parquet-go/Layout"
	"github.com/xitongsys/parquet-go/Marshal"
	"github.com/xitongsys/parquet-go/ParquetFile"
	"github.com/xitongsys/parquet-go/SchemaHandler"
	"github.com/xitongsys/parquet-go/parquet"
	"reflect"
	"strings"
	"sync"
)

type ParquetReader struct {
	SchemaHandler *SchemaHandler.SchemaHandler
	NP            int64 //parallel number
	Footer        *parquet.FileMetaData
	PFile         ParquetFile.ParquetFile

	ColumnBuffers map[string]*ColumnBufferType
}

//Create a parquet column reader
func NewParquetColumnReader(pFile ParquetFile.ParquetFile, np int64) (*ParquetReader, error) {
	var err error
	res := new(ParquetReader)
	res.NP = np
	res.PFile = pFile
	res.ReadFooter()
	res.ColumnBuffers = make(map[string]*ColumnBufferType)
	if res.SchemaHandler, err = SchemaHandler.NewSchemaHandlerFromSchemaList(res.Footer.GetSchema()); err != nil {
		return nil, err
	}

	for i := 0; i < len(res.SchemaHandler.SchemaElements); i++ {
		schema := res.SchemaHandler.SchemaElements[i]
		pathStr := res.SchemaHandler.IndexMap[int32(i)]
		numChildren := schema.GetNumChildren()
		if numChildren == 0 {
			res.ColumnBuffers[pathStr], err = NewColumnBuffer(pFile, res.Footer, res.SchemaHandler, pathStr)
			if err != nil {
				return res, err
			}
		}
	}
	return res, err
}

//Create a parquet reader
func NewParquetReader(pFile ParquetFile.ParquetFile, obj interface{}, np int64) (*ParquetReader, error) {
	var err error
	res := new(ParquetReader)
	res.NP = np
	res.PFile = pFile
	res.ReadFooter()
	res.ColumnBuffers = make(map[string]*ColumnBufferType)
	//res.SchemaHandler = SchemaHandler.NewSchemaHandlerFromSchemaList(res.Footer.GetSchema())
	if res.SchemaHandler, err = SchemaHandler.NewSchemaHandlerFromStruct(obj); err != nil {
		return nil, err
	}
	res.RenameSchema()

	for i := 0; i < len(res.SchemaHandler.SchemaElements); i++ {
		schema := res.SchemaHandler.SchemaElements[i]
		pathStr := res.SchemaHandler.IndexMap[int32(i)]
		numChildren := schema.GetNumChildren()
		if numChildren == 0 {
			res.ColumnBuffers[pathStr], err = NewColumnBuffer(pFile, res.Footer, res.SchemaHandler, pathStr)
			if err != nil {
				return res, err
			}
		}
	}
	return res, err
}

//Rename schema name to inname
func (self *ParquetReader) RenameSchema() {
	for i := 0; i < len(self.Footer.Schema); i++ {
		self.Footer.Schema[i].Name = self.SchemaHandler.Infos[i]["inname"].(string)
	}
	for _, rowGroup := range self.Footer.RowGroups {
		for _, chunk := range rowGroup.Columns {
			exPath := make([]string, 0)
			exPath = append(exPath, self.SchemaHandler.GetRootName())
			exPath = append(exPath, chunk.MetaData.GetPathInSchema()...)
			exPathStr := Common.PathToStr(exPath)

			inPathStr := self.SchemaHandler.ExPathToInPath[exPathStr]
			inPath := Common.StrToPath(inPathStr)[1:]
			chunk.MetaData.PathInSchema = inPath
		}
	}
}

func (self *ParquetReader) GetNumRows() int64 {
	return self.Footer.GetNumRows()
}

//Get the footer size
func (self *ParquetReader) GetFooterSize() (uint32, error) {
	buf := make([]byte, 4)
	if _, err := self.PFile.Seek(-8, 2); err != nil {
		return 0, err
	}
	if _, err := self.PFile.Read(buf); err != nil {
		return 0, err
	}
	size := binary.LittleEndian.Uint32(buf)
	return size, nil
}

//Read footer from parquet file
func (self *ParquetReader) ReadFooter() error {
	var (
		size uint32
		err  error
	)
	if size, err = self.GetFooterSize(); err != nil {
		return err
	}
	if _, err = self.PFile.Seek(int(-(int64)(8+size)), 2); err != nil {
		return err
	}
	self.Footer = parquet.NewFileMetaData()
	pf := thrift.NewTCompactProtocolFactory()
	protocol := pf.GetProtocol(thrift.NewStreamTransportR(self.PFile))
	if err = self.Footer.Read(protocol); err != nil {
		return err
	}
	return nil
}

//Read rows of parquet file
func (self *ParquetReader) Read(dstInterface interface{}) (err error) {
	tmap := make(map[string]*Layout.Table)
	locker := new(sync.Mutex)
	ot := reflect.TypeOf(dstInterface).Elem().Elem()
	num := reflect.ValueOf(dstInterface).Elem().Len()
	if num <= 0 {
		return
	}

	doneChan := make(chan int, self.NP)
	taskChan := make(chan string, len(self.ColumnBuffers))
	stopChan := make(chan int)

	for i := int64(0); i < self.NP; i++ {
		go func() {
			for {
				select {
				case <-stopChan:
					return
				case pathStr := <-taskChan:
					cb := self.ColumnBuffers[pathStr]
					table, _, _ := cb.ReadRows(int64(num))
					locker.Lock()
					if _, ok := tmap[pathStr]; ok {
						tmap[pathStr].Merge(table)
					} else {
						tmap[pathStr] = Layout.NewTableFromTable(table)
						tmap[pathStr].Merge(table)
					}
					locker.Unlock()
					doneChan <- 0
				}
			}
		}()
	}
	for key, _ := range self.ColumnBuffers {
		taskChan <- key
	}
	for i := 0; i < len(self.ColumnBuffers); i++ {
		<-doneChan
	}
	for i := int64(0); i < self.NP; i++ {
		stopChan <- 0
	}

	dstList := make([]interface{}, self.NP)
	delta := (int64(num) + self.NP - 1) / self.NP

	doneChan = make(chan int)
	for c := int64(0); c < self.NP; c++ {
		bgn := c * delta
		end := bgn + delta
		if end > int64(num) {
			end = int64(num)
		}
		if bgn >= int64(num) {
			bgn, end = int64(num), int64(num)
		}
		go func(b, e, index int) {
			dstList[index] = reflect.New(reflect.SliceOf(ot)).Interface()
			if r := Marshal.Unmarshal(&tmap, b, e, dstList[index], self.SchemaHandler); r != nil {
				err = r
			}
			doneChan <- 0
		}(int(bgn), int(end), int(c))
	}
	for c := int64(0); c < self.NP; c++ {
		<-doneChan
	}

	resTmp := reflect.MakeSlice(reflect.SliceOf(ot), 0, num)
	for _, dst := range dstList {
		resTmp = reflect.AppendSlice(resTmp, reflect.ValueOf(dst).Elem())
	}
	reflect.ValueOf(dstInterface).Elem().Set(resTmp)

	return nil
}

//Read column by path in schema.
func (self *ParquetReader) ReadColumnByPath(pathStr string, dstInterface *[]interface{}) (err error) {
	defer func() {
		if r := recover(); r != nil {
			err = r.(error)
		}
	}()

	num := reflect.ValueOf(dstInterface).Elem().Len()
	if num <= 0 {
		return
	}
	rootName := self.SchemaHandler.GetRootName()

	if len(pathStr) <= 0 {
		return
	} else if !strings.HasPrefix(pathStr, rootName) {
		pathStr = rootName + "." + pathStr
	}

	if cb, ok := self.ColumnBuffers[pathStr]; ok {
		table, _, _ := cb.ReadRows(int64(num))
		i, j, ln := 0, 0, len(table.Values)

		numi := 0
		for i < ln {
			rec := [][]interface{}{}
			j = i
			for j < ln {
				rec = append(rec, []interface{}{table.Values[j], table.RepetitionLevels[j], table.DefinitionLevels[j]})
				j++
				if j < ln && table.RepetitionLevels[j] == 0 {
					break
				}
			}
			(*dstInterface)[numi] = rec
			numi++
			i = j
		}
	}
	return nil
}

//Read column by index. The index of first column is 0.
func (self *ParquetReader) ReadColumnByIndex(index int, dstInterface *[]interface{}) error {
	if index >= len(self.SchemaHandler.ValueColumns) {
		return nil
	}
	pathStr := self.SchemaHandler.ValueColumns[index]
	err := self.ReadColumnByPath(pathStr, dstInterface)
	return err
}

//Stop Read
func (self *ParquetReader) ReadStop() {
	for _, cb := range self.ColumnBuffers {
		if cb != nil {
			cb.PFile.Close()
		}
	}
}
<|endoftext|>


<commit_before>package amazon

import (
	"errors"
	"fmt"
	"koding/kites/kloud/eventer"
	"koding/kites/kloud/machinestate"
	"koding/kites/kloud/waitstate"
	"time"

	"golang.org/x/net/context"

	"github.com/mitchellh/goamz/ec2"
)

var (
	ErrInstanceTerminated = errors.New("instance is terminated")
	ErrNoInstances        = errors.New("no instances found")
)

func (a *Amazon) Build(buildData *ec2.RunInstances) (string, error) {
	resp, err := a.Client.RunInstances(buildData)
	if err != nil {
		return "", err
	}

	// we do not check intentionally, because CreateInstance() is designed to
	// create only one instance. If it creates something else we catch it here
	// by panicing
	instance := resp.Instances[0]

	return instance.InstanceId, nil
}

func (a *Amazon) CheckBuild(ctx context.Context, instanceId string, start, finish int) (ec2.Instance, error) {
	ev, withPush := eventer.FromContext(ctx)
	if withPush {
		ev.Push(&eventer.Event{
			Message:    "Building machine",
			Status:     machinestate.Building,
			Percentage: start,
		})
	}

	var instance ec2.Instance
	var err error
	stateFunc := func(currentPercentage int) (machinestate.State, error) {
		if withPush {
			ev.Push(&eventer.Event{
				Message:    "Building machine",
				Status:     machinestate.Building,
				Percentage: currentPercentage,
			})
		}

		instance, err = a.InstanceById(instanceId)
		if err != nil {
			return 0, err
		}

		currentStatus := StatusToState(instance.State.Name)

		// happens when there is no volume limit. The instance will be not
		// build and it returns terminated from AWS
		if currentStatus.In(machinestate.Terminated, machinestate.Terminating) {
			return 0, ErrInstanceTerminated
		}

		return currentStatus, nil
	}

	ws := waitstate.WaitState{
		Timeout:      12 * time.Minute,
		StateFunc:    stateFunc,
		DesiredState: machinestate.Running,
		Start:        start,
		Finish:       finish,
	}

	if err := ws.Wait(); err != nil {
		return ec2.Instance{}, err
	}

	return instance, nil
}

func (a *Amazon) Instance() (ec2.Instance, error) {
	return a.InstanceById(a.Builder.InstanceId)
}

func (a *Amazon) InstanceById(id string) (ec2.Instance, error) {
	resp, err := a.Client.Instances([]string{id}, ec2.NewFilter())
	if err != nil {
		if awsErr, ok := err.(*ec2.Error); ok {
			if awsErr.Code == "InvalidInstanceID.NotFound" {
				return ec2.Instance{}, ErrNoInstances
			}
		}

		return ec2.Instance{}, err
	}

	if len(resp.Reservations) == 0 {
		fmt.Errorf("the instance ID '%s' does not exist", id)
		return ec2.Instance{}, ErrNoInstances
	}

	return resp.Reservations[0].Instances[0], nil
}

func (a *Amazon) InstancesByFilter(filter *ec2.Filter) ([]ec2.Instance, error) {
	if filter == nil {
		filter = ec2.NewFilter()
	}

	resp, err := a.Client.Instances([]string{}, filter)
	if err != nil {
		return nil, err
	}

	if len(resp.Reservations) == 0 {
		return nil, ErrNoInstances
	}

	// we don't care about reservations and every reservation struct returns
	// only on single instance. Just collect them and return a list of
	// instances
	instances := make([]ec2.Instance, len(resp.Reservations))
	for i, r := range resp.Reservations {
		instances[i] = r.Instances[0]
	}

	return instances, nil
}

func (a *Amazon) SecurityGroup(name string) (ec2.SecurityGroup, error) {
	// Somehow only filter works, defining inside SecurityGroup doesn't work
	filter := ec2.NewFilter()
	filter.Add("group-name", name)

	resp, err := a.Client.SecurityGroups([]ec2.SecurityGroup{}, filter)
	if err != nil {
		return ec2.SecurityGroup{}, err
	}

	if len(resp.Groups) == 0 {
		return ec2.SecurityGroup{}, fmt.Errorf("the security group name '%s' does not exist", name)
	}

	return resp.Groups[0].SecurityGroup, nil
}

func (a *Amazon) ListVPCs() (*ec2.VpcsResp, error) {
	return a.Client.DescribeVpcs([]string{}, ec2.NewFilter())
}

func (a *Amazon) ListSubnets() (*ec2.SubnetsResp, error) {
	return a.Client.DescribeSubnets([]string{}, ec2.NewFilter())
}

func (a *Amazon) ListSubnetsFromVPC(vpcId string) (*ec2.SubnetsResp, error) {
	filter := ec2.NewFilter()
	filter.Add("vpc-id", vpcId)

	return a.Client.DescribeSubnets([]string{}, filter)
}
<commit_msg>kloud: Increased the CheckBuild WaitiState from 12m to 15m<commit_after>package amazon

import (
	"errors"
	"fmt"
	"koding/kites/kloud/eventer"
	"koding/kites/kloud/machinestate"
	"koding/kites/kloud/waitstate"
	"time"

	"golang.org/x/net/context"

	"github.com/mitchellh/goamz/ec2"
)

var (
	ErrInstanceTerminated = errors.New("instance is terminated")
	ErrNoInstances        = errors.New("no instances found")
)

func (a *Amazon) Build(buildData *ec2.RunInstances) (string, error) {
	resp, err := a.Client.RunInstances(buildData)
	if err != nil {
		return "", err
	}

	// we do not check intentionally, because CreateInstance() is designed to
	// create only one instance. If it creates something else we catch it here
	// by panicing
	instance := resp.Instances[0]

	return instance.InstanceId, nil
}

func (a *Amazon) CheckBuild(ctx context.Context, instanceId string, start, finish int) (ec2.Instance, error) {
	ev, withPush := eventer.FromContext(ctx)
	if withPush {
		ev.Push(&eventer.Event{
			Message:    "Building machine",
			Status:     machinestate.Building,
			Percentage: start,
		})
	}

	var instance ec2.Instance
	var err error
	stateFunc := func(currentPercentage int) (machinestate.State, error) {
		if withPush {
			ev.Push(&eventer.Event{
				Message:    "Building machine",
				Status:     machinestate.Building,
				Percentage: currentPercentage,
			})
		}

		instance, err = a.InstanceById(instanceId)
		if err != nil {
			return 0, err
		}

		currentStatus := StatusToState(instance.State.Name)

		// happens when there is no volume limit. The instance will be not
		// build and it returns terminated from AWS
		if currentStatus.In(machinestate.Terminated, machinestate.Terminating) {
			return 0, ErrInstanceTerminated
		}

		return currentStatus, nil
	}

	ws := waitstate.WaitState{
		Timeout:      15 * time.Minute,
		StateFunc:    stateFunc,
		DesiredState: machinestate.Running,
		Start:        start,
		Finish:       finish,
	}

	if err := ws.Wait(); err != nil {
		return ec2.Instance{}, err
	}

	return instance, nil
}

func (a *Amazon) Instance() (ec2.Instance, error) {
	return a.InstanceById(a.Builder.InstanceId)
}

func (a *Amazon) InstanceById(id string) (ec2.Instance, error) {
	resp, err := a.Client.Instances([]string{id}, ec2.NewFilter())
	if err != nil {
		if awsErr, ok := err.(*ec2.Error); ok {
			if awsErr.Code == "InvalidInstanceID.NotFound" {
				return ec2.Instance{}, ErrNoInstances
			}
		}

		return ec2.Instance{}, err
	}

	if len(resp.Reservations) == 0 {
		fmt.Errorf("the instance ID '%s' does not exist", id)
		return ec2.Instance{}, ErrNoInstances
	}

	return resp.Reservations[0].Instances[0], nil
}

func (a *Amazon) InstancesByFilter(filter *ec2.Filter) ([]ec2.Instance, error) {
	if filter == nil {
		filter = ec2.NewFilter()
	}

	resp, err := a.Client.Instances([]string{}, filter)
	if err != nil {
		return nil, err
	}

	if len(resp.Reservations) == 0 {
		return nil, ErrNoInstances
	}

	// we don't care about reservations and every reservation struct returns
	// only on single instance. Just collect them and return a list of
	// instances
	instances := make([]ec2.Instance, len(resp.Reservations))
	for i, r := range resp.Reservations {
		instances[i] = r.Instances[0]
	}

	return instances, nil
}

func (a *Amazon) SecurityGroup(name string) (ec2.SecurityGroup, error) {
	// Somehow only filter works, defining inside SecurityGroup doesn't work
	filter := ec2.NewFilter()
	filter.Add("group-name", name)

	resp, err := a.Client.SecurityGroups([]ec2.SecurityGroup{}, filter)
	if err != nil {
		return ec2.SecurityGroup{}, err
	}

	if len(resp.Groups) == 0 {
		return ec2.SecurityGroup{}, fmt.Errorf("the security group name '%s' does not exist", name)
	}

	return resp.Groups[0].SecurityGroup, nil
}

func (a *Amazon) ListVPCs() (*ec2.VpcsResp, error) {
	return a.Client.DescribeVpcs([]string{}, ec2.NewFilter())
}

func (a *Amazon) ListSubnets() (*ec2.SubnetsResp, error) {
	return a.Client.DescribeSubnets([]string{}, ec2.NewFilter())
}

func (a *Amazon) ListSubnetsFromVPC(vpcId string) (*ec2.SubnetsResp, error) {
	filter := ec2.NewFilter()
	filter.Add("vpc-id", vpcId)

	return a.Client.DescribeSubnets([]string{}, filter)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/streadway/amqp"
	"koding/databases/mongo"
	"koding/databases/neo4j"
	"koding/tools/amqputil"
	"labix.org/v2/mgo/bson"
	"strings"
	"time"
)

var (
	EXCHANGE_NAME     = "graphFeederExchange"
	WORKER_QUEUE_NAME = "graphFeederWorkerQueue"
	TIME_FORMAT       = "2006-01-02T15:04:05.000Z"
)

type Consumer struct {
	conn    *amqp.Connection
	channel *amqp.Channel
}

type Message struct {
	Event   string                   `json:"event"`
	Payload []map[string]interface{} `json:"payload"`
}

func main() {
	startConsuming()
}

//here, mapping of decoded json
func jsonDecode(data string) (*Message, error) {
	source := &Message{}
	err := json.Unmarshal([]byte(data), &source)
	if err != nil {
		return source, err
	}

	return source, nil
}

func startConsuming() {

	c := &Consumer{
		conn:    nil,
		channel: nil,
	}

	c.conn = amqputil.CreateConnection("neo4jFeeding")
	c.channel = amqputil.CreateChannel(c.conn)

	err := c.channel.ExchangeDeclare(EXCHANGE_NAME, "fanout", true, false, false, false, nil)
	if err != nil {
		fmt.Println("exchange.declare: %s", err)
		panic(err)
	}

	//name, durable, autoDelete, exclusive, noWait, args Table
	if _, err := c.channel.QueueDeclare(WORKER_QUEUE_NAME, true, false, false, false, nil); err != nil {
		fmt.Println("queue.declare: %s", err)
		panic(err)
	}

	if err := c.channel.QueueBind(WORKER_QUEUE_NAME, "" /* binding key */, EXCHANGE_NAME, false, nil); err != nil {
		fmt.Println("queue.bind: %s", err)
		panic(err)
	}

	//(queue, consumer string, autoAck, exclusive, noLocal, noWait bool, args Table) (<-chan Delivery, error) {
	relationshipEvent, err := c.channel.Consume(WORKER_QUEUE_NAME, "neo4jFeeding", true, false, false, false, nil)
	if err != nil {
		fmt.Println("basic.consume: %s", err)
		panic(err)
	}

	fmt.Println("Neo4J Feeder worker started")

	for msg := range relationshipEvent {
		body := fmt.Sprintf("%s", msg.Body)

		message, err := jsonDecode(body)
		if err != nil {
			fmt.Println("Wrong message format", err, body)

			continue
		}

		if len(message.Payload) < 1 {
			fmt.Println("Wrong message format; payload should be an Array", message)

			continue
		}
		data := message.Payload[0]

		fmt.Println(message.Event)
		if message.Event == "RelationshipSaved" {
			createNode(data)
		} else if message.Event == "RelationshipRemoved" {
			deleteRelationship(data)
		} else if message.Event == "updateInstance" {
			updateNode(data)
		} else if message.Event == "RemovedFromCollection" {
			deleteNode(data)
		} else {
			fmt.Println(message.Event)
		}
	}
}

func checkIfEligible(sourceName, targetName string) bool {
	notAllowedSuffixes := []string{
		"Bucket",
		"BucketActivity",
	}

	for _, name := range neo4j.NotAllowedNames {
		if name == sourceName {
			fmt.Println("not eligible " + sourceName)
			return false
		}

		if name == targetName {
			fmt.Println("not eligible " + targetName)
			return false
		}
	}

	for _, name := range notAllowedSuffixes {

		if strings.HasSuffix(sourceName, name) {
			fmt.Println("not eligible " + sourceName)
			return false
		}

		if strings.HasSuffix(targetName, name) {
			fmt.Println("not eligible " + targetName)
			return false
		}

	}

	return true
}

func createNode(data map[string]interface{}) {

	sourceId := fmt.Sprintf("%s", data["sourceId"])
	sourceName := fmt.Sprintf("%s", data["sourceName"])

	targetId := fmt.Sprintf("%s", data["targetId"])
	targetName := fmt.Sprintf("%s", data["targetName"])

	if !checkIfEligible(sourceName, targetName) {
		return
	}

	sourceContent, err := mongo.FetchContent(bson.ObjectIdHex(sourceId), sourceName)
	if err != nil {
		fmt.Println("sourceContent", err)
		return
	}
	sourceNode := neo4j.CreateUniqueNode(sourceId, sourceName)
	neo4j.UpdateNode(sourceId, sourceContent)

	targetContent, err := mongo.FetchContent(bson.ObjectIdHex(targetId), targetName)
	if err != nil {
		fmt.Println("targetContent", err)
		return
	}
	targetNode := neo4j.CreateUniqueNode(targetId, targetName)
	neo4j.UpdateNode(targetId, targetContent)

	source := fmt.Sprintf("%s", sourceNode["create_relationship"])
	target := fmt.Sprintf("%s", targetNode["self"])

	if _, ok := data["as"]; !ok {
		fmt.Println("as value is not set on this relationship. Discarding this record", data)
		return
	}
	as := fmt.Sprintf("%s", data["as"])

	if _, ok := data["_id"]; !ok {
		fmt.Println("id value is not set on this relationship. Discarding this record", data)
		return
	}

	createdAt := getCreatedAtDate(data)
	relationshipData := fmt.Sprintf(`{"createdAt" : "%s", "createdAtEpoch" : %d }`, createdAt.Format(TIME_FORMAT), createdAt.Unix())
	neo4j.CreateRelationshipWithData(as, source, target, relationshipData)
}

func getCreatedAtDate(data map[string]interface{}) time.Time {

	if _, ok := data["timestamp"]; ok {
		t, err := time.Parse(TIME_FORMAT, data["timestamp"].(string))
		// if error doesnt exists, return createdAt
		if err == nil {
			return t.UTC()
		}
	}

	id := fmt.Sprintf("%s", data["_id"])
	if bson.IsObjectIdHex(id) {
		return bson.ObjectIdHex(id).Time().UTC()
	}

	fmt.Print("Couldnt determine the createdAt time, returning Now() as creatdAt")
	return time.Now().UTC()
}

func deleteNode(data map[string]interface{}) {

	if _, ok := data["_id"]; !ok {
		return
	}
	id := fmt.Sprintf("%s", data["_id"])
	neo4j.DeleteNode(id)
}

func deleteRelationship(data map[string]interface{}) {
	sourceId := fmt.Sprintf("%s", data["sourceId"])
	targetId := fmt.Sprintf("%s", data["targetId"])
	as := fmt.Sprintf("%s", data["as"])

    // we are not doing anything with result for now
    // do not pollute console
	neo4j.DeleteRelationship(sourceId, targetId, as)
	//result := neo4j.DeleteRelationship(sourceId, targetId, as)
    //if result {
	//	fmt.Println("Relationship deleted")
	//} else {
	//	fmt.Println("Relationship couldnt be deleted")
	//}
}

func updateNode(data map[string]interface{}) {

	if _, ok := data["bongo_"]; !ok {
		return
	}
	if _, ok := data["data"]; !ok {
		return
	}

	bongo := data["bongo_"].(map[string]interface{})
	obj := data["data"].(map[string]interface{})

	sourceId := fmt.Sprintf("%s", obj["_id"])
	sourceName := fmt.Sprintf("%s", bongo["constructorName"])

	if !checkIfEligible(sourceName, "") {
		return
	}

	sourceContent, err := mongo.FetchContent(bson.ObjectIdHex(sourceId), sourceName)
	if err != nil {
		fmt.Println("sourceContent", err)
		return
	}

	neo4j.CreateUniqueNode(sourceId, sourceName)
	neo4j.UpdateNode(sourceId, sourceContent)
}
<commit_msg>GraphFeeder : whitespace is fixed<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/streadway/amqp"
	"koding/databases/mongo"
	"koding/databases/neo4j"
	"koding/tools/amqputil"
	"labix.org/v2/mgo/bson"
	"strings"
	"time"
)

var (
	EXCHANGE_NAME     = "graphFeederExchange"
	WORKER_QUEUE_NAME = "graphFeederWorkerQueue"
	TIME_FORMAT       = "2006-01-02T15:04:05.000Z"
)

type Consumer struct {
	conn    *amqp.Connection
	channel *amqp.Channel
}

type Message struct {
	Event   string                   `json:"event"`
	Payload []map[string]interface{} `json:"payload"`
}

func main() {
	startConsuming()
}

//here, mapping of decoded json
func jsonDecode(data string) (*Message, error) {
	source := &Message{}
	err := json.Unmarshal([]byte(data), &source)
	if err != nil {
		return source, err
	}

	return source, nil
}

func startConsuming() {

	c := &Consumer{
		conn:    nil,
		channel: nil,
	}

	c.conn = amqputil.CreateConnection("neo4jFeeding")
	c.channel = amqputil.CreateChannel(c.conn)

	err := c.channel.ExchangeDeclare(EXCHANGE_NAME, "fanout", true, false, false, false, nil)
	if err != nil {
		fmt.Println("exchange.declare: %s", err)
		panic(err)
	}

	//name, durable, autoDelete, exclusive, noWait, args Table
	if _, err := c.channel.QueueDeclare(WORKER_QUEUE_NAME, true, false, false, false, nil); err != nil {
		fmt.Println("queue.declare: %s", err)
		panic(err)
	}

	if err := c.channel.QueueBind(WORKER_QUEUE_NAME, "" /* binding key */, EXCHANGE_NAME, false, nil); err != nil {
		fmt.Println("queue.bind: %s", err)
		panic(err)
	}

	//(queue, consumer string, autoAck, exclusive, noLocal, noWait bool, args Table) (<-chan Delivery, error) {
	relationshipEvent, err := c.channel.Consume(WORKER_QUEUE_NAME, "neo4jFeeding", true, false, false, false, nil)
	if err != nil {
		fmt.Println("basic.consume: %s", err)
		panic(err)
	}

	fmt.Println("Neo4J Feeder worker started")

	for msg := range relationshipEvent {
		body := fmt.Sprintf("%s", msg.Body)

		message, err := jsonDecode(body)
		if err != nil {
			fmt.Println("Wrong message format", err, body)

			continue
		}

		if len(message.Payload) < 1 {
			fmt.Println("Wrong message format; payload should be an Array", message)

			continue
		}
		data := message.Payload[0]

		fmt.Println(message.Event)
		if message.Event == "RelationshipSaved" {
			createNode(data)
		} else if message.Event == "RelationshipRemoved" {
			deleteRelationship(data)
		} else if message.Event == "updateInstance" {
			updateNode(data)
		} else if message.Event == "RemovedFromCollection" {
			deleteNode(data)
		} else {
			fmt.Println(message.Event)
		}
	}
}

func checkIfEligible(sourceName, targetName string) bool {
	notAllowedSuffixes := []string{
		"Bucket",
		"BucketActivity",
	}

	for _, name := range neo4j.NotAllowedNames {
		if name == sourceName {
			fmt.Println("not eligible " + sourceName)
			return false
		}

		if name == targetName {
			fmt.Println("not eligible " + targetName)
			return false
		}
	}

	for _, name := range notAllowedSuffixes {

		if strings.HasSuffix(sourceName, name) {
			fmt.Println("not eligible " + sourceName)
			return false
		}

		if strings.HasSuffix(targetName, name) {
			fmt.Println("not eligible " + targetName)
			return false
		}

	}

	return true
}

func createNode(data map[string]interface{}) {

	sourceId := fmt.Sprintf("%s", data["sourceId"])
	sourceName := fmt.Sprintf("%s", data["sourceName"])

	targetId := fmt.Sprintf("%s", data["targetId"])
	targetName := fmt.Sprintf("%s", data["targetName"])

	if !checkIfEligible(sourceName, targetName) {
		return
	}

	sourceContent, err := mongo.FetchContent(bson.ObjectIdHex(sourceId), sourceName)
	if err != nil {
		fmt.Println("sourceContent", err)
		return
	}
	sourceNode := neo4j.CreateUniqueNode(sourceId, sourceName)
	neo4j.UpdateNode(sourceId, sourceContent)

	targetContent, err := mongo.FetchContent(bson.ObjectIdHex(targetId), targetName)
	if err != nil {
		fmt.Println("targetContent", err)
		return
	}
	targetNode := neo4j.CreateUniqueNode(targetId, targetName)
	neo4j.UpdateNode(targetId, targetContent)

	source := fmt.Sprintf("%s", sourceNode["create_relationship"])
	target := fmt.Sprintf("%s", targetNode["self"])

	if _, ok := data["as"]; !ok {
		fmt.Println("as value is not set on this relationship. Discarding this record", data)
		return
	}
	as := fmt.Sprintf("%s", data["as"])

	if _, ok := data["_id"]; !ok {
		fmt.Println("id value is not set on this relationship. Discarding this record", data)
		return
	}

	createdAt := getCreatedAtDate(data)
	relationshipData := fmt.Sprintf(`{"createdAt" : "%s", "createdAtEpoch" : %d }`, createdAt.Format(TIME_FORMAT), createdAt.Unix())
	neo4j.CreateRelationshipWithData(as, source, target, relationshipData)
}

func getCreatedAtDate(data map[string]interface{}) time.Time {

	if _, ok := data["timestamp"]; ok {
		t, err := time.Parse(TIME_FORMAT, data["timestamp"].(string))
		// if error doesnt exists, return createdAt
		if err == nil {
			return t.UTC()
		}
	}

	id := fmt.Sprintf("%s", data["_id"])
	if bson.IsObjectIdHex(id) {
		return bson.ObjectIdHex(id).Time().UTC()
	}

	fmt.Print("Couldnt determine the createdAt time, returning Now() as creatdAt")
	return time.Now().UTC()
}

func deleteNode(data map[string]interface{}) {

	if _, ok := data["_id"]; !ok {
		return
	}
	id := fmt.Sprintf("%s", data["_id"])
	neo4j.DeleteNode(id)
}

func deleteRelationship(data map[string]interface{}) {
	sourceId := fmt.Sprintf("%s", data["sourceId"])
	targetId := fmt.Sprintf("%s", data["targetId"])
	as := fmt.Sprintf("%s", data["as"])

	// we are not doing anything with result for now
	// do not pollute console
	neo4j.DeleteRelationship(sourceId, targetId, as)
	//result := neo4j.DeleteRelationship(sourceId, targetId, as)
	//if result {
	//	fmt.Println("Relationship deleted")
	//} else {
	//	fmt.Println("Relationship couldnt be deleted")
	//}
}

func updateNode(data map[string]interface{}) {

	if _, ok := data["bongo_"]; !ok {
		return
	}
	if _, ok := data["data"]; !ok {
		return
	}

	bongo := data["bongo_"].(map[string]interface{})
	obj := data["data"].(map[string]interface{})

	sourceId := fmt.Sprintf("%s", obj["_id"])
	sourceName := fmt.Sprintf("%s", bongo["constructorName"])

	if !checkIfEligible(sourceName, "") {
		return
	}

	sourceContent, err := mongo.FetchContent(bson.ObjectIdHex(sourceId), sourceName)
	if err != nil {
		fmt.Println("sourceContent", err)
		return
	}

	neo4j.CreateUniqueNode(sourceId, sourceName)
	neo4j.UpdateNode(sourceId, sourceContent)
}
<|endoftext|>


<commit_before>package util

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"time"

	"github.com/getlantern/fronted"
	"github.com/getlantern/golog"
	"github.com/getlantern/keyman"
	"github.com/getlantern/waitforserver"
)

const (
	defaultAddr = "127.0.0.1:8787"
)

var (
	log = golog.LoggerFor("flashlight.util")

	// This is for doing direct domain fronting if necessary. We store this as
	// an instance variable because it caches TLS session configs.
	direct = fronted.NewDirect()
)

// HTTPFetcher is a simple interface for types that are able to fetch data over HTTP.
type HTTPFetcher interface {
	Do(req *http.Request) (*http.Response, error)
}

func success(resp *http.Response) bool {
	return resp.StatusCode > 199 && resp.StatusCode < 400
}

// NewChainedAndFronted creates a new struct for accessing resources using chained
// and direct fronted servers in parallel.
func NewChainedAndFronted() *chainedAndFronted {
	cf := &chainedAndFronted{}
	cf.fetcher = &dualFetcher{cf}
	return cf
}

// ChainedAndFronted fetches HTTP data in parallel using both chained and fronted
// servers.
type chainedAndFronted struct {
	fetcher HTTPFetcher
}

// Do will attempt to execute the specified HTTP request using only a chained fetcher
func (cf *chainedAndFronted) Do(req *http.Request) (*http.Response, error) {
	resp, err := cf.fetcher.Do(req)
	if err != nil {
		// If there's an error, switch back to using the dual fetcher.
		cf.fetcher = &dualFetcher{cf}
	} else if !success(resp) {
		cf.fetcher = &dualFetcher{cf}
	}
	return resp, err
}

type chainedFetcher struct {
}

// Do will attempt to execute the specified HTTP request using only a chained fetcher
func (cf *chainedFetcher) Do(req *http.Request) (*http.Response, error) {
	log.Debugf("Using chained fronter")
	if client, err := HTTPClient("", defaultAddr); err != nil {
		log.Errorf("Could not create HTTP client: %v", err)
		return nil, err
	} else {
		return client.Do(req)
	}
}

type dualFetcher struct {
	cf *chainedAndFronted
}

// Do will attempt to execute the specified HTTP request using both
// chained and fronted servers, simply returning the first response to
// arrive. Callers MUST use the Lantern-Fronted-URL HTTP header to
// specify the fronted URL to use.
func (df *dualFetcher) Do(req *http.Request) (*http.Response, error) {
	log.Debugf("Using dual fronter")
	frontedUrl := req.Header.Get("Lantern-Fronted-URL")
	req.Header.Del("Lantern-Fronted-URL")

	if frontedUrl == "" {
		return nil, errors.New("Callers MUST specify the fronted URL in the Lantern-Fronted-URL header")
	}
	responses := make(chan *http.Response, 2)
	errs := make(chan error, 2)

	request := func(client HTTPFetcher, req *http.Request) error {
		if resp, err := client.Do(req); err != nil {
			log.Errorf("Could not complete request with: %v, %v", frontedUrl, err)
			errs <- err
			return err
		} else {
			if success(resp) {
				log.Debugf("Got successful HTTP call!")
				responses <- resp
				return nil
			} else {
				// If the local proxy can't connect to any upstread proxies, for example,
				// it will return a 502.
				err := fmt.Errorf("Bad response code: %v", resp.StatusCode)
				errs <- err
				return err
			}
		}
	}

	go func() {
		if req, err := http.NewRequest("GET", frontedUrl, nil); err != nil {
			log.Errorf("Could not create request for: %v, %v", frontedUrl, err)
			errs <- err
		} else {
			log.Debug("Sending request via DDF")
			if err := request(direct, req); err != nil {
				log.Errorf("Fronted request failed: %v", err)
			} else {
				log.Debug("Fronted request succeeded")
			}
		}
	}()
	go func() {
		if client, err := HTTPClient("", defaultAddr); err != nil {
			log.Errorf("Could not create HTTP client: %v", err)
			errs <- err
		} else {
			log.Debug("Sending chained request")
			if err := request(client, req); err != nil {
				log.Errorf("Chained request failed %v", err)
			} else {
				log.Debug("Switching to chained fronter for future requests since it succeeded")
				df.cf.fetcher = &chainedFetcher{}
			}
		}
	}()

	// Create channels for the final response or error. The response channel will be filled
	// in the case of any successful response as well as a non-error response for the second
	// response received. The error channel will only be filled if the first response is
	// unsuccessful and the second is an error.
	finalResponseCh := make(chan *http.Response, 1)
	finalErrorCh := make(chan error, 1)

	go readResponses(finalResponseCh, responses, finalErrorCh, errs)

	select {
	case resp := <-finalResponseCh:
		return resp, nil
	case err := <-finalErrorCh:
		return nil, err
	}
}

func readResponses(finalResponse chan *http.Response, responses chan *http.Response, finalErr chan error, errs chan error) {
	select {
	case resp := <-responses:
		if success(resp) {
			log.Debug("Got good first response")
			finalResponse <- resp

			// Just ignore the second response, but still process it.
			select {
			case <-responses:
				log.Debug("Closing second response body")
				_ = resp.Body.Close()
				return
			case <-errs:
				log.Debug("Ignoring error on second response")
				return
			}
		} else {
			log.Debugf("Got bad first response -- wait for second")
			_ = resp.Body.Close()
			// Just use whatever we get from the second response.
			select {
			case resp := <-responses:
				finalResponse <- resp
			case err := <-errs:
				finalErr <- err
			}
		}
	case err := <-errs:
		log.Debugf("Got an error: %v", err)
		// Just use whatever we get from the second response.
		select {
		case resp := <-responses:
			finalResponse <- resp
		case err := <-errs:
			finalErr <- err
		}
	}
}

// PersistentHTTPClient creates an http.Client that persists across requests.
// If rootCA is specified, the client will validate the server's certificate
// on TLS connections against that RootCA. If proxyAddr is specified, the client
// will proxy through the given http proxy.
func PersistentHTTPClient(rootCA string, proxyAddr string) (*http.Client, error) {
	return httpClient(rootCA, proxyAddr, true)
}

// HTTPClient creates an http.Client. If rootCA is specified, the client will
// validate the server's certificate on TLS connections against that RootCA. If
// proxyAddr is specified, the client will proxy through the given http proxy.
func HTTPClient(rootCA string, proxyAddr string) (*http.Client, error) {
	return httpClient(rootCA, proxyAddr, false)
}

// httpClient creates an http.Client. If rootCA is specified, the client will
// validate the server's certificate on TLS connections against that RootCA. If
// proxyAddr is specified, the client will proxy through the given http proxy.
func httpClient(rootCA string, proxyAddr string, persistent bool) (*http.Client, error) {

	log.Debugf("Creating new HTTPClient with proxy: %v", proxyAddr)

	tr := &http.Transport{
		Dial: (&net.Dialer{
			Timeout:   60 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,

		// This method is typically used for creating a one-off HTTP client
		// that we don't want to keep around for future calls, making
		// persistent connections a potential source of file descriptor
		// leaks. Note the name of this variable is misleading -- it would
		// be clearer to call it DisablePersistentConnections -- i.e. it has
		// nothing to do with TCP keep alives along the lines of the KeepAlive
		// variable in net.Dialer.
		DisableKeepAlives: !persistent,
	}

	if rootCA != "" {
		caCert, err := keyman.LoadCertificateFromPEMBytes([]byte(rootCA))
		if err != nil {
			return nil, fmt.Errorf("Unable to decode rootCA: %s", err)
		}
		tr.TLSClientConfig = &tls.Config{
			RootCAs: caCert.PoolContainingCert(),
		}
	}

	if proxyAddr != "" {
		host, _, err := net.SplitHostPort(proxyAddr)
		if err != nil {
			return nil, fmt.Errorf("Unable to split host and port for %v: %v", proxyAddr, err)
		}

		noHostSpecified := host == ""
		if noHostSpecified {
			// For addresses of the form ":8080", prepend the loopback IP
			host = "127.0.0.1"
			proxyAddr = host + proxyAddr
		}

		log.Debugf("Waiting for proxy server to came online...")
		// Waiting for proxy server to came online.
		if err := waitforserver.WaitForServer("tcp", proxyAddr, 60*time.Second); err != nil {
			// Instead of finishing here we just log the error and continue, the client
			// we are going to create will surely fail when used and return errors,
			// those errors should be handled by the code that depends on such client.
			log.Errorf("Proxy never came online at %v: %q", proxyAddr, err)
		}
		log.Debugf("Connected to proxy")

		tr.Proxy = func(req *http.Request) (*url.URL, error) {
			return url.Parse("http://" + proxyAddr)
		}
	} else {
		log.Errorf("Using direct http client with no proxyAddr")
	}
	return &http.Client{Transport: tr}, nil
}
<commit_msg>Close correct response body<commit_after>package util

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"time"

	"github.com/getlantern/fronted"
	"github.com/getlantern/golog"
	"github.com/getlantern/keyman"
	"github.com/getlantern/waitforserver"
)

const (
	defaultAddr = "127.0.0.1:8787"
)

var (
	log = golog.LoggerFor("flashlight.util")

	// This is for doing direct domain fronting if necessary. We store this as
	// an instance variable because it caches TLS session configs.
	direct = fronted.NewDirect()
)

// HTTPFetcher is a simple interface for types that are able to fetch data over HTTP.
type HTTPFetcher interface {
	Do(req *http.Request) (*http.Response, error)
}

func success(resp *http.Response) bool {
	return resp.StatusCode > 199 && resp.StatusCode < 400
}

// NewChainedAndFronted creates a new struct for accessing resources using chained
// and direct fronted servers in parallel.
func NewChainedAndFronted() *chainedAndFronted {
	cf := &chainedAndFronted{}
	cf.fetcher = &dualFetcher{cf}
	return cf
}

// ChainedAndFronted fetches HTTP data in parallel using both chained and fronted
// servers.
type chainedAndFronted struct {
	fetcher HTTPFetcher
}

// Do will attempt to execute the specified HTTP request using only a chained fetcher
func (cf *chainedAndFronted) Do(req *http.Request) (*http.Response, error) {
	resp, err := cf.fetcher.Do(req)
	if err != nil {
		// If there's an error, switch back to using the dual fetcher.
		cf.fetcher = &dualFetcher{cf}
	} else if !success(resp) {
		cf.fetcher = &dualFetcher{cf}
	}
	return resp, err
}

type chainedFetcher struct {
}

// Do will attempt to execute the specified HTTP request using only a chained fetcher
func (cf *chainedFetcher) Do(req *http.Request) (*http.Response, error) {
	log.Debugf("Using chained fronter")
	if client, err := HTTPClient("", defaultAddr); err != nil {
		log.Errorf("Could not create HTTP client: %v", err)
		return nil, err
	} else {
		return client.Do(req)
	}
}

type dualFetcher struct {
	cf *chainedAndFronted
}

// Do will attempt to execute the specified HTTP request using both
// chained and fronted servers, simply returning the first response to
// arrive. Callers MUST use the Lantern-Fronted-URL HTTP header to
// specify the fronted URL to use.
func (df *dualFetcher) Do(req *http.Request) (*http.Response, error) {
	log.Debugf("Using dual fronter")
	frontedUrl := req.Header.Get("Lantern-Fronted-URL")
	req.Header.Del("Lantern-Fronted-URL")

	if frontedUrl == "" {
		return nil, errors.New("Callers MUST specify the fronted URL in the Lantern-Fronted-URL header")
	}
	responses := make(chan *http.Response, 2)
	errs := make(chan error, 2)

	request := func(client HTTPFetcher, req *http.Request) error {
		if resp, err := client.Do(req); err != nil {
			log.Errorf("Could not complete request with: %v, %v", frontedUrl, err)
			errs <- err
			return err
		} else {
			if success(resp) {
				log.Debugf("Got successful HTTP call!")
				responses <- resp
				return nil
			} else {
				// If the local proxy can't connect to any upstread proxies, for example,
				// it will return a 502.
				err := fmt.Errorf("Bad response code: %v", resp.StatusCode)
				errs <- err
				return err
			}
		}
	}

	go func() {
		if req, err := http.NewRequest("GET", frontedUrl, nil); err != nil {
			log.Errorf("Could not create request for: %v, %v", frontedUrl, err)
			errs <- err
		} else {
			log.Debug("Sending request via DDF")
			if err := request(direct, req); err != nil {
				log.Errorf("Fronted request failed: %v", err)
			} else {
				log.Debug("Fronted request succeeded")
			}
		}
	}()
	go func() {
		if client, err := HTTPClient("", defaultAddr); err != nil {
			log.Errorf("Could not create HTTP client: %v", err)
			errs <- err
		} else {
			log.Debug("Sending chained request")
			if err := request(client, req); err != nil {
				log.Errorf("Chained request failed %v", err)
			} else {
				log.Debug("Switching to chained fronter for future requests since it succeeded")
				df.cf.fetcher = &chainedFetcher{}
			}
		}
	}()

	// Create channels for the final response or error. The response channel will be filled
	// in the case of any successful response as well as a non-error response for the second
	// response received. The error channel will only be filled if the first response is
	// unsuccessful and the second is an error.
	finalResponseCh := make(chan *http.Response, 1)
	finalErrorCh := make(chan error, 1)

	go readResponses(finalResponseCh, responses, finalErrorCh, errs)

	select {
	case resp := <-finalResponseCh:
		return resp, nil
	case err := <-finalErrorCh:
		return nil, err
	}
}

func readResponses(finalResponse chan *http.Response, responses chan *http.Response, finalErr chan error, errs chan error) {
	select {
	case resp := <-responses:
		if success(resp) {
			log.Debug("Got good first response")
			finalResponse <- resp

			// Just ignore the second response, but still process it.
			select {
			case resp := <-responses:
				log.Debug("Closing second response body")
				_ = resp.Body.Close()
				return
			case <-errs:
				log.Debug("Ignoring error on second response")
				return
			}
		} else {
			log.Debugf("Got bad first response -- wait for second")
			_ = resp.Body.Close()
			// Just use whatever we get from the second response.
			select {
			case resp := <-responses:
				finalResponse <- resp
			case err := <-errs:
				finalErr <- err
			}
		}
	case err := <-errs:
		log.Debugf("Got an error: %v", err)
		// Just use whatever we get from the second response.
		select {
		case resp := <-responses:
			finalResponse <- resp
		case err := <-errs:
			finalErr <- err
		}
	}
}

// PersistentHTTPClient creates an http.Client that persists across requests.
// If rootCA is specified, the client will validate the server's certificate
// on TLS connections against that RootCA. If proxyAddr is specified, the client
// will proxy through the given http proxy.
func PersistentHTTPClient(rootCA string, proxyAddr string) (*http.Client, error) {
	return httpClient(rootCA, proxyAddr, true)
}

// HTTPClient creates an http.Client. If rootCA is specified, the client will
// validate the server's certificate on TLS connections against that RootCA. If
// proxyAddr is specified, the client will proxy through the given http proxy.
func HTTPClient(rootCA string, proxyAddr string) (*http.Client, error) {
	return httpClient(rootCA, proxyAddr, false)
}

// httpClient creates an http.Client. If rootCA is specified, the client will
// validate the server's certificate on TLS connections against that RootCA. If
// proxyAddr is specified, the client will proxy through the given http proxy.
func httpClient(rootCA string, proxyAddr string, persistent bool) (*http.Client, error) {

	log.Debugf("Creating new HTTPClient with proxy: %v", proxyAddr)

	tr := &http.Transport{
		Dial: (&net.Dialer{
			Timeout:   60 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,

		// This method is typically used for creating a one-off HTTP client
		// that we don't want to keep around for future calls, making
		// persistent connections a potential source of file descriptor
		// leaks. Note the name of this variable is misleading -- it would
		// be clearer to call it DisablePersistentConnections -- i.e. it has
		// nothing to do with TCP keep alives along the lines of the KeepAlive
		// variable in net.Dialer.
		DisableKeepAlives: !persistent,
	}

	if rootCA != "" {
		caCert, err := keyman.LoadCertificateFromPEMBytes([]byte(rootCA))
		if err != nil {
			return nil, fmt.Errorf("Unable to decode rootCA: %s", err)
		}
		tr.TLSClientConfig = &tls.Config{
			RootCAs: caCert.PoolContainingCert(),
		}
	}

	if proxyAddr != "" {
		host, _, err := net.SplitHostPort(proxyAddr)
		if err != nil {
			return nil, fmt.Errorf("Unable to split host and port for %v: %v", proxyAddr, err)
		}

		noHostSpecified := host == ""
		if noHostSpecified {
			// For addresses of the form ":8080", prepend the loopback IP
			host = "127.0.0.1"
			proxyAddr = host + proxyAddr
		}

		log.Debugf("Waiting for proxy server to came online...")
		// Waiting for proxy server to came online.
		if err := waitforserver.WaitForServer("tcp", proxyAddr, 60*time.Second); err != nil {
			// Instead of finishing here we just log the error and continue, the client
			// we are going to create will surely fail when used and return errors,
			// those errors should be handled by the code that depends on such client.
			log.Errorf("Proxy never came online at %v: %q", proxyAddr, err)
		}
		log.Debugf("Connected to proxy")

		tr.Proxy = func(req *http.Request) (*url.URL, error) {
			return url.Parse("http://" + proxyAddr)
		}
	} else {
		log.Errorf("Using direct http client with no proxyAddr")
	}
	return &http.Client{Transport: tr}, nil
}
<|endoftext|>


<commit_before>//
// Copyright (C) 2015-present  Laszlo Csontos
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>. */
//

package catalog

import (
	"fmt"
	"time"
)

type VideoType int

type InvalidVideoTypeNameError struct {
	VideoTypeName string
}

type VideoKey struct {
	VideoId   string
	VideoType VideoType
}

type LongVideoUrl struct {
	Id int64
	*VideoKey

	CreatedAt   time.Time
	PublishedAt time.Time

	NormalizedTitle string
	UrlPath         string
}

type VideoRecord struct {
	*VideoKey

	PublishedAt time.Time

	Title string
}

type NoSuchVideoError struct {
	VideoId string
}

type VideoCatalog interface {
	SearchByID(videoId string) (*VideoRecord, error)
	SearchByTitle(title string, maxResults int64) ([]VideoRecord, error)
}

type VideoCatalogFactory interface {
	NewCatalog(args interface{}) VideoCatalog
}

const (
	YouTube VideoType = (iota)

	// Reserved for future implementation
	Vimeo
	Youku
	Rutube

	// Internal use only!
	Unknown = -1
)

var videoTypesLookupMap = make(map[string]VideoType)

var videoTypesStringMap = map[VideoType]string{
	YouTube: "YouTube",
	Vimeo:   "Vimeo",
	Youku:   "Youku",
	Rutube:  "Rutube",
}

func (err *InvalidVideoTypeNameError) Error() string {
	return fmt.Sprintf("\"%s\" is an invalid video name", err.VideoTypeName)
}

func (err *NoSuchVideoError) Error() string {
	return fmt.Sprintf("Video with Id %s does not exist", err.VideoId)
}

func GetVideoTypeByName(videoTypeName string) (VideoType, error) {
	if videoType, ok := videoTypesLookupMap[videoTypeName]; !ok {
		return Unknown, &InvalidVideoTypeNameError{videoTypeName}
	} else {
		return videoType, nil
	}
}

func NewLongVideoUrl(videoRecord *VideoRecord) *LongVideoUrl {
	return &LongVideoUrl{
		VideoKey:    videoRecord.VideoKey,
		CreatedAt:   time.Now(),
		PublishedAt: videoRecord.PublishedAt,
	}
}

func NewVideoRecord(videoKey *VideoKey, publishedAt time.Time, title string) *VideoRecord {
	return &VideoRecord{
		VideoKey: videoKey, PublishedAt: publishedAt, Title: title}
}

func (url *LongVideoUrl) String() string {
	return ""
}

func (videoKey *VideoKey) String() string {
	return fmt.Sprintf("%s@%s", videoKey.VideoType, videoKey.VideoId)
}

func (videoType VideoType) String() string {
	return videoTypesStringMap[videoType]
}

func (videoRecord *VideoRecord) String() string {
	return fmt.Sprintf("[%v] %v: %v", videoRecord.VideoId, videoRecord.Title)
}

func init() {
	for videoType, videoTypeName := range videoTypesStringMap {
		videoTypesLookupMap[videoTypeName] = videoType
	}
}
<commit_msg>Issue #3: Fix: datastore: unsupported struct field type: *catalog.VideoKey<commit_after>//
// Copyright (C) 2015-present  Laszlo Csontos
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>. */
//

package catalog

import (
	"fmt"
	"time"
)

type VideoType int

type InvalidVideoTypeNameError struct {
	VideoTypeName string
}

type VideoKey struct {
	VideoId   string
	VideoType VideoType
}

type LongVideoUrl struct {
	Id        int64
	*VideoKey `datastore:"-"`

	CreatedAt   time.Time
	PublishedAt time.Time

	NormalizedTitle string
	UrlPath         string
}

type VideoRecord struct {
	*VideoKey

	PublishedAt time.Time

	Title string
}

type NoSuchVideoError struct {
	VideoId string
}

type VideoCatalog interface {
	SearchByID(videoId string) (*VideoRecord, error)
	SearchByTitle(title string, maxResults int64) ([]VideoRecord, error)
}

type VideoCatalogFactory interface {
	NewCatalog(args interface{}) VideoCatalog
}

const (
	YouTube VideoType = (iota)

	// Reserved for future implementation
	Vimeo
	Youku
	Rutube

	// Internal use only!
	Unknown = -1
)

var videoTypesLookupMap = make(map[string]VideoType)

var videoTypesStringMap = map[VideoType]string{
	YouTube: "YouTube",
	Vimeo:   "Vimeo",
	Youku:   "Youku",
	Rutube:  "Rutube",
}

func (err *InvalidVideoTypeNameError) Error() string {
	return fmt.Sprintf("\"%s\" is an invalid video name", err.VideoTypeName)
}

func (err *NoSuchVideoError) Error() string {
	return fmt.Sprintf("Video with Id %s does not exist", err.VideoId)
}

func GetVideoTypeByName(videoTypeName string) (VideoType, error) {
	if videoType, ok := videoTypesLookupMap[videoTypeName]; !ok {
		return Unknown, &InvalidVideoTypeNameError{videoTypeName}
	} else {
		return videoType, nil
	}
}

func NewLongVideoUrl(videoRecord *VideoRecord) *LongVideoUrl {
	return &LongVideoUrl{
		VideoKey:    videoRecord.VideoKey,
		CreatedAt:   time.Now(),
		PublishedAt: videoRecord.PublishedAt,
	}
}

func NewVideoRecord(videoKey *VideoKey, publishedAt time.Time, title string) *VideoRecord {
	return &VideoRecord{
		VideoKey: videoKey, PublishedAt: publishedAt, Title: title}
}

func (url *LongVideoUrl) String() string {
	return ""
}

func (videoKey *VideoKey) String() string {
	return fmt.Sprintf("%s@%s", videoKey.VideoType, videoKey.VideoId)
}

func (videoType VideoType) String() string {
	return videoTypesStringMap[videoType]
}

func (videoRecord *VideoRecord) String() string {
	return fmt.Sprintf("[%v] %v: %v", videoRecord.VideoId, videoRecord.Title)
}

func init() {
	for videoType, videoTypeName := range videoTypesStringMap {
		videoTypesLookupMap[videoTypeName] = videoType
	}
}
<|endoftext|>


<commit_before>package horizon

import (
	"github.com/jagregory/halgo"
	"github.com/stellar/horizon/render/problem"
	"github.com/stellar/horizon/txsub"
	"net/http"
)

type ResultResource struct {
	txsub.Result
}

func (res *ResultResource) Error() error {
	switch err := res.Err.(type) {
	case *txsub.FailedTransactionError:
		// TODO: Fill detail
		return &problem.P{
			Type:   "transaction_failed",
			Title:  "Transaction Failed",
			Status: http.StatusBadRequest,
			Detail: "TODO",
			Extras: map[string]interface{}{
				"envelope_xdr": res.EnvelopeXDR,
				"result_xdr":   err.ResultXDR,
			},
		}
	case *txsub.MalformedTransactionError:
		// TODO: Fill detail
		return &problem.P{
			Type:   "transaction_malformed",
			Title:  "Transaction Malformed",
			Status: http.StatusBadRequest,
			Detail: "TODO",
			Extras: map[string]interface{}{
				"envelope_xdr": err.EnvelopeXDR,
			},
		}
	default:
		return err
	}
}

func (res *ResultResource) IsSuccess() bool {
	return res.Err == nil
}

func (res *ResultResource) Success() interface{} {
	return struct {
		Links  halgo.Links
		Hash   string `json:"hash"`
		Ledger int32  `json:"ledger"`
		Env    string `json:"envelope_xdr"`
		Result string `json:"result_xdr"`
		Meta   string `json:"result_meta_xdr"`
		// TODO: add result code details
	}{
		halgo.Links{}.Link("transaction", "/transactions/%s", res.Hash),
		res.Hash,
		res.LedgerSequence,
		res.EnvelopeXDR,
		res.ResultXDR,
		res.ResultMetaXDR,
	}

}
<commit_msg>Fix "_links nested in Links" bug.<commit_after>package horizon

import (
	"github.com/jagregory/halgo"
	"github.com/stellar/horizon/render/problem"
	"github.com/stellar/horizon/txsub"
	"net/http"
)

type ResultResource struct {
	txsub.Result
}

func (res *ResultResource) Error() error {
	switch err := res.Err.(type) {
	case *txsub.FailedTransactionError:
		// TODO: Fill detail
		return &problem.P{
			Type:   "transaction_failed",
			Title:  "Transaction Failed",
			Status: http.StatusBadRequest,
			Detail: "TODO",
			Extras: map[string]interface{}{
				"envelope_xdr": res.EnvelopeXDR,
				"result_xdr":   err.ResultXDR,
			},
		}
	case *txsub.MalformedTransactionError:
		// TODO: Fill detail
		return &problem.P{
			Type:   "transaction_malformed",
			Title:  "Transaction Malformed",
			Status: http.StatusBadRequest,
			Detail: "TODO",
			Extras: map[string]interface{}{
				"envelope_xdr": err.EnvelopeXDR,
			},
		}
	default:
		return err
	}
}

func (res *ResultResource) IsSuccess() bool {
	return res.Err == nil
}

func (res *ResultResource) Success() interface{} {
	return struct {
		halgo.Links
		Hash   string `json:"hash"`
		Ledger int32  `json:"ledger"`
		Env    string `json:"envelope_xdr"`
		Result string `json:"result_xdr"`
		Meta   string `json:"result_meta_xdr"`
		// TODO: add result code details
	}{
		halgo.Links{}.Link("transaction", "/transactions/%s", res.Hash),
		res.Hash,
		res.LedgerSequence,
		res.EnvelopeXDR,
		res.ResultXDR,
		res.ResultMetaXDR,
	}

}
<|endoftext|>


<commit_before>package main_test

import (
	"code.cloudfoundry.org/bbs/cmd/bbs/testrunner"
	"code.cloudfoundry.org/bbs/models"
	"code.cloudfoundry.org/bbs/models/test/model_helpers"
	"github.com/tedsuo/ifrit/ginkgomon"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Evacuation API", func() {
	BeforeEach(func() {
		bbsRunner = testrunner.New(bbsBinPath, bbsArgs)
		bbsProcess = ginkgomon.Invoke(bbsRunner)
	})

	var actual *models.ActualLRP

	BeforeEach(func() {
		actual = model_helpers.NewValidActualLRP("some-process-guid", 1)
		actual.State = models.ActualLRPStateRunning
		desiredLRP := model_helpers.NewValidDesiredLRP(actual.ProcessGuid)
		desiredLRP.Instances = 2
		err := client.DesireLRP(logger, desiredLRP)
		Expect(err).NotTo(HaveOccurred())
		err = client.ClaimActualLRP(logger, actual.ProcessGuid, 1, &actual.ActualLRPInstanceKey)
		Expect(err).NotTo(HaveOccurred())
	})

	Describe("RemoveEvacuatingActualLRP", func() {
		It("removes the evacuating actual_lrp", func() {
			_, err := client.EvacuateClaimedActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey)
			Expect(err).NotTo(HaveOccurred())

			err = client.RemoveEvacuatingActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey)
			Expect(err).NotTo(HaveOccurred())

			group, err := client.ActualLRPGroupByProcessGuidAndIndex(logger, actual.ProcessGuid, int(actual.Index))
			Expect(err).ToNot(HaveOccurred())
			Expect(group.Evacuating).To(BeNil())
		})
	})

	Describe("EvacuateClaimedActualLRP", func() {
		It("removes the claimed actual_lrp without evacuating", func() {
			keepContainer, evacuateErr := client.EvacuateClaimedActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey)
			Expect(keepContainer).To(BeFalse())
			Expect(evacuateErr).NotTo(HaveOccurred())

			actualLRPGroup, err := client.ActualLRPGroupByProcessGuidAndIndex(logger, actual.ProcessGuid, int(actual.Index))
			Expect(err).NotTo(HaveOccurred())
			Expect(actualLRPGroup.Evacuating).To(BeNil())
			Expect(actualLRPGroup.Instance).NotTo(BeNil())
			Expect(actualLRPGroup.Instance.State).To(Equal(models.ActualLRPStateUnclaimed))
		})
	})

	Describe("EvacuateRunningActualLRP", func() {
		BeforeEach(func() {
			err := client.StartActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey, &actual.ActualLRPNetInfo)
			Expect(err).NotTo(HaveOccurred())
		})

		It("runs the evacuating ActualLRP and unclaims the instance ActualLRP", func() {
			keepContainer, err := client.EvacuateRunningActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey, &actual.ActualLRPNetInfo, uint64(10000))
			Expect(keepContainer).To(BeTrue())
			Expect(err).NotTo(HaveOccurred())

			actualLRPGroup, err := client.ActualLRPGroupByProcessGuidAndIndex(logger, actual.ProcessGuid, int(actual.Index))
			Expect(err).NotTo(HaveOccurred())
			Expect(actualLRPGroup.Evacuating).NotTo(BeNil())
			Expect(actualLRPGroup.Instance).NotTo(BeNil())
			Expect(actualLRPGroup.Evacuating.State).To(Equal(models.ActualLRPStateRunning))
			Expect(actualLRPGroup.Instance.State).To(Equal(models.ActualLRPStateUnclaimed))
		})
	})

	Describe("EvacuateStoppedActualLRP", func() {
		BeforeEach(func() {
			err := client.StartActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey, &actual.ActualLRPNetInfo)
			Expect(err).NotTo(HaveOccurred())
		})

		It("deletes the container and both actualLRPs", func() {
			keepContainer, err := client.EvacuateStoppedActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey)
			Expect(keepContainer).To(BeFalse())
			Expect(err).NotTo(HaveOccurred())
			_, err = client.ActualLRPGroupByProcessGuidAndIndex(logger, actual.ProcessGuid, int(actual.Index))
			Expect(err).To(Equal(models.ErrResourceNotFound))
		})
	})

	Describe("EvacuateCrashedActualLRP", func() {
		BeforeEach(func() {
			err := client.StartActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey, &actual.ActualLRPNetInfo)
			Expect(err).NotTo(HaveOccurred())
		})

		It("removes the crashed evacuating LRP and unclaims the instance ActualLRP", func() {
			keepContainer, evacuateErr := client.EvacuateCrashedActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey, "some-reason")
			Expect(keepContainer).Should(BeFalse())
			Expect(evacuateErr).NotTo(HaveOccurred())

			actualLRPGroup, err := client.ActualLRPGroupByProcessGuidAndIndex(logger, actual.ProcessGuid, int(actual.Index))
			Expect(err).NotTo(HaveOccurred())
			Expect(actualLRPGroup.Evacuating).To(BeNil())
			Expect(actualLRPGroup.Instance).ToNot(BeNil())
			Expect(actualLRPGroup.Instance.State).To(Equal(models.ActualLRPStateUnclaimed))
		})
	})
})
<commit_msg>Evacuation tests now wait on DesireLRP and on availability of actual LRP groups.<commit_after>package main_test

import (
	"code.cloudfoundry.org/bbs/cmd/bbs/testrunner"
	"code.cloudfoundry.org/bbs/models"
	"code.cloudfoundry.org/bbs/models/test/model_helpers"
	"github.com/tedsuo/ifrit/ginkgomon"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Evacuation API", func() {
	BeforeEach(func() {
		bbsRunner = testrunner.New(bbsBinPath, bbsArgs)
		bbsProcess = ginkgomon.Invoke(bbsRunner)
	})

	var actual *models.ActualLRP

	BeforeEach(func() {
		actual = model_helpers.NewValidActualLRP("some-process-guid", 1)
		actual.State = models.ActualLRPStateRunning
		desiredLRP := model_helpers.NewValidDesiredLRP(actual.ProcessGuid)
		desiredLRP.Instances = 2

		Eventually(func() error {
			return client.DesireLRP(logger, desiredLRP)
		}).Should(BeNil())

		err := client.ClaimActualLRP(logger, actual.ProcessGuid, 1, &actual.ActualLRPInstanceKey)
		Expect(err).NotTo(HaveOccurred())

		Eventually(func() error {
			_, err = client.ActualLRPGroupByProcessGuidAndIndex(logger, actual.ProcessGuid, int(actual.Index))
			return err
		}).Should(BeNil())
	})

	Describe("RemoveEvacuatingActualLRP", func() {
		It("removes the evacuating actual_lrp", func() {
			_, err := client.EvacuateClaimedActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey)
			Expect(err).NotTo(HaveOccurred())

			err = client.RemoveEvacuatingActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey)
			Expect(err).NotTo(HaveOccurred())

			group, err := client.ActualLRPGroupByProcessGuidAndIndex(logger, actual.ProcessGuid, int(actual.Index))
			Expect(err).ToNot(HaveOccurred())
			Expect(group.Evacuating).To(BeNil())
		})
	})

	Describe("EvacuateClaimedActualLRP", func() {
		It("removes the claimed actual_lrp without evacuating", func() {
			keepContainer, evacuateErr := client.EvacuateClaimedActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey)
			Expect(keepContainer).To(BeFalse())
			Expect(evacuateErr).NotTo(HaveOccurred())

			actualLRPGroup, err := client.ActualLRPGroupByProcessGuidAndIndex(logger, actual.ProcessGuid, int(actual.Index))
			Expect(err).NotTo(HaveOccurred())
			Expect(actualLRPGroup.Evacuating).To(BeNil())
			Expect(actualLRPGroup.Instance).NotTo(BeNil())
			Expect(actualLRPGroup.Instance.State).To(Equal(models.ActualLRPStateUnclaimed))
		})
	})

	Describe("EvacuateRunningActualLRP", func() {
		BeforeEach(func() {
			err := client.StartActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey, &actual.ActualLRPNetInfo)
			Expect(err).NotTo(HaveOccurred())
		})

		It("runs the evacuating ActualLRP and unclaims the instance ActualLRP", func() {
			keepContainer, err := client.EvacuateRunningActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey, &actual.ActualLRPNetInfo, uint64(10000))
			Expect(keepContainer).To(BeTrue())
			Expect(err).NotTo(HaveOccurred())

			actualLRPGroup, err := client.ActualLRPGroupByProcessGuidAndIndex(logger, actual.ProcessGuid, int(actual.Index))
			Expect(err).NotTo(HaveOccurred())
			Expect(actualLRPGroup.Evacuating).NotTo(BeNil())
			Expect(actualLRPGroup.Instance).NotTo(BeNil())
			Expect(actualLRPGroup.Evacuating.State).To(Equal(models.ActualLRPStateRunning))
			Expect(actualLRPGroup.Instance.State).To(Equal(models.ActualLRPStateUnclaimed))
		})
	})

	Describe("EvacuateStoppedActualLRP", func() {
		BeforeEach(func() {
			err := client.StartActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey, &actual.ActualLRPNetInfo)
			Expect(err).NotTo(HaveOccurred())
		})

		It("deletes the container and both actualLRPs", func() {
			keepContainer, err := client.EvacuateStoppedActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey)
			Expect(keepContainer).To(BeFalse())
			Expect(err).NotTo(HaveOccurred())
			_, err = client.ActualLRPGroupByProcessGuidAndIndex(logger, actual.ProcessGuid, int(actual.Index))
			Expect(err).To(Equal(models.ErrResourceNotFound))
		})
	})

	Describe("EvacuateCrashedActualLRP", func() {
		BeforeEach(func() {
			err := client.StartActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey, &actual.ActualLRPNetInfo)
			Expect(err).NotTo(HaveOccurred())
		})

		It("removes the crashed evacuating LRP and unclaims the instance ActualLRP", func() {
			keepContainer, evacuateErr := client.EvacuateCrashedActualLRP(logger, &actual.ActualLRPKey, &actual.ActualLRPInstanceKey, "some-reason")
			Expect(keepContainer).Should(BeFalse())
			Expect(evacuateErr).NotTo(HaveOccurred())

			actualLRPGroup, err := client.ActualLRPGroupByProcessGuidAndIndex(logger, actual.ProcessGuid, int(actual.Index))
			Expect(err).NotTo(HaveOccurred())
			Expect(actualLRPGroup.Evacuating).To(BeNil())
			Expect(actualLRPGroup.Instance).ToNot(BeNil())
			Expect(actualLRPGroup.Instance.State).To(Equal(models.ActualLRPStateUnclaimed))
		})
	})
})
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"

	"github.com/flant/dapp/pkg/lock"
	"github.com/flant/dapp/pkg/ruby2go"

	"github.com/flant/kubedog/pkg/kube"
	"github.com/flant/kubedog/pkg/tracker"
	"github.com/flant/kubedog/pkg/trackers/rollout"
)

func main() {
	var err error

	err = lock.Init()
	if err != nil {
		panic(err)
	}

	ruby2go.RunCli("deploy-watcher", func(args map[string]interface{}) (interface{}, error) {
		initOpts := kube.InitOptions{KubeContext: args["kubeContext"].(string)}
		err = kube.Init(initOpts)
		if err != nil {
			panic(err)
		}

		namespace := args["namespace"].(string)
		if namespace == "" {
			return nil, fmt.Errorf("namespace argument required!")
		}

		resourceName := args["resourceName"].(string)
		if resourceName == "" {
			return nil, fmt.Errorf("resourceName argument required!")
		}

		timeout := args["timeout"].(float64)

		var logsFromTime time.Time
		if logsFromTimeOption, ok := args["logsFromTime"]; ok {
			logsFromTime, err = time.Parse(time.RFC3339, logsFromTimeOption.(string))
			if err != nil {
				return nil, err
			}
		} else {
			logsFromTime = time.Time{}
		}

		switch action := args["action"]; action {
		case "watch job":
			err := rollout.TrackJobTillDone(resourceName, namespace, kube.Kubernetes, tracker.Options{Timeout: time.Second * time.Duration(timeout), LogsFromTime: logsFromTime})
			if err != nil {
				return nil, fmt.Errorf("error tracking job `%s` in namespace `%s`: %s", resourceName, namespace, err)
			}
		case "watch deployment":
			err := rollout.TrackDeploymentTillReady(resourceName, namespace, kube.Kubernetes, tracker.Options{Timeout: time.Second * time.Duration(timeout), LogsFromTime: logsFromTime})
			if err != nil {
				return nil, fmt.Errorf("error tracking deployment `%s` in namespace `%s`: %s", resourceName, namespace, err)
			}
		default:
			return nil, fmt.Errorf("unknown action \"%s\"", action)
		}

		return nil, nil
	})
}
<commit_msg>Fix kube-context not specified ruby2go error<commit_after>package main

import (
	"fmt"
	"time"

	"github.com/flant/dapp/pkg/lock"
	"github.com/flant/dapp/pkg/ruby2go"

	"github.com/flant/kubedog/pkg/kube"
	"github.com/flant/kubedog/pkg/tracker"
	"github.com/flant/kubedog/pkg/trackers/rollout"
)

func main() {
	var err error

	err = lock.Init()
	if err != nil {
		panic(err)
	}

	ruby2go.RunCli("deploy-watcher", func(args map[string]interface{}) (interface{}, error) {
		initOpts := kube.InitOptions{}
		if kubeContext, ok := args["kubeContext"]; ok && kubeContext != nil {
			initOpts.KubeContext = kubeContext.(string)
		}

		err = kube.Init(initOpts)
		if err != nil {
			panic(err)
		}

		namespace := args["namespace"].(string)
		if namespace == "" {
			return nil, fmt.Errorf("namespace argument required!")
		}

		resourceName := args["resourceName"].(string)
		if resourceName == "" {
			return nil, fmt.Errorf("resourceName argument required!")
		}

		timeout := args["timeout"].(float64)

		var logsFromTime time.Time
		if logsFromTimeOption, ok := args["logsFromTime"]; ok && logsFromTimeOption != nil {
			logsFromTime, err = time.Parse(time.RFC3339, logsFromTimeOption.(string))
			if err != nil {
				return nil, err
			}
		} else {
			logsFromTime = time.Time{}
		}

		switch action := args["action"]; action {
		case "watch job":
			err := rollout.TrackJobTillDone(resourceName, namespace, kube.Kubernetes, tracker.Options{Timeout: time.Second * time.Duration(timeout), LogsFromTime: logsFromTime})
			if err != nil {
				return nil, fmt.Errorf("error tracking job `%s` in namespace `%s`: %s", resourceName, namespace, err)
			}
		case "watch deployment":
			err := rollout.TrackDeploymentTillReady(resourceName, namespace, kube.Kubernetes, tracker.Options{Timeout: time.Second * time.Duration(timeout), LogsFromTime: logsFromTime})
			if err != nil {
				return nil, fmt.Errorf("error tracking deployment `%s` in namespace `%s`: %s", resourceName, namespace, err)
			}
		default:
			return nil, fmt.Errorf("unknown action \"%s\"", action)
		}

		return nil, nil
	})
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"reflect"
	"strings"

	"github.com/brianvoe/gofakeit/v5"
)

var port string

func init() {
	flag.StringVar(&port, "port", "8080", "server port")
}

func main() {
	gofakeit.Seed(0)

	// Set router
	mux := http.NewServeMux()
	routes(mux)

	fmt.Println("Running on port " + port)

	http.ListenAndServe(":"+port, mux)
}

func routes(mux *http.ServeMux) {
	mux.HandleFunc("/favicon.ico", favicon)
	mux.HandleFunc("/list", list)
	mux.HandleFunc("/", lookup)
}

func favicon(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "image/x-icon")
	w.WriteHeader(http.StatusOK)
	w.Write(gofakeit.ImagePng(32, 32))
}

func list(w http.ResponseWriter, r *http.Request) {
	ok(w, gofakeit.FuncLookups)
}

func lookup(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case http.MethodGet:
		lookupGet(w, r)
		return
	case http.MethodPost:
		lookupPost(w, r)
		return
	}

	badrequest(w, "Only Get and Post methods allowed")
}

func lookupGet(w http.ResponseWriter, r *http.Request) {
	// Check url query for field(func) and
	info, err := getInfoFromPath(r)
	if err != nil {
		badrequest(w, err.Error())
		return
	}

	// Get url params
	var mapString map[string][]string
	urlParams := r.URL.Query()
	for key, values := range urlParams {
		if mapString == nil {
			mapString = make(map[string][]string)
		}
		mapString[key] = values
	}

	// Call method to generate requested data
	data, err := info.Call(&mapString, info)
	if err != nil {
		badrequest(w, err.Error())
		return
	}

	ok(w, data)
}

func lookupPost(w http.ResponseWriter, r *http.Request) {
	// Expect content type json
	contentType := r.Header.Get("Content-Type")
	if contentType != "application/json" {
		badrequest(w, `Expects header to be of Content-Type "application/json"`)
		return
	}

	// Check url query for field(func) and
	info, err := getInfoFromPath(r)
	if err != nil {
		badrequest(w, err.Error())
		return
	}

	var mapString map[string][]string

	// Try to decode body if params are needed
	if len(info.Params) > 0 {
		mapString = make(map[string][]string)
		mapInterface := map[string]interface{}{}
		err = json.NewDecoder(r.Body).Decode(&mapInterface)
		if err != nil {
			badrequest(w, "Could not parse post body. Expects key(string) - value or []value")
			return
		}
		defer r.Body.Close()

		for key, value := range mapInterface {
			v := reflect.ValueOf(value)
			switch v.Kind() {
			case reflect.Bool:
				mapString[key] = []string{fmt.Sprintf("%v", value)}
			case reflect.Int, reflect.Int8, reflect.Int32, reflect.Int64:
				mapString[key] = []string{fmt.Sprintf("%v", value)}
			case reflect.Uint, reflect.Uint8, reflect.Uint32, reflect.Uint64:
				mapString[key] = []string{fmt.Sprintf("%v", value)}
			case reflect.Float32, reflect.Float64:
				mapString[key] = []string{fmt.Sprintf("%v", value)}
			case reflect.String:
				mapString[key] = []string{fmt.Sprintf("%v", value)}
			case reflect.Map:
				mapString[key] = []string{fmt.Sprintf("%v", value)}
			case reflect.Slice:
				var vals []string
				for i := 0; i < v.Len(); i++ {
					vals = append(vals, fmt.Sprintf("%v", v.Index(i).Interface()))
				}

				mapString[key] = vals
			}
		}
	}

	// Call method to generate requested data
	data, err := info.Call(&mapString, info)
	if err != nil {
		badrequest(w, err.Error())
		return
	}

	ok(w, data)
}

func getInfoFromPath(r *http.Request) (*gofakeit.Info, error) {
	path := r.URL.Path
	path = strings.Trim(path, "/")
	paths := strings.Split(path, "/")
	if len(paths) == 0 {
		return nil, errors.New("No function was called, please pass func parameter")
	}

	// Lookup fake data method
	info := gofakeit.GetLookupData(paths[0])
	if info == nil {
		return nil, errors.New("No function was called, please pass func parameter")
	}

	return info, nil
}

func encodeResponse(v interface{}) []byte {
	// Set new bytes buffer
	buf := &bytes.Buffer{}
	enc := json.NewEncoder(buf)
	enc.SetEscapeHTML(true)
	enc.Encode(v)
	return buf.Bytes()
}

func ok(w http.ResponseWriter, data interface{}) {
	var resp []byte
	d := reflect.ValueOf(data)
	switch d.Kind() {
	case reflect.Bool:
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		resp = []byte(fmt.Sprintf("%v", data))
	case reflect.Int, reflect.Int8, reflect.Int32, reflect.Int64:
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		resp = []byte(fmt.Sprintf("%v", data))
	case reflect.Uint, reflect.Uint8, reflect.Uint32, reflect.Uint64:
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		resp = []byte(fmt.Sprintf("%v", data))
	case reflect.Float32, reflect.Float64:
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		resp = []byte(fmt.Sprintf("%v", data))
	case reflect.String:
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		resp = []byte((data).(string))
	case reflect.Map:
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		resp = encodeResponse(data)
	case reflect.Slice:
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		resp = encodeResponse(data)
	default:
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		resp = encodeResponse(data)
	}

	w.WriteHeader(http.StatusOK)
	w.Write(resp)
}

func badrequest(w http.ResponseWriter, msg string) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	w.WriteHeader(http.StatusBadRequest)
	w.Write([]byte(msg))
}

func notfound(w http.ResponseWriter) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	w.WriteHeader(http.StatusNotFound)
	w.Write([]byte("404 not found"))
}
<commit_msg>server - fixed port by actually parsing flags<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"reflect"
	"strings"

	"github.com/brianvoe/gofakeit/v5"
)

var port string

func init() {
	flag.StringVar(&port, "port", "8080", "server port")
}

func main() {
	flag.Parse()

	gofakeit.Seed(0)

	// Set router
	mux := http.NewServeMux()
	routes(mux)

	fmt.Println("Running on port " + port)

	http.ListenAndServe(":"+port, mux)
}

func routes(mux *http.ServeMux) {
	mux.HandleFunc("/favicon.ico", favicon)
	mux.HandleFunc("/list", list)
	mux.HandleFunc("/", lookup)
}

func favicon(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "image/x-icon")
	w.WriteHeader(http.StatusOK)
	w.Write(gofakeit.ImagePng(32, 32))
}

func list(w http.ResponseWriter, r *http.Request) {
	ok(w, gofakeit.FuncLookups)
}

func lookup(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case http.MethodGet:
		lookupGet(w, r)
		return
	case http.MethodPost:
		lookupPost(w, r)
		return
	}

	badrequest(w, "Only Get and Post methods allowed")
}

func lookupGet(w http.ResponseWriter, r *http.Request) {
	// Check url query for field(func) and
	info, err := getInfoFromPath(r)
	if err != nil {
		badrequest(w, err.Error())
		return
	}

	// Get url params
	var mapString map[string][]string
	urlParams := r.URL.Query()
	for key, values := range urlParams {
		if mapString == nil {
			mapString = make(map[string][]string)
		}
		mapString[key] = values
	}

	// Call method to generate requested data
	data, err := info.Call(&mapString, info)
	if err != nil {
		badrequest(w, err.Error())
		return
	}

	ok(w, data)
}

func lookupPost(w http.ResponseWriter, r *http.Request) {
	// Expect content type json
	contentType := r.Header.Get("Content-Type")
	if contentType != "application/json" {
		badrequest(w, `Expects header to be of Content-Type "application/json"`)
		return
	}

	// Check url query for field(func) and
	info, err := getInfoFromPath(r)
	if err != nil {
		badrequest(w, err.Error())
		return
	}

	var mapString map[string][]string

	// Try to decode body if params are needed
	if len(info.Params) > 0 {
		mapString = make(map[string][]string)
		mapInterface := map[string]interface{}{}
		err = json.NewDecoder(r.Body).Decode(&mapInterface)
		if err != nil {
			badrequest(w, "Could not parse post body. Expects key(string) - value or []value")
			return
		}
		defer r.Body.Close()

		for key, value := range mapInterface {
			v := reflect.ValueOf(value)
			switch v.Kind() {
			case reflect.Bool:
				mapString[key] = []string{fmt.Sprintf("%v", value)}
			case reflect.Int, reflect.Int8, reflect.Int32, reflect.Int64:
				mapString[key] = []string{fmt.Sprintf("%v", value)}
			case reflect.Uint, reflect.Uint8, reflect.Uint32, reflect.Uint64:
				mapString[key] = []string{fmt.Sprintf("%v", value)}
			case reflect.Float32, reflect.Float64:
				mapString[key] = []string{fmt.Sprintf("%v", value)}
			case reflect.String:
				mapString[key] = []string{fmt.Sprintf("%v", value)}
			case reflect.Map:
				mapString[key] = []string{fmt.Sprintf("%v", value)}
			case reflect.Slice:
				var vals []string
				for i := 0; i < v.Len(); i++ {
					vals = append(vals, fmt.Sprintf("%v", v.Index(i).Interface()))
				}

				mapString[key] = vals
			}
		}
	}

	// Call method to generate requested data
	data, err := info.Call(&mapString, info)
	if err != nil {
		badrequest(w, err.Error())
		return
	}

	ok(w, data)
}

func getInfoFromPath(r *http.Request) (*gofakeit.Info, error) {
	path := r.URL.Path
	path = strings.Trim(path, "/")
	paths := strings.Split(path, "/")
	if len(paths) == 0 {
		return nil, errors.New("No function was called, please pass func parameter")
	}

	// Lookup fake data method
	info := gofakeit.GetLookupData(paths[0])
	if info == nil {
		return nil, errors.New("No function was called, please pass func parameter")
	}

	return info, nil
}

func encodeResponse(v interface{}) []byte {
	// Set new bytes buffer
	buf := &bytes.Buffer{}
	enc := json.NewEncoder(buf)
	enc.SetEscapeHTML(true)
	enc.Encode(v)
	return buf.Bytes()
}

func ok(w http.ResponseWriter, data interface{}) {
	var resp []byte
	d := reflect.ValueOf(data)
	switch d.Kind() {
	case reflect.Bool:
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		resp = []byte(fmt.Sprintf("%v", data))
	case reflect.Int, reflect.Int8, reflect.Int32, reflect.Int64:
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		resp = []byte(fmt.Sprintf("%v", data))
	case reflect.Uint, reflect.Uint8, reflect.Uint32, reflect.Uint64:
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		resp = []byte(fmt.Sprintf("%v", data))
	case reflect.Float32, reflect.Float64:
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		resp = []byte(fmt.Sprintf("%v", data))
	case reflect.String:
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		resp = []byte((data).(string))
	case reflect.Map:
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		resp = encodeResponse(data)
	case reflect.Slice:
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		resp = encodeResponse(data)
	default:
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		resp = encodeResponse(data)
	}

	w.WriteHeader(http.StatusOK)
	w.Write(resp)
}

func badrequest(w http.ResponseWriter, msg string) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	w.WriteHeader(http.StatusBadRequest)
	w.Write([]byte(msg))
}

func notfound(w http.ResponseWriter) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	w.WriteHeader(http.StatusNotFound)
	w.Write([]byte("404 not found"))
}
<|endoftext|>