<commit_before>package rapi

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"strings"
)

// Request gathers all information about request
type Request struct {
	ID          int64  // storing record ID from URL
	Root        string // default JSON root key
	action      string
	req         *http.Request
	params      map[string]interface{}
	renderJSON  func(int, JSONData)
	render      func(int, string)
	renderError func(int, string)
	loadFile    func(string, string) (string, error)
}

// Init initializing controller
func (r *Request) Init(w http.ResponseWriter, req *http.Request, root, prefix string) {
	r.req = req
	r.Root = root

	urlParts := urlParts(strings.TrimPrefix(req.URL.Path, prefix))
	r.ID, _ = strconv.ParseInt(urlParts[0], 10, 64)
	r.action = r.makeAction(req.Method, urlParts)

	if strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") {
		r.renderJSON = func(code int, s JSONData) {
			RenderJSONgzip(w, code, s)
		}
	} else {
		r.renderJSON = func(code int, s JSONData) {
			RenderJSON(w, code, s)
		}
	}

	r.render = func(code int, s string) {
		w.WriteHeader(code)
		w.Write([]byte(s))
	}

	r.renderError = func(code int, s string) {
		http.Error(w, s, code)
	}

	r.loadFile = func(field, dir string) (string, error) {
		r.req.ParseMultipartForm(32 << 20)
		file, handler, err := r.req.FormFile(field)
		if err != nil {
			return "", err
		}
		defer file.Close()
		fmt.Fprintf(w, "%v", handler.Header)
		f, err := os.OpenFile(dir+handler.Filename, os.O_WRONLY|os.O_CREATE, 0666)
		if err != nil {
			return "", err
		}
		defer f.Close()
		io.Copy(f, file)
		return handler.Filename, nil
	}

	r.params = make(map[string]interface{})
}

func (r *Request) makeAction(method string, urlParts map[int]string) string {
	if len(urlParts[1]) > 0 {
		return method + capitalize(urlParts[1])
	}
	if r.ID > 0 {
		switch method {
		case "GET":
			return "Show"
		case "POST", "PUT":
			return "Update"
		case "DELETE":
			return "Destroy"
		}
	}

	if len(urlParts[0]) > 0 {
		return method + capitalize(urlParts[0])
	}

	switch method {
	case "GET":
		return "Index"
	case "POST":
		return "Create"
	}

	return "WrongAction"
}

// LoadJSONRequest extracting JSON request by key
// from request body into interface
func (r *Request) LoadJSONRequest(root string, v interface{}) {
	if root == "" {
		extractJSONPayload(r.req.Body, &v)
		return
	}

	var s []byte
	var body JSONData
	extractJSONPayload(r.req.Body, &body)
	s, _ = json.Marshal(body[root])
	json.Unmarshal(s, &v)
}

// QueryParam returns URL query param
func (r *Request) QueryParam(s string) string {
	return r.req.URL.Query().Get(s)
}

// SetParam set custom parameter for current request
func (r *Request) SetParam(k string, v interface{}) {
	r.params[k] = v
}

// Param returns custom parameter for current request
func (r *Request) Param(k string) interface{} {
	return r.params[k]
}

// Header returns request header
func (r *Request) Header(s string) string {
	return r.req.Header.Get(s)
}

// CurrentAction returns current controller action
func (r *Request) CurrentAction() string {
	return r.action
}

// RenderJSON rendering JSON to client
func (r *Request) RenderJSON(code int, s JSONData) {
	r.renderJSON(code, s)
}

// RenderJSONError rendering error to client in JSON format
func (r *Request) RenderJSONError(code int, s string) {
	r.renderJSON(code, JSONData{"errors": JSONData{"message": []string{s}}})
}

// Render rendering string to client
func (r *Request) RenderString(code int, s string) {
	r.render(code, s)
}

// RenderError rendering error to client
func (r *Request) RenderError(code int, s string) {
	r.renderError(code, s)
}

// LoadFile handling file uploads
func (r *Request) LoadFile(field, dir string) (string, error) {
	return r.loadFile(field, dir)
}
<commit_msg>cleaning Request initialization<commit_after>package rapi

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"strings"
)

// Request gathers all information about request
type Request struct {
	ID     int64  // storing record ID from URL
	Root   string // default JSON root key
	action string
	params map[string]interface{}

	req *http.Request
	w   http.ResponseWriter
}

// Init initializing controller
func (r *Request) Init(w http.ResponseWriter, req *http.Request, root, prefix string) {
	r.req = req
	r.w = w
	r.Root = root

	urlParts := urlParts(strings.TrimPrefix(req.URL.Path, prefix))
	r.ID, _ = strconv.ParseInt(urlParts[0], 10, 64)
	r.action = r.makeAction(req.Method, urlParts)

	r.params = make(map[string]interface{})
}

func (r *Request) makeAction(method string, urlParts map[int]string) string {
	if len(urlParts[1]) > 0 {
		return method + capitalize(urlParts[1])
	}
	if r.ID > 0 {
		switch method {
		case "GET":
			return "Show"
		case "POST", "PUT":
			return "Update"
		case "DELETE":
			return "Destroy"
		}
	}

	if len(urlParts[0]) > 0 {
		return method + capitalize(urlParts[0])
	}

	switch method {
	case "GET":
		return "Index"
	case "POST":
		return "Create"
	}

	return "WrongAction"
}

// LoadJSONRequest extracting JSON request by key
// from request body into interface
func (r *Request) LoadJSONRequest(root string, v interface{}) {
	if root == "" {
		extractJSONPayload(r.req.Body, &v)
		return
	}

	var s []byte
	var body JSONData
	extractJSONPayload(r.req.Body, &body)
	s, _ = json.Marshal(body[root])
	json.Unmarshal(s, &v)
}

// QueryParam returns URL query param
func (r *Request) QueryParam(s string) string {
	return r.req.URL.Query().Get(s)
}

// SetParam set custom parameter for current request
func (r *Request) SetParam(k string, v interface{}) {
	r.params[k] = v
}

// Param returns custom parameter for current request
func (r *Request) Param(k string) interface{} {
	return r.params[k]
}

// Header returns request header
func (r *Request) Header(s string) string {
	return r.req.Header.Get(s)
}

// CurrentAction returns current controller action
func (r *Request) CurrentAction() string {
	return r.action
}

// RenderJSON rendering JSON to client
func (r *Request) RenderJSON(code int, s JSONData) {
	if strings.Contains(r.req.Header.Get("Accept-Encoding"), "gzip") {
		RenderJSONgzip(r.w, code, s)
	} else {
		RenderJSON(r.w, code, s)
	}
}

// RenderJSONError rendering error to client in JSON format
func (r *Request) RenderJSONError(code int, s string) {
	r.RenderJSON(code, JSONData{"errors": JSONData{"message": []string{s}}})
}

// Render rendering string to client
func (r *Request) RenderString(code int, s string) {
	r.w.WriteHeader(code)
	r.w.Write([]byte(s))
}

// RenderError rendering error to client
func (r *Request) RenderError(code int, s string) {
	http.Error(r.w, s, code)
}

// LoadFile handling file uploads
func (r *Request) LoadFile(field, dir string) (string, error) {
	r.req.ParseMultipartForm(32 << 20)
	file, handler, err := r.req.FormFile(field)
	if err != nil {
		return "", err
	}
	defer file.Close()
	fmt.Fprintf(r.w, "%v", handler.Header)
	f, err := os.OpenFile(dir+handler.Filename, os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		return "", err
	}
	defer f.Close()
	io.Copy(f, file)
	return handler.Filename, nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/heroku/heroku-cli/Godeps/_workspace/src/github.com/franela/goreq"
)

func init() {
	goreq.SetConnectTimeout(15 * time.Second)
	goreq.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{RootCAs: getCACerts()}
}

func apiRequestBase(authToken string) *goreq.Request {
	req := goreq.Request{
		Uri:       apiURL(),
		ShowDebug: debugging,
		Insecure:  !shouldVerifyHost(apiURL()),
	}
	if authToken != "" {
		req.AddHeader("Authorization", "Bearer "+authToken)
	}
	if os.Getenv("HEROKU_HEADERS") != "" {
		var h map[string]string
		json.Unmarshal([]byte(os.Getenv("HEROKU_HEADERS")), &h)
		for k, v := range h {
			req.AddHeader(k, v)
		}
	}
	return &req
}

func apiRequest(authToken string) *goreq.Request {
	req := apiRequestBase(authToken)
	req.AddHeader("Accept", "application/vnd.heroku+json; version=3")
	return req
}

func shouldVerifyHost(host string) bool {
	return !(os.Getenv("HEROKU_SSL_VERIFY") == "disable" || strings.HasSuffix(host, "herokudev.com"))
}

func getCACerts() *x509.CertPool {
	certs := x509.NewCertPool()
	path := filepath.Join(AppDir(), "cacert.pem")
	data, err := ioutil.ReadFile(path)
	if err != nil {
		downloadCert(path)
		data, err = ioutil.ReadFile(path)
		if err != nil {
			PrintError(err, false)
			return nil
		}
	}
	ok := certs.AppendCertsFromPEM(data)
	if !ok {
		Warn("Error parsing " + path)
		return nil
	}
	return certs
}

func downloadCert(path string) {
	f, err := os.Create(path)
	if err != nil {
		PrintError(err, false)
		return
	}
	res, err := goreq.Request{
		Uri:       "https://cli-assets.heroku.com/cacert.pem",
		ShowDebug: debugging,
	}.Do()
	if err != nil {
		PrintError(err, false)
		return
	}
	defer res.Body.Close()
	defer f.Close()
	io.Copy(f, res.Body)
}
<commit_msg>allow HEROKU_USE_SYSTEM_CERT=false to use custom cacert.pem<commit_after>package main

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/heroku/heroku-cli/Godeps/_workspace/src/github.com/franela/goreq"
)

func init() {
	goreq.SetConnectTimeout(15 * time.Second)
	if !useSystemCerts() {
		goreq.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{RootCAs: getCACerts()}
	}
}

func useSystemCerts() bool {
	e := os.Getenv("HEROKU_USE_SYSTEM_CERTS")
	return e != "false" && e != "0"
}

func apiRequestBase(authToken string) *goreq.Request {
	req := goreq.Request{
		Uri:       apiURL(),
		ShowDebug: debugging,
		Insecure:  !shouldVerifyHost(apiURL()),
	}
	if authToken != "" {
		req.AddHeader("Authorization", "Bearer "+authToken)
	}
	if os.Getenv("HEROKU_HEADERS") != "" {
		var h map[string]string
		json.Unmarshal([]byte(os.Getenv("HEROKU_HEADERS")), &h)
		for k, v := range h {
			req.AddHeader(k, v)
		}
	}
	return &req
}

func apiRequest(authToken string) *goreq.Request {
	req := apiRequestBase(authToken)
	req.AddHeader("Accept", "application/vnd.heroku+json; version=3")
	return req
}

func shouldVerifyHost(host string) bool {
	return !(os.Getenv("HEROKU_SSL_VERIFY") == "disable" || strings.HasSuffix(host, "herokudev.com"))
}

func getCACerts() *x509.CertPool {
	certs := x509.NewCertPool()
	path := filepath.Join(AppDir(), "cacert.pem")
	data, err := ioutil.ReadFile(path)
	if err != nil {
		downloadCert(path)
		data, err = ioutil.ReadFile(path)
		if err != nil {
			PrintError(err, false)
			return nil
		}
	}
	ok := certs.AppendCertsFromPEM(data)
	if !ok {
		Warn("Error parsing " + path)
		return nil
	}
	return certs
}

func downloadCert(path string) {
	f, err := os.Create(path)
	if err != nil {
		PrintError(err, false)
		return
	}
	res, err := goreq.Request{
		Uri:       "https://cli-assets.heroku.com/cacert.pem",
		ShowDebug: debugging,
	}.Do()
	if err != nil {
		PrintError(err, false)
		return
	}
	defer res.Body.Close()
	defer f.Close()
	io.Copy(f, res.Body)
}
<|endoftext|>


<commit_before>package personnummer

import (
	"math"
	"reflect"
	"regexp"
	"strconv"
	"time"
)

// The Luhn algorithm.
func luhn(str string) int {
	sum := 0

	for i, r := range str {
		c := string(r)
		v, _ := strconv.Atoi(c)
		v *= 2 - (i % 2)
		if v > 9 {
			v -= 9
		}
		sum += v
	}

	return int(math.Ceil(float64(sum)/10)*10 - float64(sum))
}

// Test date if lunh is true
func testDate(century string, year string, month string, day string) bool {
	t, err := time.Parse("01/02/2006", month+"/"+day+"/"+century+year)

	if err != nil {
		return false
	}

	y, _ := strconv.Atoi(century + year)
	m, _ := strconv.Atoi(month)
	d, _ := strconv.Atoi(day)

	if y > time.Now().Year() {
		return false
	}

	return !(t.Year() != y || int(t.Month()) != m || t.Day() != d)
}

// Get co-ordination number day
func getCoOrdinationDay(day string) string {
	d, _ := strconv.Atoi(day)
	d -= 60
	day = strconv.Itoa(d)

	if d < 10 {
		day = "0" + day
	}

	return day
}

// Validate Swedish personal identity numbers.
func Valid(str interface{}) bool {
	if reflect.TypeOf(str).Kind() != reflect.Int && reflect.TypeOf(str).Kind() != reflect.String {
		return false
	}

	pr := ""

	if reflect.TypeOf(str).Kind() == reflect.Int {
		pr = strconv.Itoa(str.(int))
	} else {
		pr = str.(string)
	}

	re, _ := regexp.Compile(`^(\d{2}){0,1}(\d{2})(\d{2})(\d{2})([\-|\+]{0,1})?(\d{3})(\d{0,1})$`)
	match := re.FindStringSubmatch(pr)

	if len(match) == 0 {
		return false
	}

	century := match[1]
	year := match[2]
	month := match[3]
	day := match[4]
	sep := match[5]
	num := match[6]
	check := match[7]

	if len(century) == 0 {
		yearNow := time.Now().Year()
		years := [...]int{yearNow, yearNow - 100, yearNow - 150}

		for _, yi := range years {
			ys := strconv.Itoa(yi)

			if Valid(ys[:2] + pr) {
				return true
			}
		}

		return false
	}

	if len(year) == 4 {
		year = year[2:]
	}

	if len(sep) == 0 {
		sep = "-"
	}

	c, _ := strconv.Atoi(check)

	valid := luhn(year+month+day+num) == c && len(check) != 0

	if valid && testDate(century, year, month, day) {
		return valid
	}

	day = getCoOrdinationDay(day)

	return valid && testDate(century, year, month, day)
}
<commit_msg>Removed unused code<commit_after>package personnummer

import (
	"math"
	"reflect"
	"regexp"
	"strconv"
	"time"
)

// The Luhn algorithm.
func luhn(str string) int {
	sum := 0

	for i, r := range str {
		c := string(r)
		v, _ := strconv.Atoi(c)
		v *= 2 - (i % 2)
		if v > 9 {
			v -= 9
		}
		sum += v
	}

	return int(math.Ceil(float64(sum)/10)*10 - float64(sum))
}

// Test date if lunh is true
func testDate(century string, year string, month string, day string) bool {
	t, err := time.Parse("01/02/2006", month+"/"+day+"/"+century+year)

	if err != nil {
		return false
	}

	y, _ := strconv.Atoi(century + year)
	m, _ := strconv.Atoi(month)
	d, _ := strconv.Atoi(day)

	if y > time.Now().Year() {
		return false
	}

	return !(t.Year() != y || int(t.Month()) != m || t.Day() != d)
}

// Get co-ordination number day
func getCoOrdinationDay(day string) string {
	d, _ := strconv.Atoi(day)
	d -= 60
	day = strconv.Itoa(d)

	if d < 10 {
		day = "0" + day
	}

	return day
}

// Validate Swedish personal identity numbers.
func Valid(str interface{}) bool {
	if reflect.TypeOf(str).Kind() != reflect.Int && reflect.TypeOf(str).Kind() != reflect.String {
		return false
	}

	pr := ""

	if reflect.TypeOf(str).Kind() == reflect.Int {
		pr = strconv.Itoa(str.(int))
	} else {
		pr = str.(string)
	}

	re, _ := regexp.Compile(`^(\d{2}){0,1}(\d{2})(\d{2})(\d{2})([\-|\+]{0,1})?(\d{3})(\d{0,1})$`)
	match := re.FindStringSubmatch(pr)

	if len(match) == 0 {
		return false
	}

	century := match[1]
	year := match[2]
	month := match[3]
	day := match[4]
	num := match[6]
	check := match[7]

	if len(century) == 0 {
		yearNow := time.Now().Year()
		years := [...]int{yearNow, yearNow - 100, yearNow - 150}

		for _, yi := range years {
			ys := strconv.Itoa(yi)

			if Valid(ys[:2] + pr) {
				return true
			}
		}

		return false
	}

	if len(year) == 4 {
		year = year[2:]
	}

	c, _ := strconv.Atoi(check)

	valid := luhn(year+month+day+num) == c && len(check) != 0

	if valid && testDate(century, year, month, day) {
		return valid
	}

	day = getCoOrdinationDay(day)

	return valid && testDate(century, year, month, day)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"log"
	"os"
	"strconv"
	"time"

	_ "github.com/denisenkom/go-mssqldb"
	_ "github.com/lib/pq"

	"github.com/lic-nz/go-datapipe/bulk"
)

type Config struct {
	maxRowBufSz    int //Maximum number of rows to buffer at a time
	maxRowTxCommit int //Maximum number of rows to process before committing the database transaction

	srcDbDriver  string //Source database driver name
	srcDbUri     string //Source database driver URI
	srcSelectSql string //Source database select SQL statement

	dstDbDriver string //Destination database driver name
	dstDbUri    string //Destination database driver URI
	dstSchema   string
	dstTable    string //Destination database table name
}

type Insert interface {
	Append(rows *sql.Rows) (err error)
	Flush() (totalRowCount int, err error)
	Close() (err error)
}

func (c *Config) Init() (err error) {
	c.maxRowBufSz, _ = c.EnvInt("MAX_ROW_BUF_SZ", 100)
	c.maxRowTxCommit, _ = c.EnvInt("MAX_ROW_TX_COMMIT", 500)

	if c.srcDbDriver, err = c.EnvStr("SRC_DB_DRIVER"); err != nil {
		return err
	}
	if c.srcDbUri, err = c.EnvStr("SRC_DB_URI"); err != nil {
		return err
	}
	if c.srcSelectSql, err = c.EnvStr("SRC_DB_SELECT_SQL"); err != nil {
		return err
	}

	if c.dstDbDriver, err = c.EnvStr("DST_DB_DRIVER"); err != nil {
		return err
	}
	if c.dstDbUri, err = c.EnvStr("DST_DB_URI"); err != nil {
		return err
	}
	if c.dstSchema, err = c.EnvStr("DST_DB_SCHEMA"); err != nil {
		return err
	}
	if c.dstTable, err = c.EnvStr("DST_DB_TABLE"); err != nil {
		return err
	}

	return nil
}

func (c *Config) EnvStr(envName string) (dst string, err error) {
	dst = os.Getenv(envName)
	if dst == "" {
		err = errors.New("Missing ENV variable: " + envName)
	} else {
		log.Printf("%s=%s", envName, dst)
	}

	return dst, err
}

func (c *Config) EnvInt(envName string, defaultValue int) (dst int, err error) {
	if dst, err = strconv.Atoi(os.Getenv(envName)); err != nil {
		dst = defaultValue
	}

	log.Printf("%s=%d", envName, dst)

	return dst, nil
}

func main() {
	cfg := &Config{}
	if err := cfg.Init(); err != nil {
		log.Fatal(err)
		return
	}

	if err := run(cfg); err != nil {
		log.Fatal(err)
		return
	}
}

func run(cfg *Config) (err error) {
	var srcDb, dstDb *sql.DB

	if srcDb, err = sql.Open(cfg.srcDbDriver, cfg.srcDbUri); err != nil {
		return err
	}

	defer srcDb.Close()

	if dstDb, err = sql.Open(cfg.dstDbDriver, cfg.dstDbUri); err != nil {
		return err
	}

	defer dstDb.Close()

	if err = clearTable(dstDb, cfg); err != nil {
		return err
	}

	if err = copyTable(srcDb, dstDb, cfg); err != nil {
		return err
	}

	return nil
}

func clearTable(dstDb *sql.DB, cfg *Config) (err error) {
	if _, err = dstDb.Exec("TRUNCATE TABLE " + cfg.dstSchema + "." + cfg.dstTable); err != nil {
		return err
	}
	return nil
}

func copyTable(srcDb *sql.DB, dstDb *sql.DB, cfg *Config) (err error) {
	var ir Insert
	var rows *sql.Rows
	var rowCount int
	var columns []string

	if rows, err = srcDb.Query(cfg.srcSelectSql); err != nil {
		return err
	}

	if columns, err = rows.Columns(); err != nil {
		return err
	}

	startTime := time.Now()

	switch cfg.dstDbDriver {
	case "postgres":
		log.Printf("Using COPY IN\n")
		if ir, err = bulk.NewCopyIn(dstDb, columns, cfg.dstSchema, cfg.dstTable); err != nil {
			return err
		}
	default:
		log.Printf("Using bulk insert\n")
		if ir, err = bulk.NewBulk(dstDb, columns, cfg.dstSchema, cfg.dstTable, cfg.maxRowBufSz, cfg.maxRowTxCommit); err != nil {
			return err
		}
	}

	rowCount, err = copyBulkRows(dstDb, rows, ir, cfg)

	if err = ir.Close(); err != nil {
		return err
	}

	duration := time.Since(startTime)

	log.Printf("%d rows in %s\n", rowCount, duration.String())

	return err
}

func copyBulkRows(dstDb *sql.DB, rows *sql.Rows, ir Insert, cfg *Config) (rowCount int, err error) {
	var totalRowCount int

	i := 1

	for rows.Next() {
		if err = ir.Append(rows); err != nil {
			return 0, err
		}

		if i%1000 == 0 {
			fmt.Print(".")
		}

		i++
	}

	if totalRowCount, err = ir.Flush(); err != nil {
		return 0, err
	}

	fmt.Println()

	return totalRowCount, nil
}
<commit_msg>Fix repository path.<commit_after>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"log"
	"os"
	"strconv"
	"time"

	_ "github.com/denisenkom/go-mssqldb"
	_ "github.com/lib/pq"

	"github.com/literatesnow/go-datapipe/bulk"
)

type Config struct {
	maxRowBufSz    int //Maximum number of rows to buffer at a time
	maxRowTxCommit int //Maximum number of rows to process before committing the database transaction

	srcDbDriver  string //Source database driver name
	srcDbUri     string //Source database driver URI
	srcSelectSql string //Source database select SQL statement

	dstDbDriver string //Destination database driver name
	dstDbUri    string //Destination database driver URI
	dstSchema   string
	dstTable    string //Destination database table name
}

type Insert interface {
	Append(rows *sql.Rows) (err error)
	Flush() (totalRowCount int, err error)
	Close() (err error)
}

func (c *Config) Init() (err error) {
	c.maxRowBufSz, _ = c.EnvInt("MAX_ROW_BUF_SZ", 100)
	c.maxRowTxCommit, _ = c.EnvInt("MAX_ROW_TX_COMMIT", 500)

	if c.srcDbDriver, err = c.EnvStr("SRC_DB_DRIVER"); err != nil {
		return err
	}
	if c.srcDbUri, err = c.EnvStr("SRC_DB_URI"); err != nil {
		return err
	}
	if c.srcSelectSql, err = c.EnvStr("SRC_DB_SELECT_SQL"); err != nil {
		return err
	}

	if c.dstDbDriver, err = c.EnvStr("DST_DB_DRIVER"); err != nil {
		return err
	}
	if c.dstDbUri, err = c.EnvStr("DST_DB_URI"); err != nil {
		return err
	}
	if c.dstSchema, err = c.EnvStr("DST_DB_SCHEMA"); err != nil {
		return err
	}
	if c.dstTable, err = c.EnvStr("DST_DB_TABLE"); err != nil {
		return err
	}

	return nil
}

func (c *Config) EnvStr(envName string) (dst string, err error) {
	dst = os.Getenv(envName)
	if dst == "" {
		err = errors.New("Missing ENV variable: " + envName)
	} else {
		log.Printf("%s=%s", envName, dst)
	}

	return dst, err
}

func (c *Config) EnvInt(envName string, defaultValue int) (dst int, err error) {
	if dst, err = strconv.Atoi(os.Getenv(envName)); err != nil {
		dst = defaultValue
	}

	log.Printf("%s=%d", envName, dst)

	return dst, nil
}

func main() {
	cfg := &Config{}
	if err := cfg.Init(); err != nil {
		log.Fatal(err)
		return
	}

	if err := run(cfg); err != nil {
		log.Fatal(err)
		return
	}
}

func run(cfg *Config) (err error) {
	var srcDb, dstDb *sql.DB

	if srcDb, err = sql.Open(cfg.srcDbDriver, cfg.srcDbUri); err != nil {
		return err
	}

	defer srcDb.Close()

	if dstDb, err = sql.Open(cfg.dstDbDriver, cfg.dstDbUri); err != nil {
		return err
	}

	defer dstDb.Close()

	if err = clearTable(dstDb, cfg); err != nil {
		return err
	}

	if err = copyTable(srcDb, dstDb, cfg); err != nil {
		return err
	}

	return nil
}

func clearTable(dstDb *sql.DB, cfg *Config) (err error) {
	if _, err = dstDb.Exec("TRUNCATE TABLE " + cfg.dstSchema + "." + cfg.dstTable); err != nil {
		return err
	}
	return nil
}

func copyTable(srcDb *sql.DB, dstDb *sql.DB, cfg *Config) (err error) {
	var ir Insert
	var rows *sql.Rows
	var rowCount int
	var columns []string

	if rows, err = srcDb.Query(cfg.srcSelectSql); err != nil {
		return err
	}

	if columns, err = rows.Columns(); err != nil {
		return err
	}

	startTime := time.Now()

	switch cfg.dstDbDriver {
	case "postgres":
		log.Printf("Using COPY IN\n")
		if ir, err = bulk.NewCopyIn(dstDb, columns, cfg.dstSchema, cfg.dstTable); err != nil {
			return err
		}
	default:
		log.Printf("Using bulk insert\n")
		if ir, err = bulk.NewBulk(dstDb, columns, cfg.dstSchema, cfg.dstTable, cfg.maxRowBufSz, cfg.maxRowTxCommit); err != nil {
			return err
		}
	}

	rowCount, err = copyBulkRows(dstDb, rows, ir, cfg)

	if err = ir.Close(); err != nil {
		return err
	}

	duration := time.Since(startTime)

	log.Printf("%d rows in %s\n", rowCount, duration.String())

	return err
}

func copyBulkRows(dstDb *sql.DB, rows *sql.Rows, ir Insert, cfg *Config) (rowCount int, err error) {
	var totalRowCount int

	i := 1

	for rows.Next() {
		if err = ir.Append(rows); err != nil {
			return 0, err
		}

		if i%1000 == 0 {
			fmt.Print(".")
		}

		i++
	}

	if totalRowCount, err = ir.Flush(); err != nil {
		return 0, err
	}

	fmt.Println()

	return totalRowCount, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
)

var cmdDataPipe = &Command{
	Usage: "datapipe <command> [<args>]",
	Short: "Manage DataPipes",
	Long: `
Manage DataPipes

Usage:

  force datapipe create -n <name> [-l masterlabel] [-t scripttype] [-c scriptcontent] [-v apiversion]

  force datapipe update -n <name> [-l masterlabel] [-t scripttype] [-c scriptcontent] [-v apiversion]

  force datapipe delete -n <name>

  force datapipe list -f <"csv" or "json">

  force datapipe query -q <query string>

  force datapipe createjob -n <pipeline name>

Commands:
  create        creates a new dataPipe
  update        update a dataPipe
  delete        delete a datapipe
  list          list all datapipes
  query         query for a specific datapipe(s)
  createjob     creates a new job for a specific datapipe
  listjobs      list the status of submitted jobs
  queryjob      returns data about a datapipeline job (not implemented)
  retrieve      (not implemented)

Examples:

  force datapipe create -n=MyPipe -l="My Pipe" -t=Pig -v=34.0 \
  -c="A = load 'force://soql/Select Id, Name From Contact' using \
  gridforce.hadoop.pig.loadstore.func.ForceStorage();"

Defaults
  -l Defaults to the name
  -t Pig (only option available currently)
  -c Pig script template
  -v Current API version *Number only

`,
}

var defaultContent = `
-- Sample script for a data pipeline
A = load 'ffx://REPLACE_ME' using gridforce.hadoop.pig.loadstore.func.ForceStorage();
Store A  into 'ffx://REPLACE_ME_TOO' using gridforce.hadoop.pig.loadstore.func.ForceStorage();
`

var (
	dpname        string
	masterlabel   string
	scriptcontent string
	apiversion    string
	scripttype    string
	query         string
	format        string
)

func init() {
	cmdDataPipe.Flag.StringVar(&dpname, "name", "", "set datapipeline name")
	cmdDataPipe.Flag.StringVar(&dpname, "n", "", "set datapipeline name")
	cmdDataPipe.Flag.StringVar(&masterlabel, "masterlabel", "", "set master label")
	cmdDataPipe.Flag.StringVar(&masterlabel, "l", "", "set master label")
	cmdDataPipe.Flag.StringVar(&scriptcontent, "scriptcontent", defaultContent, "set script content")
	cmdDataPipe.Flag.StringVar(&scriptcontent, "c", defaultContent, "set script content")
	cmdDataPipe.Flag.StringVar(&apiversion, "apiversion", apiVersionNumber, "set api version")
	cmdDataPipe.Flag.StringVar(&apiversion, "v", apiVersionNumber, "set api version")
	cmdDataPipe.Flag.StringVar(&scripttype, "scripttype", "Pig", "set script type")
	cmdDataPipe.Flag.StringVar(&scripttype, "t", "Pig", "set script type")
	cmdDataPipe.Flag.StringVar(&query, "q", "", "SOQL query string on DataPipeline object")
	cmdDataPipe.Flag.StringVar(&query, "query", "", "SOQL query string on DataPipeline object")
	cmdDataPipe.Flag.StringVar(&format, "f", "json", "format for listing datapipelines (csv or json)")
	cmdDataPipe.Flag.StringVar(&format, "format", "json", "format for listing datapipelines (csv or json)")
	cmdDataPipe.Run = runDataPipe
}

func runDataPipe(cmd *Command, args []string) {
	if len(args) == 0 {
		cmd.printUsage()
	} else {
		if err := cmd.Flag.Parse(args[1:]); err != nil {
			os.Exit(2)
		}
		switch args[0] {
		case "create":
			runDataPipelineCreate()
		case "update":
			runDataPipelineUpdate()
		case "delete":
			runDataPipelineDelete()
		case "list":
			runDataPipelineList()
		case "query":
			runDataPipelineQuery()
		case "createjob":
			runDataPipelineJob()
		case "listjobs":
			runDataPipelineListJobs()
		default:
			ErrorAndExit("no such command: %s", args[0])
		}
	}
}

func runDataPipelineJob() {
	if len(dpname) == 0 {
		ErrorAndExit("You need to provide the name of a pipeline to create a job for.")
	}
	force, _ := ActiveForce()
	id := GetDataPipelineId(dpname)
	_, err, _ := force.CreateDataPipelineJob(id)
	if err != nil {
		ErrorAndExit(err.Error())
	}
	fmt.Printf("Successfully created DataPipeline job for %s\n", dpname)
}

func runDataPipelineListJobs() {
	//query = "SELECT Id, DataPipeline.DeveloperName, Status, FailureState, LastModifiedDate, CreatedDate, CreatedById, DataPipelineId, JobErrorMessage FROM DataPipelineJob"
	query = "SELECT DataPipeline.DeveloperName, Status, FailureState FROM DataPipelineJob"
	force, _ := ActiveForce()
	result, err := force.QueryDataPipelineJob(query)
	if err != nil {
		ErrorAndExit(err.Error())
	}

	DisplayForceRecordsf(result.Records, "csv")
}

func runDataPipelineQuery() {
	if len(query) == 0 {
		ErrorAndExit("You have to supply a SOQL query using the -q flag.")
	}
	force, _ := ActiveForce()
	result, err := force.QueryDataPipeline(query)
	if err != nil {
		ErrorAndExit(err.Error())
	}

	fmt.Println("Result: \n", result)
}

func runDataPipelineCreate() {
	if len(dpname) == 0 {
		ErrorAndExit("You must specify a name for the datapipeline using the -n flag.")
	}
	if len(masterlabel) == 0 {
		masterlabel = dpname
	}

	force, _ := ActiveForce()
	_, err, _ := force.CreateDataPipeline(dpname, masterlabel, apiversion, scriptcontent, scripttype)
	if err != nil {
		ErrorAndExit(err.Error())
	}
	fmt.Printf("DataPipeline %s successfully created.\n", dpname)
}

func runDataPipelineUpdate() {
	if len(dpname) == 0 {
		ErrorAndExit("You must specify a name for the datapipeline using the -n flag.")
	}
	if len(masterlabel) == 0 && len(scriptcontent) == 0 {
		ErrorAndExit("You can change the master label or the script content.")
	}

	force, _ := ActiveForce()

	result, err := force.GetDataPipeline(dpname)
	if err != nil {
		ErrorAndExit(err.Error())
	}

	if len(result.Records) == 0 {
		ErrorAndExit("No data pipeline found named " + dpname)
	}
	for _, record := range result.Records {
		var id string
		id = record["Id"].(string)
		if len(masterlabel) == 0 {
			masterlabel = record["MasterLabel"].(string)
		}
		if len(scriptcontent) == 0 {
			scriptcontent = record["ScriptContent"].(string)
		}
		if _, err := os.Stat(scriptcontent); err == nil {
			fmt.Printf("file exists; processing...")
			scriptcontent, err = readScriptFile(scriptcontent)
			if err != nil {
				ErrorAndExit(err.Error())
			}
		}
		err = force.UpdateDataPipeline(id, masterlabel, scriptcontent)
		if err != nil {
			ErrorAndExit(err.Error())
		}
		fmt.Printf("%s successfully updated.\n", dpname)
	}
}

func readScriptFile(path string) (content string, err error) {
	data, err := ioutil.ReadFile(path)
	content = string(data)
	return
}

func GetDataPipelineId(name string) (id string) {
	force, _ := ActiveForce()

	result, err := force.GetDataPipeline(dpname)
	if err != nil {
		ErrorAndExit(err.Error())
	}

	if len(result.Records) == 0 {
		ErrorAndExit("No data pipeline found named " + dpname)
	}

	record := result.Records[0]
	id = record["Id"].(string)
	return
}

func runDataPipelineDelete() {
	force, _ := ActiveForce()

	id := GetDataPipelineId(dpname)
	err := force.DeleteDataPipeline(id)
	if err != nil {
		ErrorAndExit(err.Error())
	}
	fmt.Printf("%s successfully deleted.\n", dpname)
}

func runDataPipelineList() {
	force, _ := ActiveForce()
	query = "SELECT Id, MasterLabel, DeveloperName, ScriptType FROM DataPipeline"
	result, err := force.QueryDataPipeline(query)
	if err != nil {
		ErrorAndExit(err.Error())
	}

	DisplayForceRecordsf(result.Records, format)
}
<commit_msg>datapipe support<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
)

var cmdDataPipe = &Command{
	Usage: "datapipe <command> [<args>]",
	Short: "Manage DataPipes",
	Long: `
Manage DataPipes

Usage:

  force datapipe create -n <name> [-l masterlabel] [-t scripttype] [-c scriptcontent] [-v apiversion]

  force datapipe update -n <name> [-l masterlabel] [-t scripttype] [-c scriptcontent] [-v apiversion]

  force datapipe delete -n <name>

  force datapipe list -f <"csv" or "json">

  force datapipe query -q <query string>

  force datapipe createjob -n <pipeline name>

Commands:
  create        creates a new dataPipe
  update        update a dataPipe
  delete        delete a datapipe
  list          list all datapipes
  query         query for a specific datapipe(s)
  createjob     creates a new job for a specific datapipe
  listjobs      list the status of submitted jobs
  queryjob      returns data about a datapipeline job (not implemented)
  retrieve      (not implemented)

Examples:

  force datapipe create -n=MyPipe -l="My Pipe" -t=Pig -v=34.0 \
  -c="A = load 'force://soql/Select Id, Name From Contact' using \
  gridforce.hadoop.pig.loadstore.func.ForceStorage();"

Defaults
  -l Defaults to the name
  -t Pig (only option available currently)
  -c Pig script template
  -v Current API version *Number only

`,
}

var defaultContent = `
-- Sample script for a data pipeline
A = load 'ffx://REPLACE_ME' using gridforce.hadoop.pig.loadstore.func.ForceStorage();
Store A  into 'ffx://REPLACE_ME_TOO' using gridforce.hadoop.pig.loadstore.func.ForceStorage();
`

var (
	dpname        string
	masterlabel   string
	scriptcontent string
	apiversion    string
	scripttype    string
	query         string
	format        string
	jobid		  string
)

func init() {
	cmdDataPipe.Flag.StringVar(&dpname, "name", "", "set datapipeline name")
	cmdDataPipe.Flag.StringVar(&dpname, "n", "", "set datapipeline name")
	cmdDataPipe.Flag.StringVar(&masterlabel, "masterlabel", "", "set master label")
	cmdDataPipe.Flag.StringVar(&masterlabel, "l", "", "set master label")
	cmdDataPipe.Flag.StringVar(&scriptcontent, "scriptcontent", defaultContent, "set script content")
	cmdDataPipe.Flag.StringVar(&scriptcontent, "c", defaultContent, "set script content")
	cmdDataPipe.Flag.StringVar(&apiversion, "apiversion", apiVersionNumber, "set api version")
	cmdDataPipe.Flag.StringVar(&apiversion, "v", apiVersionNumber, "set api version")
	cmdDataPipe.Flag.StringVar(&scripttype, "scripttype", "Pig", "set script type")
	cmdDataPipe.Flag.StringVar(&scripttype, "t", "Pig", "set script type")
	cmdDataPipe.Flag.StringVar(&query, "q", "", "SOQL query string on DataPipeline object")
	cmdDataPipe.Flag.StringVar(&query, "query", "", "SOQL query string on DataPipeline object")
	cmdDataPipe.Flag.StringVar(&format, "f", "json", "format for listing datapipelines (csv or json)")
	cmdDataPipe.Flag.StringVar(&format, "format", "json", "format for listing datapipelines (csv or json)")
	cmdDataPipe.Flag.StringVar(&jobid, "j", "json", "Id of data pipline job to retrieve")
	cmdDataPipe.Flag.StringVar(&jobid, "jobid", "json", "Id of data pipline job to retrieve")
	cmdDataPipe.Run = runDataPipe
}

func runDataPipe(cmd *Command, args []string) {
	if len(args) == 0 {
		cmd.printUsage()
	} else {
		if err := cmd.Flag.Parse(args[1:]); err != nil {
			os.Exit(2)
		}
		switch args[0] {
		case "create":
			runDataPipelineCreate()
		case "update":
			runDataPipelineUpdate()
		case "delete":
			runDataPipelineDelete()
		case "list":
			runDataPipelineList()
		case "query":
			runDataPipelineQuery()
		case "createjob":
			runDataPipelineJob()
		case "listjobs":
			runDataPipelineListJobs()
		case "queryjob":
			runDataPipelineQueryJob()
		default:
			ErrorAndExit("no such command: %s", args[0])
		}
	}
}

func runDataPipelineJob() {
	if len(dpname) == 0 {
		ErrorAndExit("You need to provide the name of a pipeline to create a job for.")
	}
	force, _ := ActiveForce()
	id := GetDataPipelineId(dpname)
	_, err, _ := force.CreateDataPipelineJob(id)
	if err != nil {
		ErrorAndExit(err.Error())
	}
	fmt.Printf("Successfully created DataPipeline job for %s\n", dpname)
}

func runDataPipelineListJobs() {
	//query = "SELECT Id, DataPipeline.DeveloperName, Status, FailureState, LastModifiedDate, CreatedDate, CreatedById, DataPipelineId, JobErrorMessage FROM DataPipelineJob"
	query = "SELECT Id, DataPipeline.DeveloperName, Status, FailureState FROM DataPipelineJob"
	force, _ := ActiveForce()
	result, err := force.QueryDataPipelineJob(query)
	if err != nil {
		ErrorAndExit(err.Error())
	}

	DisplayForceRecordsf(result.Records, "csv")
}

func runDataPipelineQueryJob() {
	//query = "SELECT Id, DataPipeline.DeveloperName, Status, FailureState, LastModifiedDate, CreatedDate, CreatedById, DataPipelineId, JobErrorMessage FROM DataPipelineJob"
	query = fmt.Sprintf("SELECT Id, DataPipeline.DeveloperName, Status, FailureState, JobErrorMessage FROM DataPipelineJob Where id = '%s'", jobid)
	force, _ := ActiveForce()
	result, err := force.QueryDataPipelineJob(query)
	if err != nil {
		ErrorAndExit(err.Error())
	}

	DisplayForceRecordsf(result.Records, "csv")
}

func runDataPipelineQuery() {
	if len(query) == 0 {
		ErrorAndExit("You have to supply a SOQL query using the -q flag.")
	}
	force, _ := ActiveForce()
	result, err := force.QueryDataPipeline(query)
	if err != nil {
		ErrorAndExit(err.Error())
	}

	fmt.Println("Result: \n", result)
}

func runDataPipelineCreate() {
	if len(dpname) == 0 {
		ErrorAndExit("You must specify a name for the datapipeline using the -n flag.")
	}
	if len(masterlabel) == 0 {
		masterlabel = dpname
	}

	force, _ := ActiveForce()
	_, err, _ := force.CreateDataPipeline(dpname, masterlabel, apiversion, scriptcontent, scripttype)
	if err != nil {
		ErrorAndExit(err.Error())
	}
	fmt.Printf("DataPipeline %s successfully created.\n", dpname)
}

func runDataPipelineUpdate() {
	if len(dpname) == 0 {
		ErrorAndExit("You must specify a name for the datapipeline using the -n flag.")
	}
	if len(masterlabel) == 0 && len(scriptcontent) == 0 {
		ErrorAndExit("You can change the master label or the script content.")
	}

	force, _ := ActiveForce()

	result, err := force.GetDataPipeline(dpname)
	if err != nil {
		ErrorAndExit(err.Error())
	}

	if len(result.Records) == 0 {
		ErrorAndExit("No data pipeline found named " + dpname)
	}
	for _, record := range result.Records {
		var id string
		id = record["Id"].(string)
		if len(masterlabel) == 0 {
			masterlabel = record["MasterLabel"].(string)
		}
		if len(scriptcontent) == 0 {
			scriptcontent = record["ScriptContent"].(string)
		}
		if _, err := os.Stat(scriptcontent); err == nil {
			fmt.Printf("file exists; processing...")
			scriptcontent, err = readScriptFile(scriptcontent)
			if err != nil {
				ErrorAndExit(err.Error())
			}
		}
		err = force.UpdateDataPipeline(id, masterlabel, scriptcontent)
		if err != nil {
			ErrorAndExit(err.Error())
		}
		fmt.Printf("%s successfully updated.\n", dpname)
	}
}

func readScriptFile(path string) (content string, err error) {
	data, err := ioutil.ReadFile(path)
	content = string(data)
	return
}

func GetDataPipelineId(name string) (id string) {
	force, _ := ActiveForce()

	result, err := force.GetDataPipeline(dpname)
	if err != nil {
		ErrorAndExit(err.Error())
	}

	if len(result.Records) == 0 {
		ErrorAndExit("No data pipeline found named " + dpname)
	}

	record := result.Records[0]
	id = record["Id"].(string)
	return
}

func runDataPipelineDelete() {
	force, _ := ActiveForce()

	id := GetDataPipelineId(dpname)
	err := force.DeleteDataPipeline(id)
	if err != nil {
		ErrorAndExit(err.Error())
	}
	fmt.Printf("%s successfully deleted.\n", dpname)
}

func runDataPipelineList() {
	force, _ := ActiveForce()
	query = "SELECT Id, MasterLabel, DeveloperName, ScriptType FROM DataPipeline"
	result, err := force.QueryDataPipeline(query)
	if err != nil {
		ErrorAndExit(err.Error())
	}

	DisplayForceRecordsf(result.Records, format)
}
<|endoftext|>


<commit_before>package logmetrics

import (
	"fmt"
	"log"
	"strconv"
	"strings"
	"syseng/go-timemetrics"
	"time"
)

type dataPoint struct {
	name        string
	value       int64
	metric_type string
}

type dataPointTime struct {
	name string
	time int64
}

type tsdPoint struct {
	data             timemetrics.Metric
	lastPush         time.Time
	lastCrunchedPush time.Time
}

func (lg *LogGroup) extractTags(data []string) []string {
	tags := make([]string, lg.getNbTags())

	i := 0

	//General tags
	for tagname, position := range lg.tags {
		tags[i] = fmt.Sprintf("%s=%s", tagname, data[position])
		i++
	}

	return tags
}

func (lg *LogGroup) getKeys(data []string) ([]dataPoint, time.Time) {
	y := time.Now().Year()

	tags := lg.extractTags(data)

	nbKeys := lg.getNbKeys()
	dataPoints := make([]dataPoint, nbKeys)

	//Time
	var t time.Time
	if data[lg.date_position] == lg.last_date_str {
		t = lg.last_date
	} else {
		var err error
		t, err = time.Parse(lg.date_format, data[lg.date_position])
		if err != nil {
			log.Print(err)
			var nt time.Time
			return nil, nt
		}
	}

	//Keep time around to only parse new dates
	lg.last_date_str = data[lg.date_position]
	lg.last_date = t

	//Patch in year if missing - rfc3164
	if t.Year() == 0 {
		t = time.Date(y, t.Month(), t.Day(), t.Hour(), t.Minute(),
			t.Second(), t.Nanosecond(), t.Location())
	}

	//Make a first pass extracting the data, applying float->int conversion on multiplier
	values := make([]int64, lg.expected_matches+1)
	for position, keyTypes := range lg.metrics {
		for _, keyType := range keyTypes {
			if position == 0 {
				values[position] = 1
			} else {
				var val int64
				var err error
				if keyType.format == "float" {
					var val_float float64
					if val_float, err = strconv.ParseFloat(data[position], 64); err == nil {
						val = int64(val_float * float64(keyType.multiply))
					}
				} else {
					if val, err = strconv.ParseInt(data[position], 10, 64); err == nil {
						val = val * int64(keyType.multiply)
					}
				}

				if err != nil {
					log.Printf("Unable to extract data from value match, %s: %s", err, data[position])
					var nt time.Time
					return nil, nt
				} else {
					values[position] = val
				}
			}
		}
	}

	//Second pass applies operation and create datapoints
	var i = 0
	for position, val := range values {
		//Is the value a metric?
		for _, keyType := range lg.metrics[position] {
			//Key name
			key := fmt.Sprintf("%s.%s.%s %s %s", lg.key_prefix, keyType.key_suffix, "%s %d %s", strings.Join(tags, " "), keyType.tag)

			//Do we need to do any operation on this val?
			for op, opvalues := range keyType.operations {
				for _, op_position := range opvalues {
					//log.Printf("%s %d on pos %d, current val: %d", op, op_position, position, val)
					if op_position != 0 {
						switch op {
						case "add":
							val += values[op_position]

						case "sub":
							val -= values[op_position]
						}
					}
				}
			}

			if val < 0 && lg.fail_operation_warn {
				log.Printf("Values cannot be negative after applying operation. Offending line: %s", data[0])
				var nt time.Time
				return nil, nt
			}

			dataPoints[i] = dataPoint{name: key, value: val, metric_type: keyType.metric_type}
			i++
		}
	}

	return dataPoints, t
}

func (lg *LogGroup) getStatsKey(hostname string, v int, timePush time.Time, tsd_channel_number int) []string {
	line := make([]string, 1)
	line[0] = fmt.Sprintf("logmetrics_collector.data_pool.key_tracked %d %d host=%s log_group=%s log_group_number=%d", timePush.Unix(), v, hostname, lg.name, tsd_channel_number)

	return line

}

func (lg *LogGroup) dataPoolHandler(channel_number int, tsd_pushers []chan []string, tsd_channel_number int) {
	dataPool := make(map[string]*tsdPoint)
	tsd_push := tsd_pushers[tsd_channel_number]

	hostname := getHostname()

	log.Printf("Datapool[%s:%d] started. Pushing keys to TsdPusher[%d]", lg.name, channel_number, tsd_channel_number)

	//Start the handler
	go func() {

		//Failsafe if anything goes really wrong
		//defer func() {
		//	if r := recover(); r != nil {
		//		log.Printf("Recovered error in %s: %s", lg.name, r)
		//	}
		//}()

		var lastTimePushed *time.Time
		var lastTimeStatsPushed time.Time
		for {
			select {
			case data := <-lg.tail_data[channel_number]:
				data_points, point_time := lg.getKeys(data)

				//To start things off
				if lastTimePushed == nil {
					lastTimePushed = &point_time
				}

				for _, data_point := range data_points {
					//New metrics, add
					if _, ok := dataPool[data_point.name]; !ok {
						switch data_point.metric_type {
						case "histogram":
							s := timemetrics.NewExpDecaySample(point_time, lg.histogram_size, lg.histogram_alpha_decay, lg.histogram_rescale_threshold_min)
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewHistogram(s, lg.stale_treshold_min),
								lastPush: point_time}
						case "counter":
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewCounter(point_time, lg.stale_treshold_min),
								lastPush: point_time}
						case "meter":
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewMeter(point_time, lg.ewma_interval, lg.stale_treshold_min),
								lastPush: point_time, lastCrunchedPush: point_time}
						default:
							log.Fatalf("Unexpected metric type %s!", data_point.metric_type)
						}
					}

					//Make sure data is ordered or we risk sending duplicate data
					if dataPool[data_point.name].lastPush.Unix() > point_time.Unix() && lg.out_of_order_time_warn {
						log.Printf("Non-ordered data detected in log file. Its key already had a update at %s in the future. Offending line: %s",
							dataPool[data_point.name].lastPush, data[0])
					}

					dataPool[data_point.name].data.Update(point_time, data_point.value)
				}

				//Support for log playback - Push when <interval> has pass in the logs, not real time
				if point_time.Sub(*lastTimePushed) >= time.Duration(lg.interval)*time.Second {
					nbKeys := pushKeys(point_time, tsd_push, dataPool)

					//Push stats as well?
					if point_time.Sub(lastTimeStatsPushed) > time.Duration(lg.interval)*time.Second {
						tsd_push <- lg.getStatsKey(hostname, nbKeys, point_time, channel_number)
						lastTimeStatsPushed = point_time
					}

					lastTimePushed = &point_time
				}
			}
		}
	}()
}

func pushKeys(lastTimePushed time.Time, tsd_push chan []string, dataPool map[string]*tsdPoint) (nbKeys int) {
	for tsd_key, tsdPoint := range dataPool {
		if tsdPoint.data.PushKeysTime(tsdPoint.lastPush) {
			tsdPoint.lastPush = tsdPoint.data.GetMaxTime()
			keys := tsdPoint.data.GetKeys(lastTimePushed, tsd_key)
			tsd_push <- keys

			nbKeys += tsdPoint.data.NbKeys()
		}

		if tsdPoint.data.Stale(lastTimePushed) {
			delete(dataPool, tsd_key)
		}
	}

	return
}

func StartDataPools(config *Config, tsd_pushers []chan []string) {
	//Start a queryHandler by log group
	nb_tsd_push := 0
	for _, lg := range config.logGroups {
		for i := 0; i < lg.goroutines; i++ {
			lg.dataPoolHandler(i, tsd_pushers, nb_tsd_push)
			nb_tsd_push = (nb_tsd_push + 1) % config.GetPusherNumber()
		}
	}
}
<commit_msg>Change stale data cleanup behaviour to push the data one last time before deleting en entry.<commit_after>package logmetrics

import (
	"fmt"
	"log"
	"strconv"
	"strings"
	"syseng/go-timemetrics"
	"time"
)

type dataPoint struct {
	name        string
	value       int64
	metric_type string
}

type dataPointTime struct {
	name string
	time int64
}

type tsdPoint struct {
	data             timemetrics.Metric
	lastPush         time.Time
	lastCrunchedPush time.Time
}

func (lg *LogGroup) extractTags(data []string) []string {
	tags := make([]string, lg.getNbTags())

	i := 0

	//General tags
	for tagname, position := range lg.tags {
		tags[i] = fmt.Sprintf("%s=%s", tagname, data[position])
		i++
	}

	return tags
}

func (lg *LogGroup) getKeys(data []string) ([]dataPoint, time.Time) {
	y := time.Now().Year()

	tags := lg.extractTags(data)

	nbKeys := lg.getNbKeys()
	dataPoints := make([]dataPoint, nbKeys)

	//Time
	var t time.Time
	if data[lg.date_position] == lg.last_date_str {
		t = lg.last_date
	} else {
		var err error
		t, err = time.Parse(lg.date_format, data[lg.date_position])
		if err != nil {
			log.Print(err)
			var nt time.Time
			return nil, nt
		}
	}

	//Keep time around to only parse new dates
	lg.last_date_str = data[lg.date_position]
	lg.last_date = t

	//Patch in year if missing - rfc3164
	if t.Year() == 0 {
		t = time.Date(y, t.Month(), t.Day(), t.Hour(), t.Minute(),
			t.Second(), t.Nanosecond(), t.Location())
	}

	//Make a first pass extracting the data, applying float->int conversion on multiplier
	values := make([]int64, lg.expected_matches+1)
	for position, keyTypes := range lg.metrics {
		for _, keyType := range keyTypes {
			if position == 0 {
				values[position] = 1
			} else {
				var val int64
				var err error
				if keyType.format == "float" {
					var val_float float64
					if val_float, err = strconv.ParseFloat(data[position], 64); err == nil {
						val = int64(val_float * float64(keyType.multiply))
					}
				} else {
					if val, err = strconv.ParseInt(data[position], 10, 64); err == nil {
						val = val * int64(keyType.multiply)
					}
				}

				if err != nil {
					log.Printf("Unable to extract data from value match, %s: %s", err, data[position])
					var nt time.Time
					return nil, nt
				} else {
					values[position] = val
				}
			}
		}
	}

	//Second pass applies operation and create datapoints
	var i = 0
	for position, val := range values {
		//Is the value a metric?
		for _, keyType := range lg.metrics[position] {
			//Key name
			key := fmt.Sprintf("%s.%s.%s %s %s", lg.key_prefix, keyType.key_suffix, "%s %d %s", strings.Join(tags, " "), keyType.tag)

			//Do we need to do any operation on this val?
			for op, opvalues := range keyType.operations {
				for _, op_position := range opvalues {
					//log.Printf("%s %d on pos %d, current val: %d", op, op_position, position, val)
					if op_position != 0 {
						switch op {
						case "add":
							val += values[op_position]

						case "sub":
							val -= values[op_position]
						}
					}
				}
			}

			if val < 0 && lg.fail_operation_warn {
				log.Printf("Values cannot be negative after applying operation. Offending line: %s", data[0])
				var nt time.Time
				return nil, nt
			}

			dataPoints[i] = dataPoint{name: key, value: val, metric_type: keyType.metric_type}
			i++
		}
	}

	return dataPoints, t
}

func (lg *LogGroup) getStatsKey(hostname string, v int, timePush time.Time, tsd_channel_number int) []string {
	line := make([]string, 1)
	line[0] = fmt.Sprintf("logmetrics_collector.data_pool.key_tracked %d %d host=%s log_group=%s log_group_number=%d", timePush.Unix(), v, hostname, lg.name, tsd_channel_number)

	return line

}

func (lg *LogGroup) dataPoolHandler(channel_number int, tsd_pushers []chan []string, tsd_channel_number int) {
	dataPool := make(map[string]*tsdPoint)
	tsd_push := tsd_pushers[tsd_channel_number]

	hostname := getHostname()

	log.Printf("Datapool[%s:%d] started. Pushing keys to TsdPusher[%d]", lg.name, channel_number, tsd_channel_number)

	//Start the handler
	go func() {

		//Failsafe if anything goes really wrong
		//defer func() {
		//	if r := recover(); r != nil {
		//		log.Printf("Recovered error in %s: %s", lg.name, r)
		//	}
		//}()

		var lastTimePushed *time.Time
		var lastTimeStatsPushed time.Time
		for {
			select {
			case data := <-lg.tail_data[channel_number]:
				data_points, point_time := lg.getKeys(data)

				//To start things off
				if lastTimePushed == nil {
					lastTimePushed = &point_time
				}

				for _, data_point := range data_points {
					//New metrics, add
					if _, ok := dataPool[data_point.name]; !ok {
						switch data_point.metric_type {
						case "histogram":
							s := timemetrics.NewExpDecaySample(point_time, lg.histogram_size, lg.histogram_alpha_decay, lg.histogram_rescale_threshold_min)
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewHistogram(s, lg.stale_treshold_min),
								lastPush: point_time}
						case "counter":
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewCounter(point_time, lg.stale_treshold_min),
								lastPush: point_time}
						case "meter":
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewMeter(point_time, lg.ewma_interval, lg.stale_treshold_min),
								lastPush: point_time, lastCrunchedPush: point_time}
						default:
							log.Fatalf("Unexpected metric type %s!", data_point.metric_type)
						}
					}

					//Make sure data is ordered or we risk sending duplicate data
					if dataPool[data_point.name].lastPush.Unix() > point_time.Unix() && lg.out_of_order_time_warn {
						log.Printf("Non-ordered data detected in log file. Its key already had a update at %s in the future. Offending line: %s",
							dataPool[data_point.name].lastPush, data[0])
					}

					dataPool[data_point.name].data.Update(point_time, data_point.value)
				}

				//Support for log playback - Push when <interval> has pass in the logs, not real time
				if point_time.Sub(*lastTimePushed) >= time.Duration(lg.interval)*time.Second {
					nbKeys := pushKeys(point_time, tsd_push, dataPool)

					//Push stats as well?
					if point_time.Sub(lastTimeStatsPushed) > time.Duration(lg.interval)*time.Second {
						tsd_push <- lg.getStatsKey(hostname, nbKeys, point_time, channel_number)
						lastTimeStatsPushed = point_time
					}

					lastTimePushed = &point_time
				}
			}
		}
	}()
}

func pushKeys(lastTimePushed time.Time, tsd_push chan []string, dataPool map[string]*tsdPoint) (nbKeys int) {
	for tsd_key, tsdPoint := range dataPool {
		if tsdPoint.data.PushKeysTime(tsdPoint.lastPush) {
			tsdPoint.lastPush = tsdPoint.data.GetMaxTime()
			keys := tsdPoint.data.GetKeys(lastTimePushed, tsd_key)
			tsd_push <- keys

			nbKeys += tsdPoint.data.NbKeys()

			if tsdPoint.data.Stale(lastTimePushed) {
				//Push the key one last time to stabilize rates
				delete(dataPool, tsd_key)
			}
		}

	}

	return
}

func StartDataPools(config *Config, tsd_pushers []chan []string) {
	//Start a queryHandler by log group
	nb_tsd_push := 0
	for _, lg := range config.logGroups {
		for i := 0; i < lg.goroutines; i++ {
			lg.dataPoolHandler(i, tsd_pushers, nb_tsd_push)
			nb_tsd_push = (nb_tsd_push + 1) % config.GetPusherNumber()
		}
	}
}
<|endoftext|>


<commit_before>package controllers

import (
	"fmt"
	"net/http"
)

type MainController struct {
}

func (mc *MainController) GetHandler(responseWriter http.ResponseWriter, request *http.Request, parameters map[string]string) {
	responseWriter.Header().Add("Content-Type", "application/json")
	fmt.Fprintf(responseWriter, `{"status":200, "message": "You reached the server"}`)
}
func (mc *MainController) GetVersionHandler(responseWriter http.ResponseWriter, request *http.Request, parameters map[string]string) {
	responseWriter.Header().Add("Content-Type", "application/json")
	fmt.Fprintf(responseWriter, `{"status":200, "version": "0.1.0.8"}`)
}

func NewMainController() MainController {
	return MainController{}
}
<commit_msg>Version 9<commit_after>package controllers

import (
	"fmt"
	"net/http"
)

type MainController struct {
}

func (mc *MainController) GetHandler(responseWriter http.ResponseWriter, request *http.Request, parameters map[string]string) {
	responseWriter.Header().Add("Content-Type", "application/json")
	fmt.Fprintf(responseWriter, `{"status":200, "message": "You reached the server"}`)
}
func (mc *MainController) GetVersionHandler(responseWriter http.ResponseWriter, request *http.Request, parameters map[string]string) {
	responseWriter.Header().Add("Content-Type", "application/json")
	fmt.Fprintf(responseWriter, `{"status":200, "version": "0.1.0.9"}`)
}

func NewMainController() MainController {
	return MainController{}
}
<|endoftext|>


<commit_before>package fuse

// This file contains the internal logic of the
// FileSystemConnector. The functions for satisfying the raw interface
// are in fsops.go

import (
	"log"
	"path/filepath"
	"strings"
	"sync/atomic"
	"time"
	"unsafe"

	"github.com/hanwen/go-fuse/raw"
)

// Tests should set to true.
var paranoia = false

// FilesystemConnector is a raw FUSE filesystem that manages
// in-process mounts and inodes.  Its job is twofold:
//
// * It translates between the raw kernel interface (padded structs of
// int32 and int64) and the more abstract Go-ish NodeFileSystem
// interface.
//
// * It manages mounting and unmounting of NodeFileSystems into the
// directory hierarchy
//
// To achieve this, the connector only needs a pointer to the root
// node.
type FileSystemConnector struct {
	DefaultRawFileSystem

	Debug bool

	// Callbacks for talking back to the kernel.
	fsInit RawFsInit

	// Translate between uint64 handles and *Inode.
	inodeMap HandleMap

	// The root of the FUSE file system.
	rootNode *Inode

	// Used as the generation inodes.
	generation uint64
}

func NewFileSystemOptions() *FileSystemOptions {
	return &FileSystemOptions{
		NegativeTimeout: 0,
		AttrTimeout:     time.Second,
		EntryTimeout:    time.Second,
		Owner:           CurrentOwner(),
	}
}

func NewFileSystemConnector(nodeFs NodeFileSystem, opts *FileSystemOptions) (c *FileSystemConnector) {
	c = new(FileSystemConnector)
	if opts == nil {
		opts = NewFileSystemOptions()
	}
	c.inodeMap = NewHandleMap(opts.PortableInodes)
	c.rootNode = newInode(true, nodeFs.Root())

	// Make sure we don't reuse generation numbers.
	c.generation = uint64(time.Now().UnixNano())

	c.verify()
	c.MountRoot(nodeFs, opts)

	// FUSE does not issue a LOOKUP for 1 (obviously), but it does
	// issue a forget.  This lookupUpdate is to make the counts match.
	c.lookupUpdate(c.rootNode)

	return c
}

func (c *FileSystemConnector) nextGeneration() uint64 {
	return atomic.AddUint64(&c.generation, 1)
}

func (c *FileSystemConnector) verify() {
	if !paranoia {
		return
	}
	root := c.rootNode
	root.verify(c.rootNode.mountPoint)
}

// Generate EntryOut and increase the lookup count for an inode.
func (c *FileSystemConnector) childLookup(out *raw.EntryOut, fsi FsNode) {
	n := fsi.Inode()
	fsi.GetAttr((*Attr)(&out.Attr), nil, nil)
	n.mount.fillEntry(out)
	out.Ino = c.lookupUpdate(n)
	out.NodeId = out.Ino
	if out.Nlink == 0 {
		// With Nlink == 0, newer kernels will refuse link
		// operations.
		out.Nlink = 1
	}
}

func (c *FileSystemConnector) findMount(parent *Inode, name string) (mount *fileSystemMount) {
	parent.treeLock.RLock()
	if parent.mounts != nil {
		mount = parent.mounts[name]
	}
	parent.treeLock.RUnlock()

	return
}

func (c *FileSystemConnector) toInode(nodeid uint64) *Inode {
	if nodeid == raw.FUSE_ROOT_ID {
		return c.rootNode
	}
	i := (*Inode)(unsafe.Pointer(c.inodeMap.Decode(nodeid)))
	return i
}

// Must run outside treeLock.  Returns the nodeId.
func (c *FileSystemConnector) lookupUpdate(node *Inode) uint64 {
	node.treeLock.Lock()
	if node.lookupCount == 0 {
		node.nodeId = c.inodeMap.Register(&node.handled, node)
	}
	node.lookupCount += 1
	node.treeLock.Unlock()

	return node.nodeId
}

// Must run outside treeLock.
func (c *FileSystemConnector) forgetUpdate(node *Inode, forgetCount int) {
	defer c.verify()

	node.treeLock.Lock()
	node.lookupCount -= forgetCount
	if node.lookupCount == 0 {
		c.inodeMap.Forget(node.nodeId)
		node.nodeId = 0
	} else if node.lookupCount < 0 {
		log.Panicf("lookupCount underflow: %d: %v", node.lookupCount, c)
	}

	c.recursiveConsiderDropInode(node)
	node.treeLock.Unlock()
}

// InodeCount returns the number of inodes registered with the kernel.
func (c *FileSystemConnector) InodeHandleCount() int {
	return c.inodeMap.Count()
}

// Must hold treeLock.

func (c *FileSystemConnector) recursiveConsiderDropInode(n *Inode) (drop bool) {
	delChildren := []string{}
	for k, v := range n.children {
		// Only consider children from the same mount, or
		// already unmounted mountpoints.
		if v.mountPoint == nil && c.recursiveConsiderDropInode(v) {
			delChildren = append(delChildren, k)
		}
	}
	for _, k := range delChildren {
		ch := n.rmChild(k)
		if ch == nil {
			log.Panicf("trying to del child %q, but not present", k)
		}
		ch.fsInode.OnForget()
	}

	if len(n.children) > 0 || n.lookupCount > 0 || !n.FsNode().Deletable() {
		return false
	}
	if n == c.rootNode || n.mountPoint != nil {
		return false
	}

	n.openFilesMutex.Lock()
	ok := len(n.openFiles) == 0
	n.openFilesMutex.Unlock()

	return ok
}

// Finds a node within the currently known inodes, returns the last
// known node and the remaining unknown path components.  If parent is
// nil, start from FUSE mountpoint.
func (c *FileSystemConnector) Node(parent *Inode, fullPath string) (*Inode, []string) {
	if parent == nil {
		parent = c.rootNode
	}
	if fullPath == "" {
		return parent, nil
	}

	sep := string(filepath.Separator)
	fullPath = strings.TrimLeft(filepath.Clean(fullPath), sep)
	comps := strings.Split(fullPath, sep)

	node := parent
	if node.mountPoint == nil {
		node.treeLock.RLock()
		defer node.treeLock.RUnlock()
	}

	for i, component := range comps {
		if len(component) == 0 {
			continue
		}

		if node.mountPoint != nil {
			node.treeLock.RLock()
			defer node.treeLock.RUnlock()
		}

		next := node.children[component]
		if next == nil {
			return node, comps[i:]
		}
		node = next
	}

	return node, nil
}

func (c *FileSystemConnector) LookupNode(parent *Inode, path string) *Inode {
	if path == "" {
		return parent
	}
	components := strings.Split(path, "/")
	for _, r := range components {
		var a Attr
		child, _ := c.internalLookup(&a, parent, r, nil)
		if child == nil {
			return nil
		}

		parent = child
	}

	return parent
}

////////////////////////////////////////////////////////////////

func (c *FileSystemConnector) MountRoot(nodeFs NodeFileSystem, opts *FileSystemOptions) {
	c.rootNode.mountFs(nodeFs, opts)
	c.rootNode.mount.connector = c
	nodeFs.OnMount(c)
	c.verify()
}

// Mount() generates a synthetic directory node, and mounts the file
// system there.  If opts is nil, the mount options of the root file
// system are inherited.  The encompassing filesystem should pretend
// the mount point does not exist.  If it does, it will generate an
// Inode with the same, which will cause Mount() to return EBUSY.
//
// Return values:
//
// ENOENT: the directory containing the mount point does not exist.
//
// EBUSY: the intended mount point already exists.
func (c *FileSystemConnector) Mount(parent *Inode, name string, nodeFs NodeFileSystem, opts *FileSystemOptions) Status {
	parent.treeLock.Lock()
	defer parent.treeLock.Unlock()
	node := parent.children[name]
	if node != nil {
		return EBUSY
	}

	node = newInode(true, nodeFs.Root())
	if opts == nil {
		opts = c.rootNode.mountPoint.options
	}

	node.mountFs(nodeFs, opts)
	node.mount.connector = c
	parent.addChild(name, node)

	if parent.mounts == nil {
		parent.mounts = make(map[string]*fileSystemMount)
	}
	parent.mounts[name] = node.mountPoint
	node.mountPoint.parentInode = parent
	if c.Debug {
		log.Println("Mount: ", nodeFs, "on subdir", name,
			"parent", parent.nodeId)
	}
	c.verify()
	nodeFs.OnMount(c)
	return OK
}

// Unmount() tries to unmount the given inode.
//
// Returns the following error codes:
//
// EINVAL: path does not exist, or is not a mount point.
//
// EBUSY: there are open files, or submounts below this node.
func (c *FileSystemConnector) Unmount(node *Inode) Status {
	if node.mountPoint == nil {
		log.Println("not a mountpoint:", node.nodeId)
		return EINVAL
	}

	// Must lock parent to update tree structure.
	parentNode := node.mountPoint.parentInode
	parentNode.treeLock.Lock()
	defer parentNode.treeLock.Unlock()

	mount := node.mountPoint
	name := node.mountPoint.mountName()
	if mount.openFiles.Count() > 0 {
		return EBUSY
	}

	mountInode := mount.mountInode
	if !mountInode.canUnmount() {
		return EBUSY
	}

	mount.mountInode = nil
	mountInode.mountPoint = nil

	delete(parentNode.mounts, name)
	delete(parentNode.children, name)
	mount.fs.OnUnmount()

	c.EntryNotify(parentNode, name)

	return OK
}

func (c *FileSystemConnector) FileNotify(node *Inode, off int64, length int64) Status {
	n := node.nodeId
	if node == c.rootNode {
		n = raw.FUSE_ROOT_ID
	}
	if n == 0 {
		return OK
	}
	out := raw.NotifyInvalInodeOut{
		Length: length,
		Off:    off,
		Ino:    n,
	}
	return c.fsInit.InodeNotify(&out)
}

func (c *FileSystemConnector) EntryNotify(dir *Inode, name string) Status {
	n := dir.nodeId
	if dir == c.rootNode {
		n = raw.FUSE_ROOT_ID
	}
	if n == 0 {
		return OK
	}
	return c.fsInit.EntryNotify(n, name)
}
<commit_msg>Fix race condition: canUnmount needs tree to be locked.<commit_after>package fuse

// This file contains the internal logic of the
// FileSystemConnector. The functions for satisfying the raw interface
// are in fsops.go

import (
	"log"
	"path/filepath"
	"strings"
	"sync/atomic"
	"time"
	"unsafe"

	"github.com/hanwen/go-fuse/raw"
)

// Tests should set to true.
var paranoia = false

// FilesystemConnector is a raw FUSE filesystem that manages
// in-process mounts and inodes.  Its job is twofold:
//
// * It translates between the raw kernel interface (padded structs of
// int32 and int64) and the more abstract Go-ish NodeFileSystem
// interface.
//
// * It manages mounting and unmounting of NodeFileSystems into the
// directory hierarchy
//
// To achieve this, the connector only needs a pointer to the root
// node.
type FileSystemConnector struct {
	DefaultRawFileSystem

	Debug bool

	// Callbacks for talking back to the kernel.
	fsInit RawFsInit

	// Translate between uint64 handles and *Inode.
	inodeMap HandleMap

	// The root of the FUSE file system.
	rootNode *Inode

	// Used as the generation inodes.
	generation uint64
}

func NewFileSystemOptions() *FileSystemOptions {
	return &FileSystemOptions{
		NegativeTimeout: 0,
		AttrTimeout:     time.Second,
		EntryTimeout:    time.Second,
		Owner:           CurrentOwner(),
	}
}

func NewFileSystemConnector(nodeFs NodeFileSystem, opts *FileSystemOptions) (c *FileSystemConnector) {
	c = new(FileSystemConnector)
	if opts == nil {
		opts = NewFileSystemOptions()
	}
	c.inodeMap = NewHandleMap(opts.PortableInodes)
	c.rootNode = newInode(true, nodeFs.Root())

	// Make sure we don't reuse generation numbers.
	c.generation = uint64(time.Now().UnixNano())

	c.verify()
	c.MountRoot(nodeFs, opts)

	// FUSE does not issue a LOOKUP for 1 (obviously), but it does
	// issue a forget.  This lookupUpdate is to make the counts match.
	c.lookupUpdate(c.rootNode)

	return c
}

func (c *FileSystemConnector) nextGeneration() uint64 {
	return atomic.AddUint64(&c.generation, 1)
}

func (c *FileSystemConnector) verify() {
	if !paranoia {
		return
	}
	root := c.rootNode
	root.verify(c.rootNode.mountPoint)
}

// Generate EntryOut and increase the lookup count for an inode.
func (c *FileSystemConnector) childLookup(out *raw.EntryOut, fsi FsNode) {
	n := fsi.Inode()
	fsi.GetAttr((*Attr)(&out.Attr), nil, nil)
	n.mount.fillEntry(out)
	out.Ino = c.lookupUpdate(n)
	out.NodeId = out.Ino
	if out.Nlink == 0 {
		// With Nlink == 0, newer kernels will refuse link
		// operations.
		out.Nlink = 1
	}
}

func (c *FileSystemConnector) findMount(parent *Inode, name string) (mount *fileSystemMount) {
	parent.treeLock.RLock()
	if parent.mounts != nil {
		mount = parent.mounts[name]
	}
	parent.treeLock.RUnlock()

	return
}

func (c *FileSystemConnector) toInode(nodeid uint64) *Inode {
	if nodeid == raw.FUSE_ROOT_ID {
		return c.rootNode
	}
	i := (*Inode)(unsafe.Pointer(c.inodeMap.Decode(nodeid)))
	return i
}

// Must run outside treeLock.  Returns the nodeId.
func (c *FileSystemConnector) lookupUpdate(node *Inode) uint64 {
	node.treeLock.Lock()
	if node.lookupCount == 0 {
		node.nodeId = c.inodeMap.Register(&node.handled, node)
	}
	node.lookupCount += 1
	node.treeLock.Unlock()

	return node.nodeId
}

// Must run outside treeLock.
func (c *FileSystemConnector) forgetUpdate(node *Inode, forgetCount int) {
	defer c.verify()

	node.treeLock.Lock()
	node.lookupCount -= forgetCount
	if node.lookupCount == 0 {
		c.inodeMap.Forget(node.nodeId)
		node.nodeId = 0
	} else if node.lookupCount < 0 {
		log.Panicf("lookupCount underflow: %d: %v", node.lookupCount, c)
	}

	c.recursiveConsiderDropInode(node)
	node.treeLock.Unlock()
}

// InodeCount returns the number of inodes registered with the kernel.
func (c *FileSystemConnector) InodeHandleCount() int {
	return c.inodeMap.Count()
}

// Must hold treeLock.

func (c *FileSystemConnector) recursiveConsiderDropInode(n *Inode) (drop bool) {
	delChildren := []string{}
	for k, v := range n.children {
		// Only consider children from the same mount, or
		// already unmounted mountpoints.
		if v.mountPoint == nil && c.recursiveConsiderDropInode(v) {
			delChildren = append(delChildren, k)
		}
	}
	for _, k := range delChildren {
		ch := n.rmChild(k)
		if ch == nil {
			log.Panicf("trying to del child %q, but not present", k)
		}
		ch.fsInode.OnForget()
	}

	if len(n.children) > 0 || n.lookupCount > 0 || !n.FsNode().Deletable() {
		return false
	}
	if n == c.rootNode || n.mountPoint != nil {
		return false
	}

	n.openFilesMutex.Lock()
	ok := len(n.openFiles) == 0
	n.openFilesMutex.Unlock()

	return ok
}

// Finds a node within the currently known inodes, returns the last
// known node and the remaining unknown path components.  If parent is
// nil, start from FUSE mountpoint.
func (c *FileSystemConnector) Node(parent *Inode, fullPath string) (*Inode, []string) {
	if parent == nil {
		parent = c.rootNode
	}
	if fullPath == "" {
		return parent, nil
	}

	sep := string(filepath.Separator)
	fullPath = strings.TrimLeft(filepath.Clean(fullPath), sep)
	comps := strings.Split(fullPath, sep)

	node := parent
	if node.mountPoint == nil {
		node.treeLock.RLock()
		defer node.treeLock.RUnlock()
	}

	for i, component := range comps {
		if len(component) == 0 {
			continue
		}

		if node.mountPoint != nil {
			node.treeLock.RLock()
			defer node.treeLock.RUnlock()
		}

		next := node.children[component]
		if next == nil {
			return node, comps[i:]
		}
		node = next
	}

	return node, nil
}

func (c *FileSystemConnector) LookupNode(parent *Inode, path string) *Inode {
	if path == "" {
		return parent
	}
	components := strings.Split(path, "/")
	for _, r := range components {
		var a Attr
		child, _ := c.internalLookup(&a, parent, r, nil)
		if child == nil {
			return nil
		}

		parent = child
	}

	return parent
}

////////////////////////////////////////////////////////////////

func (c *FileSystemConnector) MountRoot(nodeFs NodeFileSystem, opts *FileSystemOptions) {
	c.rootNode.mountFs(nodeFs, opts)
	c.rootNode.mount.connector = c
	nodeFs.OnMount(c)
	c.verify()
}

// Mount() generates a synthetic directory node, and mounts the file
// system there.  If opts is nil, the mount options of the root file
// system are inherited.  The encompassing filesystem should pretend
// the mount point does not exist.  If it does, it will generate an
// Inode with the same, which will cause Mount() to return EBUSY.
//
// Return values:
//
// ENOENT: the directory containing the mount point does not exist.
//
// EBUSY: the intended mount point already exists.
func (c *FileSystemConnector) Mount(parent *Inode, name string, nodeFs NodeFileSystem, opts *FileSystemOptions) Status {
	parent.treeLock.Lock()
	defer parent.treeLock.Unlock()
	node := parent.children[name]
	if node != nil {
		return EBUSY
	}

	node = newInode(true, nodeFs.Root())
	if opts == nil {
		opts = c.rootNode.mountPoint.options
	}

	node.mountFs(nodeFs, opts)
	node.mount.connector = c
	parent.addChild(name, node)

	if parent.mounts == nil {
		parent.mounts = make(map[string]*fileSystemMount)
	}
	parent.mounts[name] = node.mountPoint
	node.mountPoint.parentInode = parent
	if c.Debug {
		log.Println("Mount: ", nodeFs, "on subdir", name,
			"parent", parent.nodeId)
	}
	c.verify()
	nodeFs.OnMount(c)
	return OK
}

// Unmount() tries to unmount the given inode.
//
// Returns the following error codes:
//
// EINVAL: path does not exist, or is not a mount point.
//
// EBUSY: there are open files, or submounts below this node.
func (c *FileSystemConnector) Unmount(node *Inode) Status {
	if node.mountPoint == nil {
		log.Println("not a mountpoint:", node.nodeId)
		return EINVAL
	}

	// Must lock parent to update tree structure.
	parentNode := node.mountPoint.parentInode
	parentNode.treeLock.Lock()
	defer parentNode.treeLock.Unlock()

	mount := node.mountPoint
	name := node.mountPoint.mountName()
	if mount.openFiles.Count() > 0 {
		return EBUSY
	}

	node.treeLock.Lock()
	defer node.treeLock.Unlock()

	mountInode := mount.mountInode
	if !mountInode.canUnmount() {
		return EBUSY
	}

	mount.mountInode = nil
	mountInode.mountPoint = nil

	delete(parentNode.mounts, name)
	delete(parentNode.children, name)
	mount.fs.OnUnmount()

	c.EntryNotify(parentNode, name)

	return OK
}

func (c *FileSystemConnector) FileNotify(node *Inode, off int64, length int64) Status {
	n := node.nodeId
	if node == c.rootNode {
		n = raw.FUSE_ROOT_ID
	}
	if n == 0 {
		return OK
	}
	out := raw.NotifyInvalInodeOut{
		Length: length,
		Off:    off,
		Ino:    n,
	}
	return c.fsInit.InodeNotify(&out)
}

func (c *FileSystemConnector) EntryNotify(dir *Inode, name string) Status {
	n := dir.nodeId
	if dir == c.rootNode {
		n = raw.FUSE_ROOT_ID
	}
	if n == 0 {
		return OK
	}
	return c.fsInit.EntryNotify(n, name)
}
<|endoftext|>


<commit_before>package pigosat

import "testing"
import "time"

// abs takes the absolute value of an int32 and casts it to int.
func abs(x int32) int {
	if x < 0 {
		return int(-x)
	}
	return int(x)
}

func equal(x, y []bool) bool {
	if len(x) != len(y) {
		return false
	}
	for i := 0; i < len(x); i++ {
		if x[i] != y[i] {
			return false
		}
	}
	return true
}

// Evaluate a formula when the variables take on the values given by the
// solution.
func evaluate(formula [][]int32, solution []bool) bool {
	var c bool // The value for the clause
	for _, clause := range formula {
		c = false
		if len(clause) == 0 {
			continue
		}
		for _, literal := range clause {
			if literal > 0 && solution[abs(literal)] ||
				literal < 0 && !solution[abs(literal)] {
				c = true
				break
			}
		}
		if !c {
			return false
		}
	}
	return true
}

type formulaTest struct {
	formula   [][]int32
	variables int // count
	clauses   int // count
	status    int
	expected  []bool // solution
}

// The first three tests are cribbed from Ilan Schnell's Pycosat. See
// https://github.com/ContinuumIO/pycosat. In particular, these are from commit
// d81df1e in test_pycosat.py.
var formulaTests = []formulaTest{
	{[][]int32{{1, -5, 4}, {-1, 5, 3, 4}, {-3, -4}},
		5, 3, Satisfiable,
		[]bool{false, true, false, false, false, true}},
	{[][]int32{{-1}, {1}},
		1, 2, Unsatisfiable,
		nil},
	{[][]int32{{-1, 2}, {-1, -2}, {1, -2}},
		2, 3, Satisfiable,
		[]bool{false, false, false}},
	// For testing that empty clauses are skipped and 0s end clauses
	{[][]int32{{1, -5, 4, 0, 9}, {-1, 5, 3, 4, 0, 100}, {}, {-3, -4, 0}, nil},
		5, 3, Satisfiable,
		[]bool{false, true, false, false, false, true}},
}

// Ensure our expected solutions are correct.
func init() {
	for i, ft := range formulaTests {
		if ft.status == Satisfiable && !evaluate(ft.formula, ft.expected) {
			panic(i)
		}
	}
}

func wasExpected(t *testing.T, i int, p *Picosat, ft *formulaTest, status int,
	solution []bool) {
	if status != ft.status {
		t.Errorf("Test %d: Expected status %d but got %d", i, ft.status, status)
	}
	if !equal(solution, ft.expected) {
		t.Errorf("Test %d: Expected solution %v but got %v", i, ft.expected,
			solution)
	}
	if p.Variables() != ft.variables {
		t.Errorf("Test %d: Expected %d variables, got %d", i, ft.variables,
			p.Variables())
	}
	if p.AddedOriginalClauses() != ft.clauses {
		t.Errorf("Test %d: Exepcted %d clauses, got %d", i, ft.clauses,
			p.AddedOriginalClauses())
	}
	if s := p.Seconds(); s <= 0 || s > time.Millisecond {
		t.Errorf("Test %d: Test took a suspicious amount of time: %v", i, s)
	}
}

func TestFormulas(t *testing.T) {
	var p *Picosat
	var status int
	var solution []bool
	for i, ft := range formulaTests {
		p = NewPicosat(0)
		p.AddClauses(ft.formula)
		status, solution = p.Solve()
		wasExpected(t, i, p, &ft, status, solution)
		p.Delete()
	}
}

// Also cribbed from Pycosat
func TestPropLimit(t *testing.T) {
	var p *Picosat
	var status int
	var solution []bool
	ft := formulaTests[0]
	var limit uint64
	for limit = 1; limit < 20; limit++ {
		p = NewPicosat(limit)
		p.AddClauses(ft.formula)
		status, solution = p.Solve()
		if limit < 8 {
			if status != Unknown {
				t.Errorf("Propagation limit %d had no effect on formula 0",
					limit)
			}
			p.Delete()
			continue
		}
		wasExpected(t, 0, p, &ft, status, solution)
		p.Delete()
	}
}

// Test that nil method calls are no ops
func TestNil(t *testing.T) {
	var a, b *Picosat
	b = NewPicosat(0)
	b.Delete()
	for name, p := range map[string]*Picosat{"uninit": a, "deleted": b} {
		// No panicking
		p.Delete()
		p.AddClauses([][]int32{{1}, {2}})
		if p.Variables() != 0 {
			t.Errorf("Test %s: Expected 0 vars, got %d", name, p.Variables())
		}
		if c := p.AddedOriginalClauses(); c != 0 {
			t.Errorf("Test %s: Expected 0 clauses, got %d", name, c)
		}
		if p.Seconds() != 0 {
			t.Errorf("Test %s: Expected 0 seconds, got %v", name, p.Seconds())
		}
		if status, solution := p.Solve(); status != NotReady || solution != nil {
			t.Errorf("Test %s: Expected status %d and nil solution, got %d and %v",
				name, status, solution)
		}
	}
}
<commit_msg>Readability improvements in tests<commit_after>package pigosat

import "testing"
import "time"

// abs takes the absolute value of an int32 and casts it to int.
func abs(x int32) int {
	if x < 0 {
		return int(-x)
	}
	return int(x)
}

func equal(x, y []bool) bool {
	if len(x) != len(y) {
		return false
	}
	for i := 0; i < len(x); i++ {
		if x[i] != y[i] {
			return false
		}
	}
	return true
}

// Evaluate a formula when the variables take on the values given by the
// solution.
func evaluate(formula [][]int32, solution []bool) bool {
	var c bool // The value for the clause
	for _, clause := range formula {
		c = false
		if len(clause) == 0 {
			continue
		}
		for _, literal := range clause {
			if literal > 0 && solution[abs(literal)] ||
				literal < 0 && !solution[abs(literal)] {
				c = true
				break
			}
		}
		if !c {
			return false
		}
	}
	return true
}

type formulaTest struct {
	formula   [][]int32
	variables int // count
	clauses   int // count
	status    int
	expected  []bool // solution
}

var formulaTests = []formulaTest{
	// The first three tests are cribbed from Ilan Schnell's Pycosat. See
	// https://github.com/ContinuumIO/pycosat. In particular, these are from
	// commit d81df1e in test_pycosat.py.
	0: {[][]int32{{1, -5, 4}, {-1, 5, 3, 4}, {-3, -4}},
		5, 3, Satisfiable,
		[]bool{false, true, false, false, false, true}},
	1: {[][]int32{{-1}, {1}},
		1, 2, Unsatisfiable,
		nil},
	2: {[][]int32{{-1, 2}, {-1, -2}, {1, -2}},
		2, 3, Satisfiable,
		[]bool{false, false, false}},
	// For testing that empty clauses are skipped and 0s end clauses
	3: {[][]int32{{1, -5, 4, 0, 9}, {-1, 5, 3, 4, 0, 100}, {}, {-3, -4, 0}, nil},
		5, 3, Satisfiable,
		[]bool{false, true, false, false, false, true}},
}

// Ensure our expected solutions are correct.
func init() {
	for i, ft := range formulaTests {
		if ft.status == Satisfiable && !evaluate(ft.formula, ft.expected) {
			panic(i)
		}
	}
}

func wasExpected(t *testing.T, i int, p *Picosat, ft *formulaTest, status int,
	solution []bool) {
	if status != ft.status {
		t.Errorf("Test %d: Expected status %d but got %d", i, ft.status, status)
	}
	if !equal(solution, ft.expected) {
		t.Errorf("Test %d: Expected solution %v but got %v", i, ft.expected,
			solution)
	}
	if p.Variables() != ft.variables {
		t.Errorf("Test %d: Expected %d variables, got %d", i, ft.variables,
			p.Variables())
	}
	if p.AddedOriginalClauses() != ft.clauses {
		t.Errorf("Test %d: Exepcted %d clauses, got %d", i, ft.clauses,
			p.AddedOriginalClauses())
	}
	if s := p.Seconds(); s <= 0 || s > time.Millisecond {
		t.Errorf("Test %d: Test took a suspicious amount of time: %v", i, s)
	}
}

func TestFormulas(t *testing.T) {
	var p *Picosat
	var status int
	var solution []bool
	for i, ft := range formulaTests {
		p = NewPicosat(0)
		p.AddClauses(ft.formula)
		status, solution = p.Solve()
		wasExpected(t, i, p, &ft, status, solution)
		p.Delete()
	}
}

// Also cribbed from Pycosat
func TestPropLimit(t *testing.T) {
	var p *Picosat
	var status int
	var solution []bool
	ft := formulaTests[0]
	var limit uint64
	for limit = 1; limit < 20; limit++ {
		p = NewPicosat(limit)
		p.AddClauses(ft.formula)
		status, solution = p.Solve()
		if limit < 8 {
			if status != Unknown {
				t.Errorf("Propagation limit %d had no effect on formula 0",
					limit)
			}
			p.Delete()
			continue
		}
		wasExpected(t, 0, p, &ft, status, solution)
		p.Delete()
	}
}

// Test that nil method calls are no ops
func TestNil(t *testing.T) {
	var a, b *Picosat
	b = NewPicosat(0)
	b.Delete()
	for name, p := range map[string]*Picosat{"uninit": a, "deleted": b} {
		// No panicking
		p.Delete()
		p.AddClauses([][]int32{{1}, {2}})
		if p.Variables() != 0 {
			t.Errorf("Test %s: Expected 0 vars, got %d", name, p.Variables())
		}
		if c := p.AddedOriginalClauses(); c != 0 {
			t.Errorf("Test %s: Expected 0 clauses, got %d", name, c)
		}
		if p.Seconds() != 0 {
			t.Errorf("Test %s: Expected 0 seconds, got %v", name, p.Seconds())
		}
		if status, solution := p.Solve(); status != NotReady || solution != nil {
			t.Errorf("Test %s: Expected status %d and nil solution, got %d and %v",
				name, status, solution)
		}
	}
}
<|endoftext|>


<commit_before>package radosAPI

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"github.com/smartystreets/go-aws-auth"
)

type API struct {
	host      string
	accessKey string
	secretKey string
	prefix    string
}

// New returns client for Ceph RADOS Gateway
func New(host, accessKey, secretKey string, adminPrefix ...string) *API {
	prefix := "admin"
	if len(adminPrefix) > 0 {
		prefix = adminPrefix[0]
	}
	return &API{host, accessKey, secretKey, prefix}
}

func (api *API) makeRequest(verb, url string) (body []byte, statusCode int, err error) {
	var apiErr apiError
	client := http.Client{}

	// fmt.Printf("URL [%v]: %v\n", verb, url)
	req, err := http.NewRequest(verb, url, nil)
	if err != nil {
		return
	}
	awsauth.SignS3(req, awsauth.Credentials{
		AccessKeyID:     api.accessKey,
		SecretAccessKey: api.secretKey,
		Expiration:      time.Now().Add(1 * time.Minute)},
	)
	resp, err := client.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return
	}
	statusCode = resp.StatusCode
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	if errMarshal := json.Unmarshal(body, &apiErr); errMarshal == nil && apiErr.Code != "" {
		err = errors.New(apiErr.Code)
	}
	return
}

func (api *API) get(route string, args url.Values, sub ...string) (body []byte, statusCode int, err error) {
	subreq := ""
	if len(sub) > 0 {
		subreq = sub[0] + "&"
	}
	body, statusCode, err = api.makeRequest("GET", fmt.Sprintf("%v%v?%v%s", api.host, route, subreq, args.Encode()))
	if statusCode != 200 {
		err = fmt.Errorf("[%v]: %v", statusCode, err)
	}
	return
}

func (api *API) delete(route string, args url.Values, sub ...string) (body []byte, statusCode int, err error) {
	subreq := ""
	if len(sub) > 0 {
		subreq = sub[0] + "&"
	}
	body, statusCode, err = api.makeRequest("DELETE", fmt.Sprintf("%v%v?%v%s", api.host, route, subreq, args.Encode()))
	if statusCode != 200 {
		err = fmt.Errorf("[%v]: %v", statusCode, err)
	}
	return
}

func (api *API) put(route string, args url.Values, sub ...string) (body []byte, statusCode int, err error) {
	subreq := ""
	if len(sub) > 0 {
		subreq = sub[0] + "&"
	}
	body, statusCode, err = api.makeRequest("PUT", fmt.Sprintf("%v%v?%v%s", api.host, route, subreq, args.Encode()))
	if statusCode != 200 {
		err = fmt.Errorf("[%v]: %v", statusCode, err)
	}
	return
}

func (api *API) post(route string, args url.Values, sub ...string) (body []byte, statusCode int, err error) {
	subreq := ""
	if len(sub) > 0 {
		subreq = sub[0] + "&"
	}
	body, statusCode, err = api.makeRequest("POST", fmt.Sprintf("%v%v?%v%s", api.host, route, subreq, args.Encode()))
	if statusCode != 200 {
		err = fmt.Errorf("[%v]: %v", statusCode, err)
	}
	return
}
<commit_msg>Remove duplicate code<commit_after>package radosAPI

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"github.com/smartystreets/go-aws-auth"
)

type API struct {
	host      string
	accessKey string
	secretKey string
	prefix    string
}

// New returns client for Ceph RADOS Gateway
func New(host, accessKey, secretKey string, adminPrefix ...string) *API {
	prefix := "admin"
	if len(adminPrefix) > 0 {
		prefix = adminPrefix[0]
	}
	return &API{host, accessKey, secretKey, prefix}
}

func (api *API) makeRequest(verb, url string) (body []byte, statusCode int, err error) {
	var apiErr apiError
	client := http.Client{}

	// fmt.Printf("URL [%v]: %v\n", verb, url)
	req, err := http.NewRequest(verb, url, nil)
	if err != nil {
		return
	}
	awsauth.SignS3(req, awsauth.Credentials{
		AccessKeyID:     api.accessKey,
		SecretAccessKey: api.secretKey,
		Expiration:      time.Now().Add(1 * time.Minute)},
	)
	resp, err := client.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return
	}
	statusCode = resp.StatusCode
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	if errMarshal := json.Unmarshal(body, &apiErr); errMarshal == nil && apiErr.Code != "" {
		err = errors.New(apiErr.Code)
	}
	return
}

func (api *API) call(verb, route string, args url.Values, usePrefix bool, sub ...string) (body []byte, statusCode int, err error) {
	subreq := ""
	if len(sub) > 0 {
		subreq = sub[0] + "&"
	}
	if usePrefix {
		route = "/" + api.prefix + route
	}
	body, statusCode, err = api.makeRequest(verb, fmt.Sprintf("%v%v?%v%s", api.host, route, subreq, args.Encode()))
	if statusCode != 200 {
		err = fmt.Errorf("[%v]: %v", statusCode, err)
	}
	return
}
<|endoftext|>


<commit_before>package log

import (
	"io"
	"os"

	"log/syslog"

	"github.com/sirupsen/logrus"
	ls "github.com/sirupsen/logrus/hooks/syslog"
)

func New(logLevel string, logServer string, logOutput string) *logrus.Logger {
	var level logrus.Level
	var output io.Writer
	var hook logrus.Hook

	switch logOutput {
	case "stdout":
		output = os.Stdout
	case "stderr":
		output = os.Stderr
	case "syslog":
		output = os.Stderr // does not matter ?
		if logServer == "" {
			panic("syslog output needs a log server (ie. 127.0.0.1:514)")
		}
		hook, _ = ls.NewSyslogHook("udp", logServer, syslog.LOG_INFO, "kube-alert")
	default:
		output = os.Stderr
	}

	switch logLevel {
	case "debug":
		level = logrus.DebugLevel
	case "info":
		level = logrus.InfoLevel
	case "warning":
		level = logrus.WarnLevel
	case "error":
		level = logrus.ErrorLevel
	case "fatal":
		level = logrus.FatalLevel
	case "panic":
		level = logrus.PanicLevel
	default:
		level = logrus.InfoLevel
	}

	formater := &logrus.TextFormatter{
		FullTimestamp: true,
		TimestampFormat: "2006-01-02 15:04:05",
	}

	log := &logrus.Logger{
		Out:       output,
		Formatter: formater,
		Hooks:     make(logrus.LevelHooks),
		Level:     level,
	}

	if logOutput == "syslog" {
		log.Hooks.Add(hook)
	}

	return log
}
<commit_msg>go fmt<commit_after>package log

import (
	"io"
	"os"

	"log/syslog"

	"github.com/sirupsen/logrus"
	ls "github.com/sirupsen/logrus/hooks/syslog"
)

func New(logLevel string, logServer string, logOutput string) *logrus.Logger {
	var level logrus.Level
	var output io.Writer
	var hook logrus.Hook

	switch logOutput {
	case "stdout":
		output = os.Stdout
	case "stderr":
		output = os.Stderr
	case "syslog":
		output = os.Stderr // does not matter ?
		if logServer == "" {
			panic("syslog output needs a log server (ie. 127.0.0.1:514)")
		}
		hook, _ = ls.NewSyslogHook("udp", logServer, syslog.LOG_INFO, "kube-alert")
	default:
		output = os.Stderr
	}

	switch logLevel {
	case "debug":
		level = logrus.DebugLevel
	case "info":
		level = logrus.InfoLevel
	case "warning":
		level = logrus.WarnLevel
	case "error":
		level = logrus.ErrorLevel
	case "fatal":
		level = logrus.FatalLevel
	case "panic":
		level = logrus.PanicLevel
	default:
		level = logrus.InfoLevel
	}

	formater := &logrus.TextFormatter{
		FullTimestamp:   true,
		TimestampFormat: "2006-01-02 15:04:05",
	}

	log := &logrus.Logger{
		Out:       output,
		Formatter: formater,
		Hooks:     make(logrus.LevelHooks),
		Level:     level,
	}

	if logOutput == "syslog" {
		log.Hooks.Add(hook)
	}

	return log
}
<|endoftext|>


<commit_before>package sse

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"sync"
)

type SSEWriter interface {
	Write([]byte) (int, error)
	Flush()
}

func NewSSEWriter(w io.Writer) SSEWriter {
	return &Writer{Writer: w}
}

type Writer struct {
	io.Writer
	sync.Mutex
}

func (w *Writer) Write(p []byte) (int, error) {
	w.Lock()
	defer w.Unlock()
	for _, line := range bytes.Split(p, []byte("\n")) {
		if _, err := fmt.Fprintf(w.Writer, "data: %s\n", line); err != nil {
			return 0, err
		}
	}
	// add a terminating newline
	_, err := w.Writer.Write([]byte("\n"))
	return len(p), err
}

func (w *Writer) Flush() {
	if fw, ok := w.Writer.(http.Flusher); ok {
		fw.Flush()
	}
}

type Reader struct {
	*bufio.Reader
}

func (r *Reader) Read() ([]byte, error) {
	for {
		line, err := r.ReadBytes('\n')
		if err != nil {
			return nil, err
		}
		if bytes.HasPrefix(line, []byte("data: ")) {
			data := bytes.TrimSuffix(bytes.TrimPrefix(line, []byte("data: ")), []byte("\n"))
			return data, nil
		}
	}
}

type Decoder struct {
	*Reader
}

func NewDecoder(r *bufio.Reader) *Decoder {
	return &Decoder{&Reader{r}}
}

// Decode finds the next "data" field and decodes it into v
func (dec *Decoder) Decode(v interface{}) error {
	data, err := dec.Reader.Read()
	if err != nil {
		return err
	}
	return json.Unmarshal(data, v)
}
<commit_msg>pkg/sse: Support multiline reads and returning server errors.<commit_after>package sse

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"sync"
)

type SSEWriter interface {
	Write([]byte) (int, error)
	Flush()
}

func NewSSEWriter(w io.Writer) SSEWriter {
	return &Writer{Writer: w}
}

type Writer struct {
	io.Writer
	sync.Mutex
}

func (w *Writer) Write(p []byte) (int, error) {
	w.Lock()
	defer w.Unlock()
	for _, line := range bytes.Split(p, []byte("\n")) {
		if _, err := fmt.Fprintf(w.Writer, "data: %s\n", line); err != nil {
			return 0, err
		}
	}
	// add a terminating newline
	_, err := w.Writer.Write([]byte("\n"))
	return len(p), err
}

func (w *Writer) Flush() {
	if fw, ok := w.Writer.(http.Flusher); ok {
		fw.Flush()
	}
}

type Reader struct {
	*bufio.Reader
}

type Error string

func (e Error) Error() string {
	return "Server error: " + string(e)
}

func (r *Reader) Read() ([]byte, error) {
	buf := []byte{}
	var isErr bool
	for {
		line, err := r.ReadBytes('\n')
		if err != nil {
			return nil, err
		}
		if bytes.HasPrefix(line, []byte("event: error")) {
			isErr = true
		}
		if bytes.HasPrefix(line, []byte("data: ")) {
			data := bytes.TrimSuffix(bytes.TrimPrefix(line, []byte("data: ")), []byte("\n"))
			buf = append(buf, data...)
		}
		// peek ahead one byte to see if we have a double newline (terminator)
		if peek, err := r.Peek(1); err == nil && string(peek) == "\n" {
			break
		}
	}
	if isErr {
		return nil, Error(string(buf))
	}
	return buf, nil
}

type Decoder struct {
	*Reader
}

func NewDecoder(r *bufio.Reader) *Decoder {
	return &Decoder{&Reader{r}}
}

// Decode finds the next "data" field and decodes it into v
func (dec *Decoder) Decode(v interface{}) error {
	data, err := dec.Reader.Read()
	if err != nil {
		return err
	}
	return json.Unmarshal(data, v)
}
<|endoftext|>