<commit_before>package dbServices

import (
	"core/extensions"
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"strings"
)

type tableSchema struct {
	CId          int
	Name         string
	FieldType    string
	NotNull      int
	DefaultValue sql.NullString
	PrimaryKey   int
}

func createSQLiteTables(tables []tableDef) {

	for _, table := range tables {

		currentSchema := getSQLiteTableSchema(table)
		doWeAlter := isAlterRequired(table, currentSchema)

		if doWeAlter {
			renameSQLiteTable(table.Name, table.Name+"_alterTemp")
		}

		fmt.Printf("%+v\n", doWeAlter)
		sqlStmt := generateSQLiteTableCreate(table, []foreignKeyDef{})

		fmt.Println(sqlStmt)

		_, errDBExec := DB.Exec(sqlStmt)
		if errDBExec != nil {
			fmt.Println("Creation of table \"" + table.Name + "\" failed:  " + errDBExec.Error())
			continue
		}

		if doWeAlter {
			copySQLiteTableWithAlter(table.Name+"_alterTemp", table.Name, currentSchema, table)
			dropSQLiteTable(table.Name + "_alterTemp")
		}

		fmt.Println("Creation of table \"" + table.Name + "\" successful.")
	}
}

func isAlterRequired(table tableDef, existingSchema []tableSchema) bool {
	if len(existingSchema) == 0 {
		return false
	}

	if len(table.Fields) < len(existingSchema) {
		return true
	}

	if len(table.Fields) > len(existingSchema) {
		return true
	}

	for i, schemaRow := range existingSchema {

		field := table.Fields[i]
		if field.Name != schemaRow.Name {
			return true
		}
		if field.FieldType != schemaRow.FieldType {
			return true
		}
		if field.AllowNull == true && schemaRow.NotNull == 1 {
			return true
		}
		if field.AllowNull == false && schemaRow.NotNull == 0 {
			return true
		}
		if field.Primary == true && schemaRow.PrimaryKey == 0 {
			return true
		}
		if field.Primary == false && schemaRow.PrimaryKey == 1 {
			return true
		}
		if field.Default == "" && schemaRow.DefaultValue.Valid == true {
			return true
		}
		if (field.Default != "" && schemaRow.DefaultValue.Valid == true) && (field.Default != schemaRow.DefaultValue.String) {
			return true
		}
	}

	return false
}

func generateSQLiteTableCreate(table tableDef, keys []foreignKeyDef) string {
	sqlStmt := "create table if not exists "
	sqlStmt += table.Name + " "
	sqlStmt += "("

	for _, field := range table.Fields {
		sqlStmt += field.Name + " "
		sqlStmt += field.FieldType + " "
		if field.AllowNull == false {
			sqlStmt += "not null "
		}
		if field.Primary == true {
			sqlStmt += "primary key "
		}
		if field.IsUnique == true {
			sqlStmt += "unique "
		}
		if field.Check != "" {
			sqlStmt += "check(" + field.Check + ") "
		}
		if field.Collate != "" {
			sqlStmt += "collate " + field.Collate + " "
		}
		if field.Default != "" {
			fieldChar := getSQLiteFieldCharacter(field.FieldType)
			sqlStmt += "Default " + fieldChar + field.Default + fieldChar + " "
		}

		sqlStmt += ","
	}

	sqlStmt = extensions.TrimSuffix(sqlStmt, ",")

	sqlStmt += genSQLiteForeignKeyCreate(keys)

	sqlStmt += ");"
	fmt.Println(sqlStmt)

	return sqlStmt
}

func genSQLiteForeignKeyCreate(keys []foreignKeyDef) string {

	sqlStmt := ""
	for _, fk := range keys {
		cascades := ""
		if fk.OnDelete == true {
			cascades += " ON DELETE CASCADE"
		}
		if fk.OnUpdate == true {
			cascades += " ON UPDATE CASCADE"
		}

		tblFields := ""
		for _, f := range fk.Fields {
			tblFields += f + ","
		}

		tblFields = extensions.TrimSuffix(tblFields, ",")

		fkFields := ""
		for _, f := range fk.FKFields {
			fkFields += f + ","
		}

		fkFields = extensions.TrimSuffix(fkFields, ",")

		sqlStmt += ", FOREIGN KEY (" + tblFields + ") references " + fk.FKTable + "(" + fkFields + ") " + cascades
	}

	return sqlStmt
}

func createSQLiteIndexes(indexes []indexDef) {
	for _, index := range indexes {
		sqlStmt := "create "
		if index.IsUnique == true {
			sqlStmt += "unique "
		}

		sqlStmt += "index if not exists "
		sqlStmt += index.Name + " on "
		sqlStmt += index.TableName + " ("

		for _, field := range index.Fields {
			sqlStmt += field
			sqlStmt += ","
		}

		sqlStmt = extensions.TrimSuffix(sqlStmt, ",")

		sqlStmt += ");"

		_, errDBExec := DB.Exec(sqlStmt)
		if errDBExec != nil {
			fmt.Println("Creation of index \"" + index.Name + "\" failed:  " + errDBExec.Error())
			continue
		}

		fmt.Println("Creation of index \"" + index.Name + "\" successful.")
	}
}

func createSQLiteForeignKeys(foreignKeys []foreignKeyTableDef, tables []tableDef) {
	for _, fk := range foreignKeys {

		renameSQLiteTable(fk.Table, fk.Table+"_temp")

		//Create the new Table with FK's
		for _, table := range tables {
			if table.Name == fk.Table {
				sqlStmt := generateSQLiteTableCreate(table, fk.Keys)
				_, errDBExec := DB.Exec(sqlStmt)
				if errDBExec != nil {
					fmt.Println("Creation of table with Foreign Keys Failed:  Table  \"" + fk.Table + "_temp" + "\" " + errDBExec.Error())
					return
				}
				fmt.Println("Creation of table \"" + fk.Table + "\" successful.")
				break
			}
		}

		copySQLiteTable(fk.Table+"_temp", fk.Table)
		dropSQLiteTable(fk.Table + "_temp")
	}
}

func renameSQLiteTable(original string, newName string) {

	sqlStmt := "ALTER TABLE " + original + " RENAME TO " + newName + ";"

	fmt.Println("Renaming table from " + original + " to " + newName)

	_, errDBExec := DB.Exec(sqlStmt)
	if errDBExec != nil {
		fmt.Println("Rename of table \"" + original + "\" failed:  " + errDBExec.Error())
	}
	fmt.Println("Renamed " + original + " to " + newName)
}

func copySQLiteTable(from string, to string) {

	sqlStmt := "INSERT INTO " + to + " SELECT * FROM " + from + ";"

	fmt.Println("Copying Table Data from " + from + " to " + to)

	_, errDBExec := DB.Exec(sqlStmt)
	if errDBExec != nil {
		fmt.Println("Copy of table \"" + from + "\" to " + "\"" + to + "\" failed:  " + errDBExec.Error())
	}
	fmt.Println("Copied Data from " + from + " to " + to + " successfully.")
}

func copySQLiteTableWithAlter(from string, to string, currentSchema []tableSchema, table tableDef) {

	sqlStmt := "INSERT INTO " + to + " ("
	sqlStmt2 := "SELECT "
	schemaLength := len(currentSchema)

	for i, field := range table.Fields {
		if schemaLength == i {
			break
		}
		sqlStmt += field.Name + ","
		sqlStmt2 += currentSchema[i].Name + ","
	}

	sqlStmt = extensions.TrimSuffix(sqlStmt, ",")
	sqlStmt2 = extensions.TrimSuffix(sqlStmt2, ",")

	sqlStmt += ") "
	sqlStmt2 += " FROM " + from + ";"

	sqlStmt += sqlStmt2

	fmt.Println("Copying Table Data from " + from + " to " + to)
	fmt.Println(sqlStmt)

	_, errDBExec := DB.Exec(sqlStmt)
	if errDBExec != nil {
		fmt.Println("Copy of table \"" + from + "\" to " + "\"" + to + "\" failed:  " + errDBExec.Error())
	}
	fmt.Println("Copied Data from " + from + " to " + to + " successfully.")
}

func dropSQLiteTable(tableName string) {

	sqlStmt := "DROP TABLE IF EXISTS " + tableName + ";"

	fmt.Println("Dropping table " + tableName)

	_, errDBExec := DB.Exec(sqlStmt)
	if errDBExec != nil {
		fmt.Println("Drop of Table \"" + tableName + "\" failed:  " + errDBExec.Error())
	}
	fmt.Println("Dropped Table " + tableName + " successfully.")
}

func getSQLiteFieldCharacter(value string) string {
	if strings.Contains(value, "CHAR") || strings.Contains(value, "TEXT") || strings.Contains(value, "CLOB") {
		return "'"
	}

	return ""
}

func getSQLiteTableSchema(table tableDef) []tableSchema {

	schemaRows := []tableSchema{}

	rows, err := DB.Query("PRAGMA table_info(" + table.Name + ");")
	if err != nil {
		fmt.Println("Prepare of DB Query Failed.  " + err.Error())
		return schemaRows
	}
	defer rows.Close()

	for rows.Next() {
		var schema tableSchema
		err = rows.Scan(&schema.CId, &schema.Name, &schema.FieldType, &schema.NotNull, &schema.DefaultValue, &schema.PrimaryKey)
		if err != nil {
			fmt.Println("Scan failed to load Table Schema for table \"" + table.Name + "\":  " + err.Error())
		}
		schemaRows = append(schemaRows, schema)
	}
	return schemaRows
}
<commit_msg>Working on Checking for FK's.<commit_after>package dbServices

import (
	"core/extensions"
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"strings"
)

type tableSchema struct {
	CId          int
	Name         string
	FieldType    string
	NotNull      int
	DefaultValue sql.NullString
	PrimaryKey   int
}

type foreignKeySchema struct {
	Id        int
	Seq       int
	Table     string
	From      string
	On_Update string
	On_Delete string
	Match     string
}

func createSQLiteTables(tables []tableDef) {

	for _, table := range tables {

		currentSchema := getSQLiteTableSchema(table)
		doWeAlter := isAlterRequired(table, currentSchema)

		if doWeAlter {
			renameSQLiteTable(table.Name, table.Name+"_alterTemp")
		}

		fmt.Printf("%+v\n", doWeAlter)
		sqlStmt := generateSQLiteTableCreate(table, []foreignKeyDef{})

		fmt.Println(sqlStmt)

		_, errDBExec := DB.Exec(sqlStmt)
		if errDBExec != nil {
			fmt.Println("Creation of table \"" + table.Name + "\" failed:  " + errDBExec.Error())
			continue
		}

		if doWeAlter {
			copySQLiteTableWithAlter(table.Name+"_alterTemp", table.Name, currentSchema, table)
			dropSQLiteTable(table.Name + "_alterTemp")
		}

		fmt.Println("Creation of table \"" + table.Name + "\" successful.")
	}
}

func isAlterRequired(table tableDef, existingSchema []tableSchema) bool {
	if len(existingSchema) == 0 {
		return false
	}

	if len(table.Fields) < len(existingSchema) {
		return true
	}

	if len(table.Fields) > len(existingSchema) {
		return true
	}

	for i, schemaRow := range existingSchema {

		field := table.Fields[i]
		if field.Name != schemaRow.Name {
			return true
		}
		if field.FieldType != schemaRow.FieldType {
			return true
		}
		if field.AllowNull == true && schemaRow.NotNull == 1 {
			return true
		}
		if field.AllowNull == false && schemaRow.NotNull == 0 {
			return true
		}
		if field.Primary == true && schemaRow.PrimaryKey == 0 {
			return true
		}
		if field.Primary == false && schemaRow.PrimaryKey == 1 {
			return true
		}
		if field.Default == "" && schemaRow.DefaultValue.Valid == true {
			return true
		}
		if (field.Default != "" && schemaRow.DefaultValue.Valid == true) && (field.Default != schemaRow.DefaultValue.String) {
			return true
		}
	}

	return false
}

func isFKChangeRequired(fkDef foreignKeyTableDef, fkSchemas []foreignKeySchema) {
	if len(fkSchemas) == 0 {
		return true
	}
	for i, fkSchema := range fkSchemas {
		if fkDef.FKTable == fkSchema.Table {

		}
	}

	return false
}

func generateSQLiteTableCreate(table tableDef, keys []foreignKeyDef) string {
	sqlStmt := "create table if not exists "
	sqlStmt += table.Name + " "
	sqlStmt += "("

	for _, field := range table.Fields {
		sqlStmt += field.Name + " "
		sqlStmt += field.FieldType + " "
		if field.AllowNull == false {
			sqlStmt += "not null "
		}
		if field.Primary == true {
			sqlStmt += "primary key "
		}
		if field.IsUnique == true {
			sqlStmt += "unique "
		}
		if field.Check != "" {
			sqlStmt += "check(" + field.Check + ") "
		}
		if field.Collate != "" {
			sqlStmt += "collate " + field.Collate + " "
		}
		if field.Default != "" {
			fieldChar := getSQLiteFieldCharacter(field.FieldType)
			sqlStmt += "Default " + fieldChar + field.Default + fieldChar + " "
		}

		sqlStmt += ","
	}

	sqlStmt = extensions.TrimSuffix(sqlStmt, ",")

	sqlStmt += genSQLiteForeignKeyCreate(keys)

	sqlStmt += ");"
	fmt.Println(sqlStmt)

	return sqlStmt
}

func genSQLiteForeignKeyCreate(keys []foreignKeyDef) string {

	sqlStmt := ""
	for _, fk := range keys {
		cascades := ""
		if fk.OnDelete == true {
			cascades += " ON DELETE CASCADE"
		}
		if fk.OnUpdate == true {
			cascades += " ON UPDATE CASCADE"
		}

		tblFields := ""
		for _, f := range fk.Fields {
			tblFields += f + ","
		}

		tblFields = extensions.TrimSuffix(tblFields, ",")

		fkFields := ""
		for _, f := range fk.FKFields {
			fkFields += f + ","
		}

		fkFields = extensions.TrimSuffix(fkFields, ",")

		sqlStmt += ", FOREIGN KEY (" + tblFields + ") references " + fk.FKTable + "(" + fkFields + ") " + cascades
	}

	return sqlStmt
}

func createSQLiteIndexes(indexes []indexDef) {
	for _, index := range indexes {
		sqlStmt := "create "
		if index.IsUnique == true {
			sqlStmt += "unique "
		}

		sqlStmt += "index if not exists "
		sqlStmt += index.Name + " on "
		sqlStmt += index.TableName + " ("

		for _, field := range index.Fields {
			sqlStmt += field
			sqlStmt += ","
		}

		sqlStmt = extensions.TrimSuffix(sqlStmt, ",")

		sqlStmt += ");"

		_, errDBExec := DB.Exec(sqlStmt)
		if errDBExec != nil {
			fmt.Println("Creation of index \"" + index.Name + "\" failed:  " + errDBExec.Error())
			continue
		}

		fmt.Println("Creation of index \"" + index.Name + "\" successful.")
	}
}

func createSQLiteForeignKeys(foreignKeys []foreignKeyTableDef, tables []tableDef) {
	for _, fk := range foreignKeys {

		renameSQLiteTable(fk.Table, fk.Table+"_temp")

		//Create the new Table with FK's
		for _, table := range tables {
			if table.Name == fk.Table {
				sqlStmt := generateSQLiteTableCreate(table, fk.Keys)
				_, errDBExec := DB.Exec(sqlStmt)
				if errDBExec != nil {
					fmt.Println("Creation of table with Foreign Keys Failed:  Table  \"" + fk.Table + "_temp" + "\" " + errDBExec.Error())
					return
				}
				fmt.Println("Creation of table \"" + fk.Table + "\" successful.")
				break
			}
		}

		copySQLiteTable(fk.Table+"_temp", fk.Table)
		dropSQLiteTable(fk.Table + "_temp")
	}
}

func renameSQLiteTable(original string, newName string) {

	sqlStmt := "ALTER TABLE " + original + " RENAME TO " + newName + ";"

	fmt.Println("Renaming table from " + original + " to " + newName)

	_, errDBExec := DB.Exec(sqlStmt)
	if errDBExec != nil {
		fmt.Println("Rename of table \"" + original + "\" failed:  " + errDBExec.Error())
	}
	fmt.Println("Renamed " + original + " to " + newName)
}

func copySQLiteTable(from string, to string) {

	sqlStmt := "INSERT INTO " + to + " SELECT * FROM " + from + ";"

	fmt.Println("Copying Table Data from " + from + " to " + to)

	_, errDBExec := DB.Exec(sqlStmt)
	if errDBExec != nil {
		fmt.Println("Copy of table \"" + from + "\" to " + "\"" + to + "\" failed:  " + errDBExec.Error())
	}
	fmt.Println("Copied Data from " + from + " to " + to + " successfully.")
}

func copySQLiteTableWithAlter(from string, to string, currentSchema []tableSchema, table tableDef) {

	sqlStmt := "INSERT INTO " + to + " ("
	sqlStmt2 := "SELECT "
	schemaLength := len(currentSchema)

	for i, field := range table.Fields {
		if schemaLength == i {
			break
		}
		sqlStmt += field.Name + ","
		sqlStmt2 += currentSchema[i].Name + ","
	}

	sqlStmt = extensions.TrimSuffix(sqlStmt, ",")
	sqlStmt2 = extensions.TrimSuffix(sqlStmt2, ",")

	sqlStmt += ") "
	sqlStmt2 += " FROM " + from + ";"

	sqlStmt += sqlStmt2

	fmt.Println("Copying Table Data from " + from + " to " + to)
	fmt.Println(sqlStmt)

	_, errDBExec := DB.Exec(sqlStmt)
	if errDBExec != nil {
		fmt.Println("Copy of table \"" + from + "\" to " + "\"" + to + "\" failed:  " + errDBExec.Error())
	}
	fmt.Println("Copied Data from " + from + " to " + to + " successfully.")
}

func dropSQLiteTable(tableName string) {

	sqlStmt := "DROP TABLE IF EXISTS " + tableName + ";"

	fmt.Println("Dropping table " + tableName)

	_, errDBExec := DB.Exec(sqlStmt)
	if errDBExec != nil {
		fmt.Println("Drop of Table \"" + tableName + "\" failed:  " + errDBExec.Error())
	}
	fmt.Println("Dropped Table " + tableName + " successfully.")
}

func getSQLiteFieldCharacter(value string) string {
	if strings.Contains(value, "CHAR") || strings.Contains(value, "TEXT") || strings.Contains(value, "CLOB") {
		return "'"
	}

	return ""
}

func getSQLiteTableSchema(table tableDef) []tableSchema {

	schemaRows := []tableSchema{}

	rows, err := DB.Query("PRAGMA table_info(" + table.Name + ");")
	if err != nil {
		fmt.Println("Prepare of DB Query Failed.  " + err.Error())
		return schemaRows
	}
	defer rows.Close()

	for rows.Next() {
		var schema tableSchema
		err = rows.Scan(&schema.CId, &schema.Name, &schema.FieldType, &schema.NotNull, &schema.DefaultValue, &schema.PrimaryKey)
		if err != nil {
			fmt.Println("Scan failed to load Table Schema for table \"" + table.Name + "\":  " + err.Error())
		}
		schemaRows = append(schemaRows, schema)
	}
	return schemaRows
}

func getTableForeignKeys(table tableDef) []foreignKeySchema {

	schemaRows := []foreignKeySchema{}

	rows, err := DB.Query("PRAGMA foreign_key_list(" + table.Name + ");")
	if err != nil {
		fmt.Println("Prepare of DB Query Failed.  " + err.Error())
		return schemaRows
	}
	defer rows.Close()

	for rows.Next() {
		var schema tableSchema
		err = rows.Scan(&schema.Id, &schema.Seq, &schema.Table, &schema.From, &schema.To, &schema.On_Update, &schema.On_Delete, &schema.Match)
		if err != nil {
			fmt.Println("Scan failed to load Foreign Key Schema for table \"" + table.Name + "\":  " + err.Error())
		}
		schemaRows = append(schemaRows, schema)
	}
	return schemaRows
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package httplog

import (
	"bufio"
	"fmt"
	"net"
	"net/http"
	"runtime"
	"time"

	"github.com/golang/glog"
)

// Handler wraps all HTTP calls to delegate with nice logging.
// delegate may use LogOf(w).Addf(...) to write additional info to
// the per-request log message.
//
// Intended to wrap calls to your ServeMux.
func Handler(delegate http.Handler, pred StacktracePred) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		defer NewLogged(req, &w).StacktraceWhen(pred).Log()
		delegate.ServeHTTP(w, req)
	})
}

// StacktracePred returns true if a stacktrace should be logged for this status.
type StacktracePred func(httpStatus int) (logStacktrace bool)

type logger interface {
	Addf(format string, data ...interface{})
}

// Add a layer on top of ResponseWriter, so we can track latency and error
// message sources.
//
// TODO now that we're using go-restful, we shouldn't need to be wrapping
// the http.ResponseWriter. We can recover panics from go-restful, and
// the logging value is questionable.
type respLogger struct {
	status      int
	statusStack string
	addedInfo   string
	startTime   time.Time

	req *http.Request
	w   http.ResponseWriter

	logStacktracePred StacktracePred
}

// Simple logger that logs immediately when Addf is called
type passthroughLogger struct{}

// Addf logs info immediately.
func (passthroughLogger) Addf(format string, data ...interface{}) {
	glog.InfoDepth(1, fmt.Sprintf(format, data...))
}

// DefaultStacktracePred is the default implementation of StacktracePred.
func DefaultStacktracePred(status int) bool {
	return (status < http.StatusOK || status >= http.StatusBadRequest) && status != http.StatusSwitchingProtocols
}

// NewLogged turns a normal response writer into a logged response writer.
//
// Usage:
//
// defer NewLogged(req, &w).StacktraceWhen(StatusIsNot(200, 202)).Log()
//
// (Only the call to Log() is defered, so you can set everything up in one line!)
//
// Note that this *changes* your writer, to route response writing actions
// through the logger.
//
// Use LogOf(w).Addf(...) to log something along with the response result.
func NewLogged(req *http.Request, w *http.ResponseWriter) *respLogger {
	if _, ok := (*w).(*respLogger); ok {
		// Don't double-wrap!
		panic("multiple NewLogged calls!")
	}
	rl := &respLogger{
		startTime:         time.Now(),
		req:               req,
		w:                 *w,
		logStacktracePred: DefaultStacktracePred,
	}
	*w = rl // hijack caller's writer!
	return rl
}

// LogOf returns the logger hiding in w. If there is not an existing logger
// then a passthroughLogger will be created which will log to stdout immediately
// when Addf is called.
func LogOf(req *http.Request, w http.ResponseWriter) logger {
	if _, exists := w.(*respLogger); !exists {
		pl := &passthroughLogger{}
		return pl
	}
	if rl, ok := w.(*respLogger); ok {
		return rl
	}
	panic("Unable to find or create the logger!")
}

// Unlogged returns the original ResponseWriter, or w if it is not our inserted logger.
func Unlogged(w http.ResponseWriter) http.ResponseWriter {
	if rl, ok := w.(*respLogger); ok {
		return rl.w
	}
	return w
}

// StacktraceWhen sets the stacktrace logging predicate, which decides when to log a stacktrace.
// There's a default, so you don't need to call this unless you don't like the default.
func (rl *respLogger) StacktraceWhen(pred StacktracePred) *respLogger {
	rl.logStacktracePred = pred
	return rl
}

// StatusIsNot returns a StacktracePred which will cause stacktraces to be logged
// for any status *not* in the given list.
func StatusIsNot(statuses ...int) StacktracePred {
	return func(status int) bool {
		for _, s := range statuses {
			if status == s {
				return false
			}
		}
		return true
	}
}

// Addf adds additional data to be logged with this request.
func (rl *respLogger) Addf(format string, data ...interface{}) {
	rl.addedInfo += "\n" + fmt.Sprintf(format, data...)
}

// Log is intended to be called once at the end of your request handler, via defer
func (rl *respLogger) Log() {
	latency := time.Since(rl.startTime)
	if glog.V(2) {
		glog.InfoDepth(1, fmt.Sprintf("%s %s: (%v) %v%v%v", rl.req.Method, rl.req.RequestURI, latency, rl.status, rl.statusStack, rl.addedInfo))
	}
}

// Header implements http.ResponseWriter.
func (rl *respLogger) Header() http.Header {
	return rl.w.Header()
}

// Write implements http.ResponseWriter.
func (rl *respLogger) Write(b []byte) (int, error) {
	return rl.w.Write(b)
}

// Flush implements http.Flusher even if the underlying http.Writer doesn't implement it.
// Flush is used for streaming purposes and allows to flush buffered data to the client.
func (rl *respLogger) Flush() {
	if flusher, ok := rl.w.(http.Flusher); ok {
		flusher.Flush()
	} else if glog.V(2) {
		glog.InfoDepth(1, fmt.Sprintf("Unable to convert %+v into http.Flusher", rl.w))
	}
}

// WriteHeader implements http.ResponseWriter.
func (rl *respLogger) WriteHeader(status int) {
	rl.status = status
	if rl.logStacktracePred(status) {
		// Only log stacks for errors
		stack := make([]byte, 2048)
		stack = stack[:runtime.Stack(stack, false)]
		rl.statusStack = "\n" + string(stack)
	} else {
		rl.statusStack = ""
	}
	rl.w.WriteHeader(status)
}

// Hijack implements http.Hijacker.
func (rl *respLogger) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	return rl.w.(http.Hijacker).Hijack()
}
<commit_msg>Stop logging stack traces for 4xx requests<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package httplog

import (
	"bufio"
	"fmt"
	"net"
	"net/http"
	"runtime"
	"time"

	"github.com/golang/glog"
)

// Handler wraps all HTTP calls to delegate with nice logging.
// delegate may use LogOf(w).Addf(...) to write additional info to
// the per-request log message.
//
// Intended to wrap calls to your ServeMux.
func Handler(delegate http.Handler, pred StacktracePred) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		defer NewLogged(req, &w).StacktraceWhen(pred).Log()
		delegate.ServeHTTP(w, req)
	})
}

// StacktracePred returns true if a stacktrace should be logged for this status.
type StacktracePred func(httpStatus int) (logStacktrace bool)

type logger interface {
	Addf(format string, data ...interface{})
}

// Add a layer on top of ResponseWriter, so we can track latency and error
// message sources.
//
// TODO now that we're using go-restful, we shouldn't need to be wrapping
// the http.ResponseWriter. We can recover panics from go-restful, and
// the logging value is questionable.
type respLogger struct {
	status      int
	statusStack string
	addedInfo   string
	startTime   time.Time

	req *http.Request
	w   http.ResponseWriter

	logStacktracePred StacktracePred
}

// Simple logger that logs immediately when Addf is called
type passthroughLogger struct{}

// Addf logs info immediately.
func (passthroughLogger) Addf(format string, data ...interface{}) {
	glog.InfoDepth(1, fmt.Sprintf(format, data...))
}

// DefaultStacktracePred is the default implementation of StacktracePred.
func DefaultStacktracePred(status int) bool {
	return (status < http.StatusOK || status >= http.StatusInternalServerError) && status != http.StatusSwitchingProtocols
}

// NewLogged turns a normal response writer into a logged response writer.
//
// Usage:
//
// defer NewLogged(req, &w).StacktraceWhen(StatusIsNot(200, 202)).Log()
//
// (Only the call to Log() is defered, so you can set everything up in one line!)
//
// Note that this *changes* your writer, to route response writing actions
// through the logger.
//
// Use LogOf(w).Addf(...) to log something along with the response result.
func NewLogged(req *http.Request, w *http.ResponseWriter) *respLogger {
	if _, ok := (*w).(*respLogger); ok {
		// Don't double-wrap!
		panic("multiple NewLogged calls!")
	}
	rl := &respLogger{
		startTime:         time.Now(),
		req:               req,
		w:                 *w,
		logStacktracePred: DefaultStacktracePred,
	}
	*w = rl // hijack caller's writer!
	return rl
}

// LogOf returns the logger hiding in w. If there is not an existing logger
// then a passthroughLogger will be created which will log to stdout immediately
// when Addf is called.
func LogOf(req *http.Request, w http.ResponseWriter) logger {
	if _, exists := w.(*respLogger); !exists {
		pl := &passthroughLogger{}
		return pl
	}
	if rl, ok := w.(*respLogger); ok {
		return rl
	}
	panic("Unable to find or create the logger!")
}

// Unlogged returns the original ResponseWriter, or w if it is not our inserted logger.
func Unlogged(w http.ResponseWriter) http.ResponseWriter {
	if rl, ok := w.(*respLogger); ok {
		return rl.w
	}
	return w
}

// StacktraceWhen sets the stacktrace logging predicate, which decides when to log a stacktrace.
// There's a default, so you don't need to call this unless you don't like the default.
func (rl *respLogger) StacktraceWhen(pred StacktracePred) *respLogger {
	rl.logStacktracePred = pred
	return rl
}

// StatusIsNot returns a StacktracePred which will cause stacktraces to be logged
// for any status *not* in the given list.
func StatusIsNot(statuses ...int) StacktracePred {
	return func(status int) bool {
		for _, s := range statuses {
			if status == s {
				return false
			}
		}
		return true
	}
}

// Addf adds additional data to be logged with this request.
func (rl *respLogger) Addf(format string, data ...interface{}) {
	rl.addedInfo += "\n" + fmt.Sprintf(format, data...)
}

// Log is intended to be called once at the end of your request handler, via defer
func (rl *respLogger) Log() {
	latency := time.Since(rl.startTime)
	if glog.V(2) {
		glog.InfoDepth(1, fmt.Sprintf("%s %s: (%v) %v%v%v", rl.req.Method, rl.req.RequestURI, latency, rl.status, rl.statusStack, rl.addedInfo))
	}
}

// Header implements http.ResponseWriter.
func (rl *respLogger) Header() http.Header {
	return rl.w.Header()
}

// Write implements http.ResponseWriter.
func (rl *respLogger) Write(b []byte) (int, error) {
	return rl.w.Write(b)
}

// Flush implements http.Flusher even if the underlying http.Writer doesn't implement it.
// Flush is used for streaming purposes and allows to flush buffered data to the client.
func (rl *respLogger) Flush() {
	if flusher, ok := rl.w.(http.Flusher); ok {
		flusher.Flush()
	} else if glog.V(2) {
		glog.InfoDepth(1, fmt.Sprintf("Unable to convert %+v into http.Flusher", rl.w))
	}
}

// WriteHeader implements http.ResponseWriter.
func (rl *respLogger) WriteHeader(status int) {
	rl.status = status
	if rl.logStacktracePred(status) {
		// Only log stacks for errors
		stack := make([]byte, 2048)
		stack = stack[:runtime.Stack(stack, false)]
		rl.statusStack = "\n" + string(stack)
	} else {
		rl.statusStack = ""
	}
	rl.w.WriteHeader(status)
}

// Hijack implements http.Hijacker.
func (rl *respLogger) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	return rl.w.(http.Hijacker).Hijack()
}
<|endoftext|>


<commit_before>package jobs

import (
	"fmt"
	"math/rand"
	"sync/atomic"
	"time"

	log "github.com/Sirupsen/logrus"
)

var (
	defaultConcurrency  = 1
	defaultMaxExecCount = 3
	defaultMaxExecTime  = 60 * time.Second
	defaultRetryDelay   = 60 * time.Millisecond
	defaultTimeout      = 10 * time.Second

	maxMaxExecCount = 5
	maxMaxExecTime  = 5 * time.Minute
	maxTimeout      = 1 * time.Minute
)

type (
	// WorkerFunc represent the work function that a worker should implement.
	WorkerFunc func(msg *Message, timeout <-chan time.Time) error

	// Worker is a unit of work that will consume from a queue and execute the do
	// method for each jobs it pulls.
	Worker struct {
		Domain string
		Type   string
		Conf   *WorkerConfig

		jobs    Queue
		started int32
	}
)

// Start is used to start the worker consumption of messages from its queue.
func (w *Worker) Start(q Queue) {
	if !atomic.CompareAndSwapInt32(&w.started, 0, 1) {
		return
	}
	w.jobs = q
	for i := 0; i < int(w.Conf.Concurrency); i++ {
		name := fmt.Sprintf("%s/%s/%d", w.Domain, w.Type, i)
		go w.work(name)
	}
}

func (w *Worker) work(workerID string) {
	// TODO: err handling and persistence
	for {
		job, err := w.jobs.Consume()
		if err != nil {
			if err != ErrQueueClosed {
				log.Errorf("[job] %s: error while consuming queue (%s)",
					workerID, err.Error())
			}
			return
		}
		infos := job.Infos()
		if err = job.AckConsumed(); err != nil {
			log.Errorf("[job] %s: error acking consume job %s (%s)",
				workerID, infos.ID, err.Error())
			continue
		}
		t := &task{
			infos: infos,
			conf:  w.defaultedConf(infos.Options),
		}
		if err = t.run(); err != nil {
			log.Errorf("[job] %s: error while performing job %s (%s)",
				workerID, infos.ID, err.Error())
			err = job.Nack(err)
		} else {
			err = job.Ack()
		}
		if err != nil {
			log.Errorf("[job] %s: error while acking job done %s (%s)",
				workerID, infos.ID, err.Error())
		}
	}
}

func (w *Worker) defaultedConf(opts *JobOptions) *WorkerConfig {
	c := &(*w.Conf)
	if c.Concurrency == 0 {
		c.Concurrency = uint(defaultConcurrency)
	}
	if c.MaxExecCount == 0 {
		c.MaxExecCount = uint(defaultMaxExecCount)
	}
	if c.MaxExecTime == 0 {
		c.MaxExecTime = defaultMaxExecTime
	}
	if c.RetryDelay == 0 {
		c.RetryDelay = defaultRetryDelay
	}
	if c.Timeout == 0 {
		c.Timeout = defaultTimeout
	}
	if opts == nil {
		return c
	}
	if opts.MaxExecCount != 0 && opts.MaxExecCount < c.MaxExecCount {
		c.MaxExecCount = opts.MaxExecCount
	}
	if opts.MaxExecTime > 0 && opts.MaxExecTime < c.MaxExecTime {
		c.MaxExecTime = opts.MaxExecTime
	}
	if opts.Timeout > 0 && opts.Timeout < c.Timeout {
		c.Timeout = opts.Timeout
	}
	return c
}

// Stop will stop the worker's consumption of its queue. It will also close the
// associated queue.
func (w *Worker) Stop() {
	if !atomic.CompareAndSwapInt32(&w.started, 1, 0) {
		return
	}
	w.jobs.Close()
}

type task struct {
	infos *JobInfos
	conf  *WorkerConfig

	startTime time.Time
	execCount uint
}

func (t *task) run() (err error) {
	t.startTime = time.Now()
	t.execCount = 0
	for {
		retry, delay, timeout := t.nextDelay()
		if !retry {
			return err
		}
		if err != nil {
			log.Warnf("[job] %s: %s (retry in %s)", t.infos.ID, err.Error(), delay)
		}
		if delay > 0 {
			time.Sleep(delay)
		}
		log.Debugf("[job] %s: run %d (timeout %s)", t.infos.ID, t.execCount, timeout)
		err = t.conf.WorkerFunc(t.infos.Message, time.After(timeout))
		if err == nil {
			break
		}
		t.execCount++
	}
	return nil
}

func (t *task) nextDelay() (bool, time.Duration, time.Duration) {
	c := t.conf
	execTime := time.Since(t.startTime)

	if t.execCount >= c.MaxExecCount || execTime > c.MaxExecTime {
		return false, 0, 0
	}

	// the worker timeout should take into account the maximum execution time
	// allowed to the task
	timeout := c.Timeout
	if execTime+timeout > c.MaxExecTime {
		timeout = execTime - c.MaxExecTime
	}

	var nextDelay time.Duration
	if t.execCount == 0 {
		// on first execution, execute immediatly
		nextDelay = 0
	} else {
		nextDelay = c.RetryDelay << (t.execCount - 1)

		// fuzzDelay number between delay * (1 +/- 0.1)
		fuzzDelay := int(0.1 * float64(nextDelay))
		nextDelay = nextDelay + time.Duration((rand.Intn(2*fuzzDelay) - fuzzDelay))
	}

	if execTime+nextDelay > c.MaxExecTime {
		return false, 0, 0
	}

	return true, nextDelay, timeout
}
<commit_msg>Remove dead constants<commit_after>package jobs

import (
	"fmt"
	"math/rand"
	"sync/atomic"
	"time"

	log "github.com/Sirupsen/logrus"
)

var (
	defaultConcurrency  = 1
	defaultMaxExecCount = 3
	defaultMaxExecTime  = 60 * time.Second
	defaultRetryDelay   = 60 * time.Millisecond
	defaultTimeout      = 10 * time.Second
)

type (
	// WorkerFunc represent the work function that a worker should implement.
	WorkerFunc func(msg *Message, timeout <-chan time.Time) error

	// Worker is a unit of work that will consume from a queue and execute the do
	// method for each jobs it pulls.
	Worker struct {
		Domain string
		Type   string
		Conf   *WorkerConfig

		jobs    Queue
		started int32
	}
)

// Start is used to start the worker consumption of messages from its queue.
func (w *Worker) Start(q Queue) {
	if !atomic.CompareAndSwapInt32(&w.started, 0, 1) {
		return
	}
	w.jobs = q
	for i := 0; i < int(w.Conf.Concurrency); i++ {
		name := fmt.Sprintf("%s/%s/%d", w.Domain, w.Type, i)
		go w.work(name)
	}
}

func (w *Worker) work(workerID string) {
	// TODO: err handling and persistence
	for {
		job, err := w.jobs.Consume()
		if err != nil {
			if err != ErrQueueClosed {
				log.Errorf("[job] %s: error while consuming queue (%s)",
					workerID, err.Error())
			}
			return
		}
		infos := job.Infos()
		if err = job.AckConsumed(); err != nil {
			log.Errorf("[job] %s: error acking consume job %s (%s)",
				workerID, infos.ID, err.Error())
			continue
		}
		t := &task{
			infos: infos,
			conf:  w.defaultedConf(infos.Options),
		}
		if err = t.run(); err != nil {
			log.Errorf("[job] %s: error while performing job %s (%s)",
				workerID, infos.ID, err.Error())
			err = job.Nack(err)
		} else {
			err = job.Ack()
		}
		if err != nil {
			log.Errorf("[job] %s: error while acking job done %s (%s)",
				workerID, infos.ID, err.Error())
		}
	}
}

func (w *Worker) defaultedConf(opts *JobOptions) *WorkerConfig {
	c := &(*w.Conf)
	if c.Concurrency == 0 {
		c.Concurrency = uint(defaultConcurrency)
	}
	if c.MaxExecCount == 0 {
		c.MaxExecCount = uint(defaultMaxExecCount)
	}
	if c.MaxExecTime == 0 {
		c.MaxExecTime = defaultMaxExecTime
	}
	if c.RetryDelay == 0 {
		c.RetryDelay = defaultRetryDelay
	}
	if c.Timeout == 0 {
		c.Timeout = defaultTimeout
	}
	if opts == nil {
		return c
	}
	if opts.MaxExecCount != 0 && opts.MaxExecCount < c.MaxExecCount {
		c.MaxExecCount = opts.MaxExecCount
	}
	if opts.MaxExecTime > 0 && opts.MaxExecTime < c.MaxExecTime {
		c.MaxExecTime = opts.MaxExecTime
	}
	if opts.Timeout > 0 && opts.Timeout < c.Timeout {
		c.Timeout = opts.Timeout
	}
	return c
}

// Stop will stop the worker's consumption of its queue. It will also close the
// associated queue.
func (w *Worker) Stop() {
	if !atomic.CompareAndSwapInt32(&w.started, 1, 0) {
		return
	}
	w.jobs.Close()
}

type task struct {
	infos *JobInfos
	conf  *WorkerConfig

	startTime time.Time
	execCount uint
}

func (t *task) run() (err error) {
	t.startTime = time.Now()
	t.execCount = 0
	for {
		retry, delay, timeout := t.nextDelay()
		if !retry {
			return err
		}
		if err != nil {
			log.Warnf("[job] %s: %s (retry in %s)", t.infos.ID, err.Error(), delay)
		}
		if delay > 0 {
			time.Sleep(delay)
		}
		log.Debugf("[job] %s: run %d (timeout %s)", t.infos.ID, t.execCount, timeout)
		err = t.conf.WorkerFunc(t.infos.Message, time.After(timeout))
		if err == nil {
			break
		}
		t.execCount++
	}
	return nil
}

func (t *task) nextDelay() (bool, time.Duration, time.Duration) {
	c := t.conf
	execTime := time.Since(t.startTime)

	if t.execCount >= c.MaxExecCount || execTime > c.MaxExecTime {
		return false, 0, 0
	}

	// the worker timeout should take into account the maximum execution time
	// allowed to the task
	timeout := c.Timeout
	if execTime+timeout > c.MaxExecTime {
		timeout = execTime - c.MaxExecTime
	}

	var nextDelay time.Duration
	if t.execCount == 0 {
		// on first execution, execute immediatly
		nextDelay = 0
	} else {
		nextDelay = c.RetryDelay << (t.execCount - 1)

		// fuzzDelay number between delay * (1 +/- 0.1)
		fuzzDelay := int(0.1 * float64(nextDelay))
		nextDelay = nextDelay + time.Duration((rand.Intn(2*fuzzDelay) - fuzzDelay))
	}

	if execTime+nextDelay > c.MaxExecTime {
		return false, 0, 0
	}

	return true, nextDelay, timeout
}
<|endoftext|>


<commit_before>// Copyright 2015-2017 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ns

import (
	"fmt"
	"os"
	"runtime"
	"sync"
	"syscall"

	"golang.org/x/sys/unix"
)

// Returns an object representing the current OS thread's network namespace
func GetCurrentNS() (NetNS, error) {
	// Lock the thread in case other goroutine executes in it and changes its
	// network namespace after getCurrentThreadNetNSPath(), otherwise it might
	// return an unexpected network namespace.
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()
	return GetNS(getCurrentThreadNetNSPath())
}

func getCurrentThreadNetNSPath() string {
	// /proc/self/ns/net returns the namespace of the main thread, not
	// of whatever thread this goroutine is running on.  Make sure we
	// use the thread's net namespace since the thread is switching around
	return fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid())
}

func (ns *netNS) Close() error {
	if err := ns.errorIfClosed(); err != nil {
		return err
	}

	if err := ns.file.Close(); err != nil {
		return fmt.Errorf("Failed to close %q: %v", ns.file.Name(), err)
	}
	ns.closed = true

	return nil
}

func (ns *netNS) Set() error {
	if err := ns.errorIfClosed(); err != nil {
		return err
	}

	if err := unix.Setns(int(ns.Fd()), unix.CLONE_NEWNET); err != nil {
		return fmt.Errorf("Error switching to ns %v: %v", ns.file.Name(), err)
	}

	return nil
}

type NetNS interface {
	// Executes the passed closure in this object's network namespace,
	// attempting to restore the original namespace before returning.
	// However, since each OS thread can have a different network namespace,
	// and Go's thread scheduling is highly variable, callers cannot
	// guarantee any specific namespace is set unless operations that
	// require that namespace are wrapped with Do().  Also, no code called
	// from Do() should call runtime.UnlockOSThread(), or the risk
	// of executing code in an incorrect namespace will be greater.  See
	// https://github.com/golang/go/wiki/LockOSThread for further details.
	Do(toRun func(NetNS) error) error

	// Sets the current network namespace to this object's network namespace.
	// Note that since Go's thread scheduling is highly variable, callers
	// cannot guarantee the requested namespace will be the current namespace
	// after this function is called; to ensure this wrap operations that
	// require the namespace with Do() instead.
	Set() error

	// Returns the filesystem path representing this object's network namespace
	Path() string

	// Returns a file descriptor representing this object's network namespace
	Fd() uintptr

	// Cleans up this instance of the network namespace; if this instance
	// is the last user the namespace will be destroyed
	Close() error
}

type netNS struct {
	file   *os.File
	closed bool
}

// netNS implements the NetNS interface
var _ NetNS = &netNS{}

const (
	// https://github.com/torvalds/linux/blob/master/include/uapi/linux/magic.h
	NSFS_MAGIC   = 0x6e736673
	PROCFS_MAGIC = 0x9fa0
)

type NSPathNotExistErr struct{ msg string }

func (e NSPathNotExistErr) Error() string { return e.msg }

type NSPathNotNSErr struct{ msg string }

func (e NSPathNotNSErr) Error() string { return e.msg }

func IsNSorErr(nspath string) error {
	stat := syscall.Statfs_t{}
	if err := syscall.Statfs(nspath, &stat); err != nil {
		if os.IsNotExist(err) {
			err = NSPathNotExistErr{msg: fmt.Sprintf("failed to Statfs %q: %v", nspath, err)}
		} else {
			err = fmt.Errorf("failed to Statfs %q: %v", nspath, err)
		}
		return err
	}

	switch stat.Type {
	case PROCFS_MAGIC, NSFS_MAGIC:
		return nil
	default:
		return NSPathNotNSErr{msg: fmt.Sprintf("unknown FS magic on %q: %x", nspath, stat.Type)}
	}
}

// Returns an object representing the namespace referred to by @path
func GetNS(nspath string) (NetNS, error) {
	err := IsNSorErr(nspath)
	if err != nil {
		return nil, err
	}

	fd, err := os.Open(nspath)
	if err != nil {
		return nil, err
	}

	return &netNS{file: fd}, nil
}

func (ns *netNS) Path() string {
	return ns.file.Name()
}

func (ns *netNS) Fd() uintptr {
	return ns.file.Fd()
}

func (ns *netNS) errorIfClosed() error {
	if ns.closed {
		return fmt.Errorf("%q has already been closed", ns.file.Name())
	}
	return nil
}

func (ns *netNS) Do(toRun func(NetNS) error) error {
	if err := ns.errorIfClosed(); err != nil {
		return err
	}

	containedCall := func(hostNS NetNS) error {
		threadNS, err := GetCurrentNS()
		if err != nil {
			return fmt.Errorf("failed to open current netns: %v", err)
		}
		defer threadNS.Close()

		// switch to target namespace
		if err = ns.Set(); err != nil {
			return fmt.Errorf("error switching to ns %v: %v", ns.file.Name(), err)
		}
		defer func() {
			err := threadNS.Set() // switch back
			if err == nil {
				// Unlock the current thread only when we successfully switched back
				// to the original namespace; otherwise leave the thread locked which
				// will force the runtime to scrap the current thread, that is maybe
				// not as optimal but at least always safe to do.
				runtime.UnlockOSThread()
			}
		}()

		return toRun(hostNS)
	}

	// save a handle to current network namespace
	hostNS, err := GetCurrentNS()
	if err != nil {
		return fmt.Errorf("Failed to open current namespace: %v", err)
	}
	defer hostNS.Close()

	var wg sync.WaitGroup
	wg.Add(1)

	// Start the callback in a new green thread so that if we later fail
	// to switch the namespace back to the original one, we can safely
	// leave the thread locked to die without a risk of the current thread
	// left lingering with incorrect namespace.
	var innerError error
	go func() {
		defer wg.Done()
		runtime.LockOSThread()
		innerError = containedCall(hostNS)
	}()
	wg.Wait()

	return innerError
}

// WithNetNSPath executes the passed closure under the given network
// namespace, restoring the original namespace afterwards.
func WithNetNSPath(nspath string, toRun func(NetNS) error) error {
	ns, err := GetNS(nspath)
	if err != nil {
		return err
	}
	defer ns.Close()
	return ns.Do(toRun)
}
<commit_msg>pkg/ns: use file system magic numbers from golang.org/x/sys/unix<commit_after>// Copyright 2015-2017 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ns

import (
	"fmt"
	"os"
	"runtime"
	"sync"
	"syscall"

	"golang.org/x/sys/unix"
)

// Returns an object representing the current OS thread's network namespace
func GetCurrentNS() (NetNS, error) {
	// Lock the thread in case other goroutine executes in it and changes its
	// network namespace after getCurrentThreadNetNSPath(), otherwise it might
	// return an unexpected network namespace.
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()
	return GetNS(getCurrentThreadNetNSPath())
}

func getCurrentThreadNetNSPath() string {
	// /proc/self/ns/net returns the namespace of the main thread, not
	// of whatever thread this goroutine is running on.  Make sure we
	// use the thread's net namespace since the thread is switching around
	return fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid())
}

func (ns *netNS) Close() error {
	if err := ns.errorIfClosed(); err != nil {
		return err
	}

	if err := ns.file.Close(); err != nil {
		return fmt.Errorf("Failed to close %q: %v", ns.file.Name(), err)
	}
	ns.closed = true

	return nil
}

func (ns *netNS) Set() error {
	if err := ns.errorIfClosed(); err != nil {
		return err
	}

	if err := unix.Setns(int(ns.Fd()), unix.CLONE_NEWNET); err != nil {
		return fmt.Errorf("Error switching to ns %v: %v", ns.file.Name(), err)
	}

	return nil
}

type NetNS interface {
	// Executes the passed closure in this object's network namespace,
	// attempting to restore the original namespace before returning.
	// However, since each OS thread can have a different network namespace,
	// and Go's thread scheduling is highly variable, callers cannot
	// guarantee any specific namespace is set unless operations that
	// require that namespace are wrapped with Do().  Also, no code called
	// from Do() should call runtime.UnlockOSThread(), or the risk
	// of executing code in an incorrect namespace will be greater.  See
	// https://github.com/golang/go/wiki/LockOSThread for further details.
	Do(toRun func(NetNS) error) error

	// Sets the current network namespace to this object's network namespace.
	// Note that since Go's thread scheduling is highly variable, callers
	// cannot guarantee the requested namespace will be the current namespace
	// after this function is called; to ensure this wrap operations that
	// require the namespace with Do() instead.
	Set() error

	// Returns the filesystem path representing this object's network namespace
	Path() string

	// Returns a file descriptor representing this object's network namespace
	Fd() uintptr

	// Cleans up this instance of the network namespace; if this instance
	// is the last user the namespace will be destroyed
	Close() error
}

type netNS struct {
	file   *os.File
	closed bool
}

// netNS implements the NetNS interface
var _ NetNS = &netNS{}

const (
	// https://github.com/torvalds/linux/blob/master/include/uapi/linux/magic.h
	NSFS_MAGIC   = unix.NSFS_MAGIC
	PROCFS_MAGIC = unix.PROC_SUPER_MAGIC
)

type NSPathNotExistErr struct{ msg string }

func (e NSPathNotExistErr) Error() string { return e.msg }

type NSPathNotNSErr struct{ msg string }

func (e NSPathNotNSErr) Error() string { return e.msg }

func IsNSorErr(nspath string) error {
	stat := syscall.Statfs_t{}
	if err := syscall.Statfs(nspath, &stat); err != nil {
		if os.IsNotExist(err) {
			err = NSPathNotExistErr{msg: fmt.Sprintf("failed to Statfs %q: %v", nspath, err)}
		} else {
			err = fmt.Errorf("failed to Statfs %q: %v", nspath, err)
		}
		return err
	}

	switch stat.Type {
	case PROCFS_MAGIC, NSFS_MAGIC:
		return nil
	default:
		return NSPathNotNSErr{msg: fmt.Sprintf("unknown FS magic on %q: %x", nspath, stat.Type)}
	}
}

// Returns an object representing the namespace referred to by @path
func GetNS(nspath string) (NetNS, error) {
	err := IsNSorErr(nspath)
	if err != nil {
		return nil, err
	}

	fd, err := os.Open(nspath)
	if err != nil {
		return nil, err
	}

	return &netNS{file: fd}, nil
}

func (ns *netNS) Path() string {
	return ns.file.Name()
}

func (ns *netNS) Fd() uintptr {
	return ns.file.Fd()
}

func (ns *netNS) errorIfClosed() error {
	if ns.closed {
		return fmt.Errorf("%q has already been closed", ns.file.Name())
	}
	return nil
}

func (ns *netNS) Do(toRun func(NetNS) error) error {
	if err := ns.errorIfClosed(); err != nil {
		return err
	}

	containedCall := func(hostNS NetNS) error {
		threadNS, err := GetCurrentNS()
		if err != nil {
			return fmt.Errorf("failed to open current netns: %v", err)
		}
		defer threadNS.Close()

		// switch to target namespace
		if err = ns.Set(); err != nil {
			return fmt.Errorf("error switching to ns %v: %v", ns.file.Name(), err)
		}
		defer func() {
			err := threadNS.Set() // switch back
			if err == nil {
				// Unlock the current thread only when we successfully switched back
				// to the original namespace; otherwise leave the thread locked which
				// will force the runtime to scrap the current thread, that is maybe
				// not as optimal but at least always safe to do.
				runtime.UnlockOSThread()
			}
		}()

		return toRun(hostNS)
	}

	// save a handle to current network namespace
	hostNS, err := GetCurrentNS()
	if err != nil {
		return fmt.Errorf("Failed to open current namespace: %v", err)
	}
	defer hostNS.Close()

	var wg sync.WaitGroup
	wg.Add(1)

	// Start the callback in a new green thread so that if we later fail
	// to switch the namespace back to the original one, we can safely
	// leave the thread locked to die without a risk of the current thread
	// left lingering with incorrect namespace.
	var innerError error
	go func() {
		defer wg.Done()
		runtime.LockOSThread()
		innerError = containedCall(hostNS)
	}()
	wg.Wait()

	return innerError
}

// WithNetNSPath executes the passed closure under the given network
// namespace, restoring the original namespace afterwards.
func WithNetNSPath(nspath string, toRun func(NetNS) error) error {
	ns, err := GetNS(nspath)
	if err != nil {
		return err
	}
	defer ns.Close()
	return ns.Do(toRun)
}
<|endoftext|>


<commit_before>package pwalk

import (
	"os"
	"path/filepath"
	"runtime"
	"sync"

	"github.com/pkg/errors"
)

type WalkFunc = filepath.WalkFunc

// Walk is a wrapper for filepath.Walk which can call multiple walkFn
// in parallel, allowing to handle each item concurrently. A maximum of
// twice the runtime.NumCPU() walkFn will be called at any one time.
// If you want to change the maximum, use WalkN instead.
//
// The order of calls is non-deterministic.
//
// Note that this implementation only supports primitive error handling:
//
// * no errors are ever passed to WalkFn
//
// * once a walkFn returns any error, all further processing stops
//   and the error is returned to the caller of Walk;
//
// * filepath.SkipDir is not supported;
//
// * if more than one walkFn instance will return an error, only one
//   of such errors will be propagated and returned by Walk, others
//   will be silently discarded.
//
func Walk(root string, walkFn WalkFunc) error {
	return WalkN(root, walkFn, runtime.NumCPU()*2)
}

// WalkN is a wrapper for filepath.Walk which can call multiple walkFn
// in parallel, allowing to handle each item concurrently. A maximum of
// num walkFn will be called at any one time.
func WalkN(root string, walkFn WalkFunc, num int) error {
	// make sure limit is sensible
	if num < 1 {
		return errors.Errorf("walk(%q): num must be > 0", root)
	}

	files := make(chan *walkArgs, 2*num)
	errCh := make(chan error, 1) // get the first error, ignore others

	// Start walking a tree asap
	var err error
	go func() {
		err = filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
			if err != nil {
				close(files)
				return err
			}
			// add a file to the queue unless a callback sent an error
			select {
			case e := <-errCh:
				close(files)
				return e
			default:
				files <- &walkArgs{path: p, info: &info}
				return nil
			}
		})
		if err == nil {
			close(files)
		}
	}()

	var wg sync.WaitGroup
	wg.Add(num)
	for i := 0; i < num; i++ {
		go func() {
			for file := range files {
				if e := walkFn(file.path, *file.info, nil); e != nil {
					select {
					case errCh <- e: // sent ok
					default: // buffer full
					}
				}
			}
			wg.Done()
		}()
	}

	wg.Wait()

	return err
}

// walkArgs holds the arguments that were passed to the Walk or WalkLimit
// functions.
type walkArgs struct {
	path string
	info *os.FileInfo
}
<commit_msg>pkg/pwalk: fix data race with err<commit_after>package pwalk

import (
	"os"
	"path/filepath"
	"runtime"
	"sync"

	"github.com/pkg/errors"
)

type WalkFunc = filepath.WalkFunc

// Walk is a wrapper for filepath.Walk which can call multiple walkFn
// in parallel, allowing to handle each item concurrently. A maximum of
// twice the runtime.NumCPU() walkFn will be called at any one time.
// If you want to change the maximum, use WalkN instead.
//
// The order of calls is non-deterministic.
//
// Note that this implementation only supports primitive error handling:
//
// * no errors are ever passed to WalkFn
//
// * once a walkFn returns any error, all further processing stops
//   and the error is returned to the caller of Walk;
//
// * filepath.SkipDir is not supported;
//
// * if more than one walkFn instance will return an error, only one
//   of such errors will be propagated and returned by Walk, others
//   will be silently discarded.
//
func Walk(root string, walkFn WalkFunc) error {
	return WalkN(root, walkFn, runtime.NumCPU()*2)
}

// WalkN is a wrapper for filepath.Walk which can call multiple walkFn
// in parallel, allowing to handle each item concurrently. A maximum of
// num walkFn will be called at any one time.
func WalkN(root string, walkFn WalkFunc, num int) error {
	// make sure limit is sensible
	if num < 1 {
		return errors.Errorf("walk(%q): num must be > 0", root)
	}

	files := make(chan *walkArgs, 2*num)
	errCh := make(chan error, 1) // get the first error, ignore others

	// Start walking a tree asap
	var (
		err error
		wg  sync.WaitGroup
	)
	wg.Add(1)
	go func() {
		err = filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
			if err != nil {
				close(files)
				return err
			}
			// add a file to the queue unless a callback sent an error
			select {
			case e := <-errCh:
				close(files)
				return e
			default:
				files <- &walkArgs{path: p, info: &info}
				return nil
			}
		})
		if err == nil {
			close(files)
		}
		wg.Done()
	}()

	wg.Add(num)
	for i := 0; i < num; i++ {
		go func() {
			for file := range files {
				if e := walkFn(file.path, *file.info, nil); e != nil {
					select {
					case errCh <- e: // sent ok
					default: // buffer full
					}
				}
			}
			wg.Done()
		}()
	}

	wg.Wait()

	return err
}

// walkArgs holds the arguments that were passed to the Walk or WalkLimit
// functions.
type walkArgs struct {
	path string
	info *os.FileInfo
}
<|endoftext|>


<commit_before>package storage

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	stringz "github.com/appscode/go/strings"
	"github.com/appscode/go/types"
	meta_util "github.com/appscode/kutil/meta"
	otx "github.com/appscode/osm/context"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/session"
	_s3 "github.com/aws/aws-sdk-go/service/s3"
	"github.com/ghodss/yaml"
	"github.com/graymeta/stow"
	"github.com/graymeta/stow/azure"
	gcs "github.com/graymeta/stow/google"
	"github.com/graymeta/stow/local"
	"github.com/graymeta/stow/s3"
	"github.com/graymeta/stow/swift"
	api "github.com/kubedb/apimachinery/apis/kubedb/v1alpha1"
	"github.com/pkg/errors"
	core "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

const (
	SecretMountPath = "/etc/osm"
	CaCertFileName  = "ca.crt"
)

// NewOSMSecret creates a secret that contains the config file of OSM.
// So, generally, if this secret is mounted in `etc/osm`,
// the tree of `/etc/osm` directory will be similar to,
//
// /etc/osm
// └── config
//
// However, if the EndPoint is `S3 Minio Server`, then the secret will contain two file,
// `config` and `ca.crt`. So, the tree of the file path will look as,
//
// /etc/osm
// ├── ca.crt
// └── config

func NewOSMSecret(client kubernetes.Interface, snapshot *api.Snapshot) (*core.Secret, error) {
	osmCtx, err := NewOSMContext(client, snapshot.Spec.SnapshotStorageSpec, snapshot.Namespace)
	if err != nil {
		return nil, err
	}
	osmCfg := &otx.OSMConfig{
		CurrentContext: osmCtx.Name,
		Contexts:       []*otx.Context{osmCtx},
	}
	osmBytes, err := yaml.Marshal(osmCfg)
	if err != nil {
		return nil, err
	}
	return upserCaCertFile(osmCtx, &core.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name:      snapshot.OSMSecretName(),
			Namespace: snapshot.Namespace,
		},
		Data: map[string][]byte{
			"config": osmBytes,
		},
	})
}

func upserCaCertFile(osmCtx *otx.Context, secret *core.Secret) (*core.Secret, error) {
	if osmCtx != nil {
		if certFileName, err := meta_util.GetStringValue(osmCtx.Config, s3.ConfigCACertFile); err == nil {
			caCertData, err := ioutil.ReadFile(certFileName)
			if err != nil {
				return nil, fmt.Errorf("error in reading CaCertFile in path %v, err: %v", certFileName, err)
			}
			secret.Data[CaCertFileName] = caCertData
		}
	}
	return secret, nil
}

func CheckBucketAccess(client kubernetes.Interface, spec api.SnapshotStorageSpec, namespace string) error {
	cfg, err := NewOSMContext(client, spec, namespace)
	if err != nil {
		return err
	}
	loc, err := stow.Dial(cfg.Provider, cfg.Config)
	if err != nil {
		return err
	}
	c, err := spec.Container()
	if err != nil {
		return err
	}
	container, err := loc.Container(c)
	if err != nil {
		return err
	}
	r := bytes.NewReader([]byte("CheckBucketAccess"))
	item, err := container.Put(".kubedb", r, r.Size(), nil)
	if err != nil {
		return err
	}
	if err := container.RemoveItem(item.ID()); err != nil {
		return err
	}
	return nil
}

func NewOSMContext(client kubernetes.Interface, spec api.SnapshotStorageSpec, namespace string) (*otx.Context, error) {
	config := make(map[string][]byte)

	if spec.StorageSecretName != "" {
		secret, err := client.CoreV1().Secrets(namespace).Get(spec.StorageSecretName, metav1.GetOptions{})
		if err != nil {
			return nil, err
		}
		config = secret.Data
	}

	nc := &otx.Context{
		Name:   "kubedb",
		Config: stow.ConfigMap{},
	}

	if spec.S3 != nil {
		nc.Provider = s3.Kind

		keyID, foundKeyID := config[api.AWS_ACCESS_KEY_ID]
		key, foundKey := config[api.AWS_SECRET_ACCESS_KEY]
		if foundKey && foundKeyID {
			nc.Config[s3.ConfigAccessKeyID] = string(keyID)
			nc.Config[s3.ConfigSecretKey] = string(key)
			nc.Config[s3.ConfigAuthType] = "accesskey"
		} else {
			nc.Config[s3.ConfigAuthType] = "iam"
		}
		if strings.HasSuffix(spec.S3.Endpoint, ".amazonaws.com") {
			// find region
			var sess *session.Session
			var err error
			if nc.Config[s3.ConfigAuthType] == "iam" {
				sess, err = session.NewSessionWithOptions(session.Options{
					Config: *aws.NewConfig(),
					// Support MFA when authing using assumed roles.
					SharedConfigState:       session.SharedConfigEnable,
					AssumeRoleTokenProvider: stscreds.StdinTokenProvider,
				})
			} else {
				config := &aws.Config{
					Credentials: credentials.NewStaticCredentials(string(keyID), string(key), ""),
					Region:      aws.String("us-east-1"),
				}
				sess, err = session.NewSessionWithOptions(session.Options{
					Config: *config,
					// Support MFA when authing using assumed roles.
					SharedConfigState:       session.SharedConfigEnable,
					AssumeRoleTokenProvider: stscreds.StdinTokenProvider,
				})
			}
			if err != nil {
				return nil, err
			}
			svc := _s3.New(sess)
			out, err := svc.GetBucketLocation(&_s3.GetBucketLocationInput{
				Bucket: types.StringP(spec.S3.Bucket),
			})
			nc.Config[s3.ConfigRegion] = stringz.Val(types.String(out.LocationConstraint), "us-east-1")
		} else {
			nc.Config[s3.ConfigEndpoint] = spec.S3.Endpoint
			u, err := url.Parse(spec.S3.Endpoint)
			if err != nil {
				return nil, err
			}
			nc.Config[s3.ConfigDisableSSL] = strconv.FormatBool(u.Scheme == "http")

			cacertData, ok := config[api.CA_CERT_DATA]
			if ok && u.Scheme == "https" {
				certFileName := filepath.Join(SecretMountPath, CaCertFileName)
				err = os.MkdirAll(filepath.Dir(certFileName), 0755)
				if err != nil {
					return nil, err
				}
				err = ioutil.WriteFile(certFileName, cacertData, 0755)
				if err != nil {
					return nil, err
				}
				nc.Config[s3.ConfigCACertFile] = certFileName
			}
		}
		return nc, nil
	} else if spec.GCS != nil {
		nc.Provider = gcs.Kind
		nc.Config[gcs.ConfigProjectId] = string(config[api.GOOGLE_PROJECT_ID])
		nc.Config[gcs.ConfigJSON] = string(config[api.GOOGLE_SERVICE_ACCOUNT_JSON_KEY])
		return nc, nil
	} else if spec.Azure != nil {
		nc.Provider = azure.Kind
		nc.Config[azure.ConfigAccount] = string(config[api.AZURE_ACCOUNT_NAME])
		nc.Config[azure.ConfigKey] = string(config[api.AZURE_ACCOUNT_KEY])
		return nc, nil
	} else if spec.Local != nil {
		nc.Provider = local.Kind
		nc.Config[local.ConfigKeyPath] = spec.Local.MountPath
		return nc, nil
	} else if spec.Swift != nil {
		nc.Provider = swift.Kind
		// https://github.com/restic/restic/blob/master/src/restic/backend/swift/config.go
		for _, val := range []struct {
			stowKey   string
			secretKey string
		}{
			// v2/v3 specific
			{swift.ConfigUsername, api.OS_USERNAME},
			{swift.ConfigKey, api.OS_PASSWORD},
			{swift.ConfigRegion, api.OS_REGION_NAME},
			{swift.ConfigTenantAuthURL, api.OS_AUTH_URL},

			// v3 specific
			{swift.ConfigDomain, api.OS_USER_DOMAIN_NAME},
			{swift.ConfigTenantName, api.OS_PROJECT_NAME},
			{swift.ConfigTenantDomain, api.OS_PROJECT_DOMAIN_NAME},

			// v2 specific
			{swift.ConfigTenantId, api.OS_TENANT_ID},
			{swift.ConfigTenantName, api.OS_TENANT_NAME},

			// v1 specific
			{swift.ConfigTenantAuthURL, api.ST_AUTH},
			{swift.ConfigUsername, api.ST_USER},
			{swift.ConfigKey, api.ST_KEY},

			// Manual authentication
			{swift.ConfigStorageURL, api.OS_STORAGE_URL},
			{swift.ConfigAuthToken, api.OS_AUTH_TOKEN},
		} {
			if _, exists := nc.Config.Config(val.stowKey); !exists {
				nc.Config[val.stowKey] = string(config[val.secretKey])
			}
		}
		return nc, nil
	}
	return nil, errors.New("no storage provider is configured")
}
<commit_msg>Don't ignore error for s3 GetBucketLocation api call (#230)<commit_after>package storage

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	stringz "github.com/appscode/go/strings"
	"github.com/appscode/go/types"
	meta_util "github.com/appscode/kutil/meta"
	otx "github.com/appscode/osm/context"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/session"
	_s3 "github.com/aws/aws-sdk-go/service/s3"
	"github.com/ghodss/yaml"
	"github.com/graymeta/stow"
	"github.com/graymeta/stow/azure"
	gcs "github.com/graymeta/stow/google"
	"github.com/graymeta/stow/local"
	"github.com/graymeta/stow/s3"
	"github.com/graymeta/stow/swift"
	api "github.com/kubedb/apimachinery/apis/kubedb/v1alpha1"
	"github.com/pkg/errors"
	core "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

const (
	SecretMountPath = "/etc/osm"
	CaCertFileName  = "ca.crt"
)

// NewOSMSecret creates a secret that contains the config file of OSM.
// So, generally, if this secret is mounted in `etc/osm`,
// the tree of `/etc/osm` directory will be similar to,
//
// /etc/osm
// └── config
//
// However, if the EndPoint is `S3 Minio Server`, then the secret will contain two file,
// `config` and `ca.crt`. So, the tree of the file path will look as,
//
// /etc/osm
// ├── ca.crt
// └── config

func NewOSMSecret(client kubernetes.Interface, snapshot *api.Snapshot) (*core.Secret, error) {
	osmCtx, err := NewOSMContext(client, snapshot.Spec.SnapshotStorageSpec, snapshot.Namespace)
	if err != nil {
		return nil, err
	}
	osmCfg := &otx.OSMConfig{
		CurrentContext: osmCtx.Name,
		Contexts:       []*otx.Context{osmCtx},
	}
	osmBytes, err := yaml.Marshal(osmCfg)
	if err != nil {
		return nil, err
	}
	return upserCaCertFile(osmCtx, &core.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name:      snapshot.OSMSecretName(),
			Namespace: snapshot.Namespace,
		},
		Data: map[string][]byte{
			"config": osmBytes,
		},
	})
}

func upserCaCertFile(osmCtx *otx.Context, secret *core.Secret) (*core.Secret, error) {
	if osmCtx != nil {
		if certFileName, err := meta_util.GetStringValue(osmCtx.Config, s3.ConfigCACertFile); err == nil {
			caCertData, err := ioutil.ReadFile(certFileName)
			if err != nil {
				return nil, fmt.Errorf("error in reading CaCertFile in path %v, err: %v", certFileName, err)
			}
			secret.Data[CaCertFileName] = caCertData
		}
	}
	return secret, nil
}

func CheckBucketAccess(client kubernetes.Interface, spec api.SnapshotStorageSpec, namespace string) error {
	cfg, err := NewOSMContext(client, spec, namespace)
	if err != nil {
		return err
	}
	loc, err := stow.Dial(cfg.Provider, cfg.Config)
	if err != nil {
		return err
	}
	c, err := spec.Container()
	if err != nil {
		return err
	}
	container, err := loc.Container(c)
	if err != nil {
		return err
	}
	r := bytes.NewReader([]byte("CheckBucketAccess"))
	item, err := container.Put(".kubedb", r, r.Size(), nil)
	if err != nil {
		return err
	}
	if err := container.RemoveItem(item.ID()); err != nil {
		return err
	}
	return nil
}

func NewOSMContext(client kubernetes.Interface, spec api.SnapshotStorageSpec, namespace string) (*otx.Context, error) {
	config := make(map[string][]byte)

	if spec.StorageSecretName != "" {
		secret, err := client.CoreV1().Secrets(namespace).Get(spec.StorageSecretName, metav1.GetOptions{})
		if err != nil {
			return nil, err
		}
		config = secret.Data
	}

	nc := &otx.Context{
		Name:   "kubedb",
		Config: stow.ConfigMap{},
	}

	if spec.S3 != nil {
		nc.Provider = s3.Kind

		keyID, foundKeyID := config[api.AWS_ACCESS_KEY_ID]
		key, foundKey := config[api.AWS_SECRET_ACCESS_KEY]
		if foundKey && foundKeyID {
			nc.Config[s3.ConfigAccessKeyID] = string(keyID)
			nc.Config[s3.ConfigSecretKey] = string(key)
			nc.Config[s3.ConfigAuthType] = "accesskey"
		} else {
			nc.Config[s3.ConfigAuthType] = "iam"
		}
		if strings.HasSuffix(spec.S3.Endpoint, ".amazonaws.com") {
			// find region
			var sess *session.Session
			var err error
			if nc.Config[s3.ConfigAuthType] == "iam" {
				sess, err = session.NewSessionWithOptions(session.Options{
					Config: *aws.NewConfig(),
					// Support MFA when authing using assumed roles.
					SharedConfigState:       session.SharedConfigEnable,
					AssumeRoleTokenProvider: stscreds.StdinTokenProvider,
				})
			} else {
				sess, err = session.NewSessionWithOptions(session.Options{
					Config: aws.Config{
						Credentials: credentials.NewStaticCredentials(string(keyID), string(key), ""),
						Region:      aws.String("us-east-1"),
					},
					// Support MFA when authing using assumed roles.
					SharedConfigState:       session.SharedConfigEnable,
					AssumeRoleTokenProvider: stscreds.StdinTokenProvider,
				})
			}
			if err != nil {
				return nil, err
			}
			svc := _s3.New(sess)
			out, err := svc.GetBucketLocation(&_s3.GetBucketLocationInput{
				Bucket: types.StringP(spec.S3.Bucket),
			})
			if err != nil {
				return nil, err
			}
			nc.Config[s3.ConfigRegion] = stringz.Val(types.String(out.LocationConstraint), "us-east-1")
		} else {
			nc.Config[s3.ConfigEndpoint] = spec.S3.Endpoint
			u, err := url.Parse(spec.S3.Endpoint)
			if err != nil {
				return nil, err
			}
			nc.Config[s3.ConfigDisableSSL] = strconv.FormatBool(u.Scheme == "http")

			cacertData, ok := config[api.CA_CERT_DATA]
			if ok && u.Scheme == "https" {
				certFileName := filepath.Join(SecretMountPath, CaCertFileName)
				err = os.MkdirAll(filepath.Dir(certFileName), 0755)
				if err != nil {
					return nil, err
				}
				err = ioutil.WriteFile(certFileName, cacertData, 0755)
				if err != nil {
					return nil, err
				}
				nc.Config[s3.ConfigCACertFile] = certFileName
			}
		}
		return nc, nil
	} else if spec.GCS != nil {
		nc.Provider = gcs.Kind
		nc.Config[gcs.ConfigProjectId] = string(config[api.GOOGLE_PROJECT_ID])
		nc.Config[gcs.ConfigJSON] = string(config[api.GOOGLE_SERVICE_ACCOUNT_JSON_KEY])
		return nc, nil
	} else if spec.Azure != nil {
		nc.Provider = azure.Kind
		nc.Config[azure.ConfigAccount] = string(config[api.AZURE_ACCOUNT_NAME])
		nc.Config[azure.ConfigKey] = string(config[api.AZURE_ACCOUNT_KEY])
		return nc, nil
	} else if spec.Local != nil {
		nc.Provider = local.Kind
		nc.Config[local.ConfigKeyPath] = spec.Local.MountPath
		return nc, nil
	} else if spec.Swift != nil {
		nc.Provider = swift.Kind
		// https://github.com/restic/restic/blob/master/src/restic/backend/swift/config.go
		for _, val := range []struct {
			stowKey   string
			secretKey string
		}{
			// v2/v3 specific
			{swift.ConfigUsername, api.OS_USERNAME},
			{swift.ConfigKey, api.OS_PASSWORD},
			{swift.ConfigRegion, api.OS_REGION_NAME},
			{swift.ConfigTenantAuthURL, api.OS_AUTH_URL},

			// v3 specific
			{swift.ConfigDomain, api.OS_USER_DOMAIN_NAME},
			{swift.ConfigTenantName, api.OS_PROJECT_NAME},
			{swift.ConfigTenantDomain, api.OS_PROJECT_DOMAIN_NAME},

			// v2 specific
			{swift.ConfigTenantId, api.OS_TENANT_ID},
			{swift.ConfigTenantName, api.OS_TENANT_NAME},

			// v1 specific
			{swift.ConfigTenantAuthURL, api.ST_AUTH},
			{swift.ConfigUsername, api.ST_USER},
			{swift.ConfigKey, api.ST_KEY},

			// Manual authentication
			{swift.ConfigStorageURL, api.OS_STORAGE_URL},
			{swift.ConfigAuthToken, api.OS_AUTH_TOKEN},
		} {
			if _, exists := nc.Config.Config(val.stowKey); !exists {
				nc.Config[val.stowKey] = string(config[val.secretKey])
			}
		}
		return nc, nil
	}
	return nil, errors.New("no storage provider is configured")
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/pkg/errors"

	fluxerr "github.com/weaveworks/flux/errors"
)

func run(args []string) int {
	rootCmd := newRoot().Command()
	rootCmd.SetArgs(args)
	if cmd, err := rootCmd.ExecuteC(); err != nil {
		err = errors.Cause(err)
		switch err := err.(type) {
		case *fluxerr.Error:
			cmd.Println("== Error ==\n\n" + err.Help)
		default:
			cmd.Println("Error: " + err.Error())
			cmd.Printf("Run '%v --help' for usage.\n", cmd.CommandPath())
		}
		return 1
	}
	return 0
}

func main() {
	os.Exit(run(os.Args[1:]))
}
<commit_msg>show more context for errors from fluxctl<commit_after>package main

import (
	"os"

	"github.com/pkg/errors"

	fluxerr "github.com/weaveworks/flux/errors"
)

func run(args []string) int {
	rootCmd := newRoot().Command()
	rootCmd.SetArgs(args)
	if cmd, err := rootCmd.ExecuteC(); err != nil {
		// Format flux-specific errors. They can come wrapped,
		// so we use the cause instead.
		if cause, ok := errors.Cause(err).(*fluxerr.Error); ok {
			cmd.Println("== Error ==\n\n" + cause.Help)
		}
		cmd.Println("Error: " + err.Error())
		cmd.Printf("Run '%v --help' for usage.\n", cmd.CommandPath())
		return 1
	}
	return 0
}

func main() {
	os.Exit(run(os.Args[1:]))
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"io"
	"net/http"
	"time"

	"github.com/gorilla/mux"

	"github.com/skygeario/skygear-server/pkg/core/logging"
	coreMiddleware "github.com/skygeario/skygear-server/pkg/core/middleware"
	"github.com/skygeario/skygear-server/pkg/core/server"
	gatewayConfig "github.com/skygeario/skygear-server/pkg/gateway/config"
	pqStore "github.com/skygeario/skygear-server/pkg/gateway/db/pq"
	"github.com/skygeario/skygear-server/pkg/gateway/handler"
	"github.com/skygeario/skygear-server/pkg/gateway/middleware"
	"github.com/skygeario/skygear-server/pkg/gateway/provider"
)

var config gatewayConfig.Configuration

func init() {
	// logging initialization
	logging.SetModule("gateway")

	logger := logging.LoggerEntry("gateway")
	if err := config.ReadFromEnv(); err != nil {
		logger.WithError(err).Panic(
			"Fail to load config for starting gateway server")
	}

	logger.WithField("config", config).Debug("Gateway config")
}

func main() {
	logger := logging.LoggerEntry("gateway")

	// create gateway store
	store, connErr := pqStore.NewGatewayStore(
		context.Background(),
		config.DB.ConnectionStr,
	)
	if connErr != nil {
		logger.WithError(connErr).Panic("Fail to create db conn")
	}
	defer store.Close()

	r := mux.NewRouter()
	r.HandleFunc("/healthz", HealthCheckHandler)

	r = r.PathPrefix("/").Subrouter()
	// RecoverMiddleware must come first
	r.Use(coreMiddleware.RecoverMiddleware{
		RecoverHandler: server.DefaultRecoverPanicHandler,
	}.Handle)

	r.Use(middleware.FindAppMiddleware{Store: store}.Handle)

	gr := r.PathPrefix("/_{gear}").Subrouter()

	gr.Use(coreMiddleware.TenantConfigurationMiddleware{
		ConfigurationProvider: provider.GatewayTenantConfigurationProvider{
			Store: store,
		},
	}.Handle)
	gr.Use(middleware.TenantAuthzMiddleware{
		Store:        store,
		RouterConfig: config.Router,
	}.Handle)

	gr.HandleFunc("/{rest:.*}", handler.NewGearHandler("rest"))

	cr := r.PathPrefix("/").Subrouter()

	cr.Use(middleware.FindCloudCodeMiddleware{
		RestPathIdentifier: "rest",
		Store:              store,
	}.Handle)

	cr.HandleFunc("/{rest:.*}", handler.NewCloudCodeHandler(config.Router))

	srv := &http.Server{
		Addr: config.HTTP.Host,
		// Good practice to set timeouts to avoid Slowloris attacks.
		WriteTimeout: time.Second * 15,
		ReadTimeout:  time.Second * 15,
		IdleTimeout:  time.Second * 60,
		Handler:      r, // Pass our instance of gorilla/mux in.
	}

	logger.Info("Start gateway server")
	if err := srv.ListenAndServe(); err != nil {
		logger.Errorf("Fail to start gateway server %v", err)
	}
}

// HealthCheckHandler is basic handler for server health check
func HealthCheckHandler(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusOK)
	w.Header().Set("Content-Type", "application/json")
	io.WriteString(w, "OK")
}
<commit_msg>Implement injecting auth info HTTP headers<commit_after>package main

import (
	"context"
	"io"
	"net/http"
	"time"

	"github.com/gorilla/mux"

	"github.com/skygeario/skygear-server/pkg/core/db"
	"github.com/skygeario/skygear-server/pkg/core/logging"
	coreMiddleware "github.com/skygeario/skygear-server/pkg/core/middleware"
	"github.com/skygeario/skygear-server/pkg/core/server"
	"github.com/skygeario/skygear-server/pkg/gateway"
	gatewayConfig "github.com/skygeario/skygear-server/pkg/gateway/config"
	pqStore "github.com/skygeario/skygear-server/pkg/gateway/db/pq"
	"github.com/skygeario/skygear-server/pkg/gateway/handler"
	"github.com/skygeario/skygear-server/pkg/gateway/middleware"
	"github.com/skygeario/skygear-server/pkg/gateway/provider"
)

var config gatewayConfig.Configuration

func init() {
	// logging initialization
	logging.SetModule("gateway")

	logger := logging.LoggerEntry("gateway")
	if err := config.ReadFromEnv(); err != nil {
		logger.WithError(err).Panic(
			"Fail to load config for starting gateway server")
	}

	logger.WithField("config", config).Debug("Gateway config")
}

func main() {
	logger := logging.LoggerEntry("gateway")

	// create gateway store
	store, connErr := pqStore.NewGatewayStore(
		context.Background(),
		config.DB.ConnectionStr,
	)
	if connErr != nil {
		logger.WithError(connErr).Panic("Fail to create db conn")
	}
	defer store.Close()

	gatewayDependency := gateway.DependencyMap{}

	r := mux.NewRouter()
	r.HandleFunc("/healthz", HealthCheckHandler)

	r = r.PathPrefix("/").Subrouter()
	// RecoverMiddleware must come first
	r.Use(coreMiddleware.RecoverMiddleware{
		RecoverHandler: server.DefaultRecoverPanicHandler,
	}.Handle)

	r.Use(middleware.FindAppMiddleware{Store: store}.Handle)

	gr := r.PathPrefix("/_{gear}").Subrouter()

	gr.Use(coreMiddleware.TenantConfigurationMiddleware{
		ConfigurationProvider: provider.GatewayTenantConfigurationProvider{
			Store: store,
		},
	}.Handle)
	gr.Use(middleware.TenantAuthzMiddleware{
		Store:        store,
		RouterConfig: config.Router,
	}.Handle)

	gr.HandleFunc("/{rest:.*}", handler.NewGearHandler("rest"))

	cr := r.PathPrefix("/").Subrouter()

	cr.Use(func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			r = db.InitRequestDBContext(r)
			next.ServeHTTP(w, r)
		})
	})

	cr.Use(coreMiddleware.TenantConfigurationMiddleware{
		ConfigurationProvider: provider.GatewayTenantConfigurationProvider{
			Store: store,
		},
	}.Handle)

	cr.Use(middleware.FindCloudCodeMiddleware{
		RestPathIdentifier: "rest",
		Store:              store,
	}.Handle)

	cr.Use(middleware.Injecter{
		Middleware: &middleware.AuthInfoMiddleware{},
		Dependency: gatewayDependency,
	}.Handle)

	cr.HandleFunc("/{rest:.*}", handler.NewCloudCodeHandler(config.Router))

	srv := &http.Server{
		Addr: config.HTTP.Host,
		// Good practice to set timeouts to avoid Slowloris attacks.
		WriteTimeout: time.Second * 15,
		ReadTimeout:  time.Second * 15,
		IdleTimeout:  time.Second * 60,
		Handler:      r, // Pass our instance of gorilla/mux in.
	}

	logger.Info("Start gateway server")
	if err := srv.ListenAndServe(); err != nil {
		logger.Errorf("Fail to start gateway server %v", err)
	}
}

// HealthCheckHandler is basic handler for server health check
func HealthCheckHandler(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusOK)
	w.Header().Set("Content-Type", "application/json")
	io.WriteString(w, "OK")
}
<|endoftext|>