<commit_before>package avatica

import (
	"database/sql/driver"
	"io"
	"time"

	"reflect"

	"math"

	"fmt"

	"github.com/Boostport/avatica/message"
	"golang.org/x/net/context"
)

type precisionScale struct {
	precision int64
	scale     int64
}

type column struct {
	name           string
	typeName       string
	rep            message.Rep
	length         int64
	nullable       bool
	precisionScale *precisionScale
	scanType       reflect.Type
}

type resultSet struct {
	columns    []*column
	done       bool
	offset     uint64
	data       [][]*message.TypedValue
	currentRow int
}

type rows struct {
	conn             *conn
	statementID      uint32
	resultSets       []*resultSet
	currentResultSet int
}

// Columns returns the names of the columns. The number of
// columns of the result is inferred from the length of the
// slice.  If a particular column name isn't known, an empty
// string should be returned for that entry.
func (r *rows) Columns() []string {

	cols := []string{}

	for _, column := range r.resultSets[r.currentResultSet].columns {
		cols = append(cols, column.name)
	}

	return cols
}

// Close closes the rows iterator.
func (r *rows) Close() error {

	r.conn = nil
	return nil
}

// Next is called to populate the next row of data into
// the provided slice. The provided slice will be the same
// size as the Columns() are wide.
//
// The dest slice may be populated only with
// a driver Value type, but excluding string.
// All string values must be converted to []byte.
//
// Next should return io.EOF when there are no more rows.
func (r *rows) Next(dest []driver.Value) error {

	resultSet := r.resultSets[r.currentResultSet]

	if resultSet.currentRow >= len(resultSet.data) {

		if resultSet.done {
			// Finished iterating through all results
			return io.EOF
		}

		// Fetch more results from the server
		res, err := r.conn.httpClient.post(context.Background(), &message.FetchRequest{
			ConnectionId: r.conn.connectionId,
			StatementId:  r.statementID,
			Offset:       resultSet.offset,
			FrameMaxSize: r.conn.config.frameMaxSize,
		})

		if err != nil {
			return err
		}

		frame := res.(*message.FetchResponse).Frame

		data := [][]*message.TypedValue{}

		// In some cases the server does not return done as true
		// until it returns a result with no rows
		if len(frame.Rows) == 0 {
			return io.EOF
		}

		for _, row := range frame.Rows {
			rowData := []*message.TypedValue{}

			for _, col := range row.Value {
				rowData = append(rowData, col.ScalarValue)
			}

			data = append(data, rowData)
		}

		resultSet.done = frame.Done
		resultSet.data = data
		resultSet.currentRow = 0

	}

	for i, val := range resultSet.data[resultSet.currentRow] {
		dest[i] = typedValueToNative(resultSet.columns[i].rep, val, r.conn.config)
	}

	resultSet.currentRow++

	return nil
}

// newRows create a new set of rows from a result set.
func newRows(conn *conn, statementID uint32, resultSets []*message.ResultSetResponse) *rows {

	rsets := []*resultSet{}

	for _, result := range resultSets {
		columns := []*column{}

		for _, col := range result.Signature.Columns {

			column := &column{
				name:     col.ColumnName,
				typeName: col.Type.Name,
				nullable: col.Nullable != 0,
			}

			// Handle precision and length
			switch col.Type.Name {
			case "DECIMAL":

				precision := int64(col.Precision)

				if precision == 0 {
					precision = math.MaxInt64
				}

				scale := int64(col.Scale)

				if scale == 0 {
					scale = math.MaxInt64
				}

				column.precisionScale = &precisionScale{
					precision: precision,
					scale:     scale,
				}
			case "VARCHAR", "CHAR", "BINARY":
				column.length = int64(col.Precision)
			case "VARBINARY":
				column.length = math.MaxInt64
			}

			// Handle scan types
			switch col.Type.Name {
			case "INTEGER", "UNSIGNED_INT", "BIGINT", "UNSIGNED_LONG", "TINYINT", "UNSIGNED_TINYINT", "SMALLINT", "UNSIGNED_SMALLINT":
				column.scanType = reflect.TypeOf(int64(0))

			case "FLOAT", "UNSIGNED_FLOAT", "DOUBLE", "UNSIGNED_DOUBLE":
				column.scanType = reflect.TypeOf(float64(0))

			case "DECIMAL", "VARCHAR", "CHAR":
				column.scanType = reflect.TypeOf("")

			case "BOOLEAN":
				column.scanType = reflect.TypeOf(bool(false))

			case "TIME", "DATE", "TIMESTAMP", "UNSIGNED_TIME", "UNSIGNED_DATE", "UNSIGNED_TIMESTAMP":
				column.scanType = reflect.TypeOf(time.Time{})

			case "BINARY", "VARBINARY":
				column.scanType = reflect.TypeOf([]byte{})

			default:
				panic(fmt.Sprintf("scantype for %s is not implemented", col.Type.Name))
			}

			// Handle rep type special cases for decimals, floats, date, time and timestamp
			switch col.Type.Name {
			case "DECIMAL":
				column.rep = message.Rep_BIG_DECIMAL
			case "FLOAT":
				column.rep = message.Rep_FLOAT
			case "UNSIGNED_FLOAT":
				column.rep = message.Rep_FLOAT
			case "TIME", "UNSIGNED_TIME":
				column.rep = message.Rep_JAVA_SQL_TIME
			case "DATE", "UNSIGNED_DATE":
				column.rep = message.Rep_JAVA_SQL_DATE
			case "TIMESTAMP", "UNSIGNED_TIMESTAMP":
				column.rep = message.Rep_JAVA_SQL_TIMESTAMP
			default:
				column.rep = col.Type.Rep
			}

			columns = append(columns, column)
		}

		frame := result.FirstFrame

		data := [][]*message.TypedValue{}

		for _, row := range frame.Rows {
			rowData := []*message.TypedValue{}

			for _, col := range row.Value {
				rowData = append(rowData, col.ScalarValue)
			}

			data = append(data, rowData)
		}

		rsets = append(rsets, &resultSet{
			columns: columns,
			done:    frame.Done,
			offset:  frame.Offset,
			data:    data,
		})
	}

	return &rows{
		conn:             conn,
		statementID:      statementID,
		resultSets:       rsets,
		currentResultSet: 0,
	}
}

// typedValueToNative converts values from avatica's types to Go's native types
func typedValueToNative(rep message.Rep, v *message.TypedValue, config *Config) interface{} {

	switch rep {
	case message.Rep_BOOLEAN, message.Rep_PRIMITIVE_BOOLEAN:
		return v.BoolValue

	case message.Rep_STRING, message.Rep_PRIMITIVE_CHAR, message.Rep_CHARACTER, message.Rep_BIG_DECIMAL:
		return v.StringValue

	case message.Rep_FLOAT, message.Rep_PRIMITIVE_FLOAT:
		return float32(v.DoubleValue)

	case message.Rep_LONG,
		message.Rep_PRIMITIVE_LONG,
		message.Rep_INTEGER,
		message.Rep_PRIMITIVE_INT,
		message.Rep_BIG_INTEGER,
		message.Rep_NUMBER,
		message.Rep_BYTE,
		message.Rep_PRIMITIVE_BYTE,
		message.Rep_SHORT,
		message.Rep_PRIMITIVE_SHORT:
		return v.NumberValue

	case message.Rep_BYTE_STRING:
		return v.BytesValue

	case message.Rep_DOUBLE, message.Rep_PRIMITIVE_DOUBLE:
		return v.DoubleValue

	case message.Rep_JAVA_SQL_DATE, message.Rep_JAVA_UTIL_DATE:

		// We receive the number of days since 1970/1/1 from the server
		// Because a location can have multiple time zones due to daylight savings,
		// we first do all our calculations in UTC and then force the timezone to
		// the one the user has chosen.
		t, _ := time.ParseInLocation("2006-Jan-02", "1970-Jan-01", time.UTC)
		days := time.Hour * 24 * time.Duration(v.NumberValue)
		t = t.Add(days)

		return forceTimezone(t, config.location)

	case message.Rep_JAVA_SQL_TIME:

		// We receive the number of milliseconds since 00:00:00.000 from the server
		// Because a location can have multiple time zones due to daylight savings,
		// we first do all our calculations in UTC and then force the timezone to
		// the one the user has chosen.
		t, _ := time.ParseInLocation("15:04:05", "00:00:00", time.UTC)
		ms := time.Millisecond * time.Duration(v.NumberValue)
		t = t.Add(ms)
		return forceTimezone(t, config.location)

	case message.Rep_JAVA_SQL_TIMESTAMP:

		// We receive the number of milliseconds since 1970-01-01 00:00:00.000 from the server
		// Force to UTC for consistency because time.Unix uses the local timezone
		t := time.Unix(0, v.NumberValue*int64(time.Millisecond)).In(time.UTC)
		return forceTimezone(t, config.location)

	default:
		return nil
	}
}

// forceTimezone takes a time.Time and changes its location without shifting the timezone.
func forceTimezone(t time.Time, loc *time.Location) time.Time {
	return time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), loc)
}
<commit_msg>Fixes #34<commit_after>package avatica

import (
	"database/sql/driver"
	"io"
	"time"

	"reflect"

	"math"

	"fmt"

	"github.com/Boostport/avatica/message"
	"golang.org/x/net/context"
)

type precisionScale struct {
	precision int64
	scale     int64
}

type column struct {
	name           string
	typeName       string
	rep            message.Rep
	length         int64
	nullable       bool
	precisionScale *precisionScale
	scanType       reflect.Type
}

type resultSet struct {
	columns    []*column
	done       bool
	offset     uint64
	data       [][]*message.TypedValue
	currentRow int
}

type rows struct {
	conn             *conn
	statementID      uint32
	resultSets       []*resultSet
	currentResultSet int
}

// Columns returns the names of the columns. The number of
// columns of the result is inferred from the length of the
// slice.  If a particular column name isn't known, an empty
// string should be returned for that entry.
func (r *rows) Columns() []string {

	cols := []string{}

	for _, column := range r.resultSets[r.currentResultSet].columns {
		cols = append(cols, column.name)
	}

	return cols
}

// Close closes the rows iterator.
func (r *rows) Close() error {

	r.conn = nil
	return nil
}

// Next is called to populate the next row of data into
// the provided slice. The provided slice will be the same
// size as the Columns() are wide.
//
// The dest slice may be populated only with
// a driver Value type, but excluding string.
// All string values must be converted to []byte.
//
// Next should return io.EOF when there are no more rows.
func (r *rows) Next(dest []driver.Value) error {

	resultSet := r.resultSets[r.currentResultSet]

	if resultSet.currentRow >= len(resultSet.data) {

		if resultSet.done {
			// Finished iterating through all results
			return io.EOF
		}

		// Fetch more results from the server
		res, err := r.conn.httpClient.post(context.Background(), &message.FetchRequest{
			ConnectionId: r.conn.connectionId,
			StatementId:  r.statementID,
			Offset:       resultSet.offset,
			FrameMaxSize: r.conn.config.frameMaxSize,
		})

		if err != nil {
			return err
		}

		frame := res.(*message.FetchResponse).Frame

		data := [][]*message.TypedValue{}

		// In some cases the server does not return done as true
		// until it returns a result with no rows
		if len(frame.Rows) == 0 {
			return io.EOF
		}

		for _, row := range frame.Rows {
			rowData := []*message.TypedValue{}

			for _, col := range row.Value {
				rowData = append(rowData, col.ScalarValue)
			}

			data = append(data, rowData)
		}

		resultSet.done = frame.Done
		resultSet.data = data
		resultSet.currentRow = 0

	}

	for i, val := range resultSet.data[resultSet.currentRow] {
		dest[i] = typedValueToNative(resultSet.columns[i].rep, val, r.conn.config)
	}

	resultSet.currentRow++

	return nil
}

// newRows create a new set of rows from a result set.
func newRows(conn *conn, statementID uint32, resultSets []*message.ResultSetResponse) *rows {

	rsets := []*resultSet{}

	for _, result := range resultSets {
		if result.Signature == nil {
			break
		}
		columns := []*column{}

		for _, col := range result.Signature.Columns {

			column := &column{
				name:     col.ColumnName,
				typeName: col.Type.Name,
				nullable: col.Nullable != 0,
			}

			// Handle precision and length
			switch col.Type.Name {
			case "DECIMAL":

				precision := int64(col.Precision)

				if precision == 0 {
					precision = math.MaxInt64
				}

				scale := int64(col.Scale)

				if scale == 0 {
					scale = math.MaxInt64
				}

				column.precisionScale = &precisionScale{
					precision: precision,
					scale:     scale,
				}
			case "VARCHAR", "CHAR", "BINARY":
				column.length = int64(col.Precision)
			case "VARBINARY":
				column.length = math.MaxInt64
			}

			// Handle scan types
			switch col.Type.Name {
			case "INTEGER", "UNSIGNED_INT", "BIGINT", "UNSIGNED_LONG", "TINYINT", "UNSIGNED_TINYINT", "SMALLINT", "UNSIGNED_SMALLINT":
				column.scanType = reflect.TypeOf(int64(0))

			case "FLOAT", "UNSIGNED_FLOAT", "DOUBLE", "UNSIGNED_DOUBLE":
				column.scanType = reflect.TypeOf(float64(0))

			case "DECIMAL", "VARCHAR", "CHAR":
				column.scanType = reflect.TypeOf("")

			case "BOOLEAN":
				column.scanType = reflect.TypeOf(bool(false))

			case "TIME", "DATE", "TIMESTAMP", "UNSIGNED_TIME", "UNSIGNED_DATE", "UNSIGNED_TIMESTAMP":
				column.scanType = reflect.TypeOf(time.Time{})

			case "BINARY", "VARBINARY":
				column.scanType = reflect.TypeOf([]byte{})

			default:
				panic(fmt.Sprintf("scantype for %s is not implemented", col.Type.Name))
			}

			// Handle rep type special cases for decimals, floats, date, time and timestamp
			switch col.Type.Name {
			case "DECIMAL":
				column.rep = message.Rep_BIG_DECIMAL
			case "FLOAT":
				column.rep = message.Rep_FLOAT
			case "UNSIGNED_FLOAT":
				column.rep = message.Rep_FLOAT
			case "TIME", "UNSIGNED_TIME":
				column.rep = message.Rep_JAVA_SQL_TIME
			case "DATE", "UNSIGNED_DATE":
				column.rep = message.Rep_JAVA_SQL_DATE
			case "TIMESTAMP", "UNSIGNED_TIMESTAMP":
				column.rep = message.Rep_JAVA_SQL_TIMESTAMP
			default:
				column.rep = col.Type.Rep
			}

			columns = append(columns, column)
		}

		frame := result.FirstFrame

		data := [][]*message.TypedValue{}

		for _, row := range frame.Rows {
			rowData := []*message.TypedValue{}

			for _, col := range row.Value {
				rowData = append(rowData, col.ScalarValue)
			}

			data = append(data, rowData)
		}

		rsets = append(rsets, &resultSet{
			columns: columns,
			done:    frame.Done,
			offset:  frame.Offset,
			data:    data,
		})
	}

	return &rows{
		conn:             conn,
		statementID:      statementID,
		resultSets:       rsets,
		currentResultSet: 0,
	}
}

// typedValueToNative converts values from avatica's types to Go's native types
func typedValueToNative(rep message.Rep, v *message.TypedValue, config *Config) interface{} {

	switch rep {
	case message.Rep_BOOLEAN, message.Rep_PRIMITIVE_BOOLEAN:
		return v.BoolValue

	case message.Rep_STRING, message.Rep_PRIMITIVE_CHAR, message.Rep_CHARACTER, message.Rep_BIG_DECIMAL:
		return v.StringValue

	case message.Rep_FLOAT, message.Rep_PRIMITIVE_FLOAT:
		return float32(v.DoubleValue)

	case message.Rep_LONG,
		message.Rep_PRIMITIVE_LONG,
		message.Rep_INTEGER,
		message.Rep_PRIMITIVE_INT,
		message.Rep_BIG_INTEGER,
		message.Rep_NUMBER,
		message.Rep_BYTE,
		message.Rep_PRIMITIVE_BYTE,
		message.Rep_SHORT,
		message.Rep_PRIMITIVE_SHORT:
		return v.NumberValue

	case message.Rep_BYTE_STRING:
		return v.BytesValue

	case message.Rep_DOUBLE, message.Rep_PRIMITIVE_DOUBLE:
		return v.DoubleValue

	case message.Rep_JAVA_SQL_DATE, message.Rep_JAVA_UTIL_DATE:

		// We receive the number of days since 1970/1/1 from the server
		// Because a location can have multiple time zones due to daylight savings,
		// we first do all our calculations in UTC and then force the timezone to
		// the one the user has chosen.
		t, _ := time.ParseInLocation("2006-Jan-02", "1970-Jan-01", time.UTC)
		days := time.Hour * 24 * time.Duration(v.NumberValue)
		t = t.Add(days)

		return forceTimezone(t, config.location)

	case message.Rep_JAVA_SQL_TIME:

		// We receive the number of milliseconds since 00:00:00.000 from the server
		// Because a location can have multiple time zones due to daylight savings,
		// we first do all our calculations in UTC and then force the timezone to
		// the one the user has chosen.
		t, _ := time.ParseInLocation("15:04:05", "00:00:00", time.UTC)
		ms := time.Millisecond * time.Duration(v.NumberValue)
		t = t.Add(ms)
		return forceTimezone(t, config.location)

	case message.Rep_JAVA_SQL_TIMESTAMP:

		// We receive the number of milliseconds since 1970-01-01 00:00:00.000 from the server
		// Force to UTC for consistency because time.Unix uses the local timezone
		t := time.Unix(0, v.NumberValue*int64(time.Millisecond)).In(time.UTC)
		return forceTimezone(t, config.location)

	default:
		return nil
	}
}

// forceTimezone takes a time.Time and changes its location without shifting the timezone.
func forceTimezone(t time.Time, loc *time.Location) time.Time {
	return time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), loc)
}
<|endoftext|>


<commit_before>package messages

// Messages is a map of centralized strings to be used on the project
var Messages = map[string]string{
	"direct-not-allowed":             "this look like a direct message. Containers are related to a channels.",
	"admin-only":                     "Command available only for admins. ⛔",
	"command-not-found":              "Not sure what you are asking for. Type `@claimr help` for valid commands.",
	"same-name":                      "There is a container with the same name on this channel. Try a different one.",
	"added-to-channel":               "Container `%s` added to channel <#%s>.",
	"field-name-too-big":             "try a smaller container name up to %d characters",
	"field-name-required":            "can not continue without a container name 🙄",
	"container-not-found-on-channel": "I couldn't find the container `%s` on <#%s>.",
	"container-in-use":               "Container `%s` is already in use, try another one.",
	"fail-to-update":                 "Fail to update the container.",
	"container-claimed":              "Got it. Container `%s` is all yours <@%s>.",
	"container-in-use-by-other":      "Hum Container `%s` is not being used by you.",
	"container-in-use-by-this":       "Can't remove. Container `%s` is in used by <@%s> since _%s_.",
	"container-in-use-by-you":        "Container `%s` is already in use by you.",
	"container-free":                 "Got it. Container `%s` is now available.",
	"fail-getting-containers":        "Fail to list containers.",
	"empty-containers-list":          "No containers to list.",
	"containers-list":                "Here is a list of containers for this channel:",
	"no-level-provided":              "No log level provided, keeping in `%s`",
	"same-log-level":                 "Same log level than actual. Nothing change.",
	"invalid-log-level":              "not a valid logrus Level: \"%s\"",
	"level-log-changed":              "Log level changed from `%s` to `%s`",
	"x-purged":                       "%d Container rows purged.",
	"only-owner-can-remove":          "Only who created the container `%s` can remove it. Please check with <@%s>.",
	"container-removed":              "Container `%s` removed.",
	"container-created-by":           "Container `%s`.\nCreated by <@%s>.\n",
	"container-in-use-by-w-reason":   "In use by <@%s>%s since _%s_.",
	"shouldnt-mention-user":          "This message shouldn't contain user mentions.",
	"shouldnt-mention-channel":       "This message shouldn't contain channel mentions.",
	"x-admin-loaded":                 "%d admins loaded.",
}
<commit_msg>Fixing field-required and captalizing error messages<commit_after>package messages

// Messages is a map of centralized strings to be used on the project
var Messages = map[string]string{
	"direct-not-allowed":             "This look like a direct message. Containers are related to a channels.",
	"admin-only":                     "Command available only for admins. ⛔",
	"command-not-found":              "Not sure what you are asking for. Type `@claimr help` for valid commands.",
	"same-name":                      "There is a container with the same name on this channel. Try a different one.",
	"container-not-found":            "Unable to find a container with this name on this channel.",
	"added-to-channel":               "Container `%s` added to channel <#%s>.",
	"field-name-too-big":             "Try a smaller container name up to %d characters",
	"field-required":                 "Can not continue without a %s 🙄",
	"container-not-found-on-channel": "I couldn't find the container `%s` on <#%s>.",
	"container-in-use":               "Container `%s` is already in use, try another one.",
	"fail-to-update":                 "Fail to update the container.",
	"container-claimed":              "Got it. Container `%s` is all yours <@%s>.",
	"container-in-use-by-other":      "Hum Container `%s` is not being used by you.",
	"container-in-use-by-this":       "Can't remove. Container `%s` is in used by <@%s> since _%s_.",
	"container-in-use-by-you":        "Container `%s` is already in use by you.",
	"container-free":                 "Got it. Container `%s` is now available.",
	"fail-getting-containers":        "Fail to list containers.",
	"empty-containers-list":          "No containers to list.",
	"containers-list":                "Here is a list of containers for this channel:",
	"no-level-provided":              "No log level provided, keeping in `%s`",
	"same-log-level":                 "Same log level than actual. Nothing change.",
	"invalid-log-level":              "not a valid logrus Level: \"%s\"",
	"level-log-changed":              "Log level changed from `%s` to `%s`",
	"x-purged":                       "%d Container rows purged.",
	"only-owner-can-remove":          "Only who created the container `%s` can remove it. Please check with <@%s>.",
	"container-removed":              "Container `%s` removed.",
	"container-created-by":           "Container `%s`.\nCreated by <@%s>.\n",
	"container-in-use-by-w-reason":   "In use by <@%s>%s since _%s_.",
	"shouldnt-mention-user":          "This message shouldn't contain user mentions.",
	"shouldnt-mention-channel":       "This message shouldn't contain channel mentions.",
	"x-admin-loaded":                 "%d admins loaded.",
}
<|endoftext|>


<commit_before>package metadata

import "fmt"

type Metadata struct {
	Release      *Release      `yaml:"release,omitempty"`
	ProductFiles []ProductFile `yaml:"product_files,omitempty"`
	Dependencies []Dependency  `yaml:"dependencies,omitempty"`
	UpgradePaths []UpgradePath `yaml:"upgrade_paths,omitempty"`
	FileGroups   []FileGroup   `yaml:"file_groups,omitempty"`
}

type Release struct {
	Version               string   `yaml:"version"`
	ReleaseType           string   `yaml:"release_type"`
	EULASlug              string   `yaml:"eula_slug"`
	ReleaseDate           string   `yaml:"release_date"`
	Description           string   `yaml:"description"`
	ReleaseNotesURL       string   `yaml:"release_notes_url"`
	Availability          string   `yaml:"availability"`
	UserGroupIDs          []string `yaml:"user_group_ids,omitempty"`
	Controlled            bool     `yaml:"controlled"`
	ECCN                  string   `yaml:"eccn"`
	LicenseException      string   `yaml:"license_exception"`
	EndOfSupportDate      string   `yaml:"end_of_support_date"`
	EndOfGuidanceDate     string   `yaml:"end_of_guidance_date"`
	EndOfAvailabilityDate string   `yaml:"end_of_availability_date"`
}

type ProductFile struct {
	File         string `yaml:"file,omitempty"`
	Description  string `yaml:"description,omitempty"`
	UploadAs     string `yaml:"upload_as,omitempty"`
	AWSObjectKey string `yaml:"aws_object_key,omitempty"`
	FileType     string `yaml:"file_type,omitempty"`
	FileVersion  string `yaml:"file_version,omitempty"`
	MD5          string `yaml:"md5,omitempty"`
	ID           int    `yaml:"id,omitempty"`
}

type FileGroup struct {
	ID           int           `yaml:"id,omitempty"`
	Name         string        `yaml:"id,omitempty"`
	ProductFiles []ProductFile `yaml:"product_files,omitempty"`
}

type Dependency struct {
	Release DependentRelease `yaml:"release,omitempty"`
}

type UpgradePath struct {
	ID      int    `yaml:"id,omitempty"`
	Version string `yaml:"version,omitempty"`
}

type DependentRelease struct {
	ID      int     `yaml:"id,omitempty"`
	Version string  `yaml:"version,omitempty"`
	Product Product `yaml:"product,omitempty"`
}

type Product struct {
	ID   int    `yaml:"id,omitempty"`
	Slug string `yaml:"slug,omitempty"`
	Name string `yaml:"name,omitempty"`
}

func (m Metadata) Validate() error {
	for _, productFile := range m.ProductFiles {
		if productFile.File == "" {
			return fmt.Errorf("empty value for file")
		}
	}

	if m.Release == nil {
		return fmt.Errorf("missing required value %q", "release")
	}

	if m.Release.Version == "" {
		return fmt.Errorf("missing required value %q", "version")
	}

	if m.Release.ReleaseType == "" {
		return fmt.Errorf("missing required value %q", "release_type")
	}

	if m.Release.EULASlug == "" {
		return fmt.Errorf("missing required value %q", "eula_slug")
	}

	for i, d := range m.Dependencies {
		dependentReleaseID := d.Release.ID
		if dependentReleaseID == 0 {
			if d.Release.Version == "" || d.Release.Product.Slug == "" {
				return fmt.Errorf(
					"Either ReleaseID or release version and product slug must be provided for dependency[%d]",
					i,
				)
			}
		}
	}

	for i, u := range m.UpgradePaths {
		if u.ID == 0 && u.Version == "" {
			return fmt.Errorf(
				"Either id or version must be provided for upgrade_paths[%d]",
				i,
			)
		}
	}

	return nil
}
<commit_msg>Fix typo in FileGroup metadata. [#133377483]<commit_after>package metadata

import "fmt"

type Metadata struct {
	Release      *Release      `yaml:"release,omitempty"`
	ProductFiles []ProductFile `yaml:"product_files,omitempty"`
	Dependencies []Dependency  `yaml:"dependencies,omitempty"`
	UpgradePaths []UpgradePath `yaml:"upgrade_paths,omitempty"`
	FileGroups   []FileGroup   `yaml:"file_groups,omitempty"`
}

type Release struct {
	Version               string   `yaml:"version"`
	ReleaseType           string   `yaml:"release_type"`
	EULASlug              string   `yaml:"eula_slug"`
	ReleaseDate           string   `yaml:"release_date"`
	Description           string   `yaml:"description"`
	ReleaseNotesURL       string   `yaml:"release_notes_url"`
	Availability          string   `yaml:"availability"`
	UserGroupIDs          []string `yaml:"user_group_ids,omitempty"`
	Controlled            bool     `yaml:"controlled"`
	ECCN                  string   `yaml:"eccn"`
	LicenseException      string   `yaml:"license_exception"`
	EndOfSupportDate      string   `yaml:"end_of_support_date"`
	EndOfGuidanceDate     string   `yaml:"end_of_guidance_date"`
	EndOfAvailabilityDate string   `yaml:"end_of_availability_date"`
}

type ProductFile struct {
	File         string `yaml:"file,omitempty"`
	Description  string `yaml:"description,omitempty"`
	UploadAs     string `yaml:"upload_as,omitempty"`
	AWSObjectKey string `yaml:"aws_object_key,omitempty"`
	FileType     string `yaml:"file_type,omitempty"`
	FileVersion  string `yaml:"file_version,omitempty"`
	MD5          string `yaml:"md5,omitempty"`
	ID           int    `yaml:"id,omitempty"`
}

type FileGroup struct {
	ID           int           `yaml:"id,omitempty"`
	Name         string        `yaml:"name,omitempty"`
	ProductFiles []ProductFile `yaml:"product_files,omitempty"`
}

type Dependency struct {
	Release DependentRelease `yaml:"release,omitempty"`
}

type UpgradePath struct {
	ID      int    `yaml:"id,omitempty"`
	Version string `yaml:"version,omitempty"`
}

type DependentRelease struct {
	ID      int     `yaml:"id,omitempty"`
	Version string  `yaml:"version,omitempty"`
	Product Product `yaml:"product,omitempty"`
}

type Product struct {
	ID   int    `yaml:"id,omitempty"`
	Slug string `yaml:"slug,omitempty"`
	Name string `yaml:"name,omitempty"`
}

func (m Metadata) Validate() error {
	for _, productFile := range m.ProductFiles {
		if productFile.File == "" {
			return fmt.Errorf("empty value for file")
		}
	}

	if m.Release == nil {
		return fmt.Errorf("missing required value %q", "release")
	}

	if m.Release.Version == "" {
		return fmt.Errorf("missing required value %q", "version")
	}

	if m.Release.ReleaseType == "" {
		return fmt.Errorf("missing required value %q", "release_type")
	}

	if m.Release.EULASlug == "" {
		return fmt.Errorf("missing required value %q", "eula_slug")
	}

	for i, d := range m.Dependencies {
		dependentReleaseID := d.Release.ID
		if dependentReleaseID == 0 {
			if d.Release.Version == "" || d.Release.Product.Slug == "" {
				return fmt.Errorf(
					"Either ReleaseID or release version and product slug must be provided for dependency[%d]",
					i,
				)
			}
		}
	}

	for i, u := range m.UpgradePaths {
		if u.ID == 0 && u.Version == "" {
			return fmt.Errorf(
				"Either id or version must be provided for upgrade_paths[%d]",
				i,
			)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package service

import (
	"errors"
	"fmt"
	"os"
	"regexp"
	"sort"
	"strconv"
	"time"

	"github.com/bborbe/backup/dto"
	"github.com/bborbe/backup/host"
	"github.com/bborbe/backup/rootdir"
	"github.com/bborbe/backup/util"
	"github.com/bborbe/log"
)

type BackupService interface {
	GetHost(host string) (dto.Host, error)
	ListHosts() ([]dto.Host, error)
	ListBackups(host dto.Host) ([]dto.Backup, error)
	ListOldBackups(host dto.Host) ([]dto.Backup, error)
	ListKeepBackups(host dto.Host) ([]dto.Backup, error)
	GetLatestBackup(host dto.Host) (dto.Backup, error)
	Cleanup(host dto.Host) error
	Resume(host dto.Host) error
}

type backupService struct {
	rootdir rootdir.Rootdir
}

var logger = log.DefaultLogger

func NewBackupService(rootdirectory string) *backupService {
	s := new(backupService)
	s.rootdir = rootdir.New(rootdirectory)
	return s
}

func (s *backupService) Resume(host dto.Host) error {
	return nil
}

func (s *backupService) ListHosts() ([]dto.Host, error) {
	hosts, err := host.All(s.rootdir)
	if err != nil {
		return nil, err
	}
	return s.createHosts(hosts)
}

func (s *backupService) createHosts(hosts []host.Host) ([]dto.Host, error) {
	result := []dto.Host{}
	for _, host := range hosts {
		dir := host.Path()
		isDir, err := isDir(dir)
		if err != nil {
			logger.Debugf("is dir failed: %v", err)
			return nil, err
		}
		if isDir {
			result = append(result, createHost(host.Name()))
		} else {
			logger.Debugf("createHost for %s failed, is not a directory", host)
		}
	}
	return result, nil
}

func isDir(dir string) (bool, error) {
	file, err := os.Open(dir)
	if err != nil {
		logger.Debugf("open dir %s failed: %v", dir, err)
		return false, nil
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		return false, err
	}
	return fileinfo.IsDir(), nil
}

func createHost(host string) dto.Host {
	h := dto.NewHost()
	h.SetName(host)
	return h
}

func (s *backupService) ListBackups(host dto.Host) ([]dto.Backup, error) {
	if host == nil {
		return nil, errors.New("parameter host missing")
	}
	dir := fmt.Sprintf("%s%c%s", s.rootdir.Path(), os.PathSeparator, host.GetName())
	file, err := os.Open(dir)
	if err != nil {
		logger.Debugf("open dir failed: %v", err)
		return nil, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		logger.Debugf("file stat failed: %v", err)
		return nil, err
	}
	if !fileinfo.IsDir() {
		return nil, fmt.Errorf("dir %s is not a directory", dir)
	}
	dirnames, err := file.Readdirnames(0)
	if err != nil {
		logger.Debugf("read dir names failed: %v", err)
		return nil, err
	}
	var names []string
	for _, name := range dirnames {
		if validBackupName(name) {
			names = append(names, name)
		}
	}
	backups := createBackups(names)
	return backups, nil
}

func validBackupName(name string) bool {
	re := regexp.MustCompile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}")
	return re.MatchString(name)
}

func createBackups(backups []string) []dto.Backup {
	result := make([]dto.Backup, len(backups))
	for i, backup := range backups {
		result[i] = createBackup(backup)
	}
	return result
}

func createBackup(backup string) dto.Backup {
	h := dto.NewBackup()
	h.SetName(backup)
	return h
}

func (s *backupService) GetHost(host string) (dto.Host, error) {
	dir := fmt.Sprintf("%s%c%s", s.rootdir.Path(), os.PathSeparator, host)
	file, err := os.Open(dir)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		return nil, err
	}
	if !fileinfo.IsDir() {
		return nil, fmt.Errorf("dir %s is not a directory", dir)
	}
	h := dto.NewHost()
	h.SetName(host)
	return h, nil
}

func (s *backupService) GetLatestBackup(host dto.Host) (dto.Backup, error) {
	list, err := s.ListBackups(host)
	if err != nil {
		logger.Debugf("list backups failed: %v", err)
		return nil, err
	}
	if len(list) == 0 {
		return nil, nil
	}
	var names []string
	backups := make(map[string]dto.Backup, 0)
	for _, backup := range list {
		backups[backup.GetName()] = backup
		names = append(names, backup.GetName())
	}
	sort.Strings(names)
	return backups[names[len(names)-1]], nil
}

func (s *backupService) ListOldBackups(host dto.Host) ([]dto.Backup, error) {
	backups, err := s.ListBackups(host)
	if err != nil {
		return nil, err
	}
	keep, err := getKeepBackups(backups)
	if err != nil {
		return nil, err
	}
	keepMap := make(map[string]bool)
	for _, b := range keep {
		keepMap[b.GetName()] = true
	}
	var result []dto.Backup
	for _, backup := range backups {
		if !keepMap[backup.GetName()] {
			result = append(result, backup)
		}
	}
	return result, nil
}

func getKeepBackups(backups []dto.Backup) ([]dto.Backup, error) {
	keep := make(map[string]dto.Backup, 0)
	now := time.Now()
	// keep all backups from today
	{
		b, err := getKeepToday(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per day if age <= 7 days
	{
		b, err := getKeepDay(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per week if age <= 28 days
	{
		b, err := getKeepWeek(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per month
	{
		b, err := getKeepMonth(backups)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep latest backup / current
	{
		backup := latestBackup(backups)
		if backup != nil {
			keep[backup.GetName()] = backup
		}
	}

	var result []dto.Backup
	for _, backup := range keep {
		result = append(result, backup)
	}
	return result, nil
}

func getKeepToday(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	var result []dto.Backup
	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		if isToday(t, now) {
			result = append(result, backup)
		}
	}
	return result, nil
}

func isToday(t time.Time, now time.Time) bool {
	return t.Year() == now.Year() && t.Month() == now.Month() && t.Day() == now.Day()
}

func getKeepDay(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var result []dto.Backup

	var lastYear int = -1
	var lastMonth time.Month = -1
	var lastDay int = -1

	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		if ageLessThanDays(t, now, 7) && (t.Year() != lastYear || t.Month() != lastMonth || t.Day() != lastDay) {
			result = append(result, backup)
			lastYear = t.Year()
			lastMonth = t.Month()
			lastDay = t.Day()
		}
	}
	return result, nil
}

func ageLessThanDays(t time.Time, now time.Time, days int64) bool {
	diff := now.Unix() - t.Unix()
	return diff <= 24*60*60*days
}

func getKeepMonth(backups []dto.Backup) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var lastYear int64 = -1
	var lastMonth int64 = -1
	var result []dto.Backup
	for _, backup := range backups {
		name := backup.GetName()
		year, err := strconv.ParseInt(name[0:4], 10, 64)
		if err != nil {
			return nil, err
		}
		month, err := strconv.ParseInt(name[5:7], 10, 64)
		if err != nil {
			return nil, err
		}
		logger.Tracef("year %d month %d", year, month)

		if year != lastYear || month != lastMonth {
			result = append(result, backup)
		}
		lastYear = year
		lastMonth = month
	}
	return result, nil
}

func (s *backupService) Cleanup(hostDto dto.Host) error {
	if hostDto == nil {
		return errors.New("parameter host missing")
	}
	backups, err := s.ListOldBackups(hostDto)
	if err != nil {
		return err
	}
	logger.Debugf("found %d backup to delete for host %s", len(backups), hostDto.GetName())
	for _, backup := range backups {
		dir := fmt.Sprintf("%s%c%s%c%s", s.rootdir.Path(), os.PathSeparator, hostDto.GetName(), os.PathSeparator, backup.GetName())
		logger.Infof("delete %s started", dir)
		os.RemoveAll(dir)
		logger.Infof("delete %s finished", dir)
	}
	return nil
}

func (s *backupService) ListKeepBackups(host dto.Host) ([]dto.Backup, error) {
	backups, err := s.ListBackups(host)
	if err != nil {
		return nil, err
	}
	return getKeepBackups(backups)
}

func getTimeByName(backupName string) (time.Time, error) {
	return time.Parse("2006-01-02T15:04:05", backupName)
}

func getKeepWeek(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var result []dto.Backup
	var lastWeek int = -1
	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		_, week := t.ISOWeek()
		if ageLessThanDays(t, now, 40) && week != lastWeek {
			result = append(result, backup)
			lastWeek = week
		}
	}
	return result, nil
}

func latestBackup(backups []dto.Backup) dto.Backup {
	if backups != nil && len(backups) > 0 {
		sort.Sort(util.BackupByDate(backups))
		return backups[len(backups)-1]
	}
	return nil
}
<commit_msg>remove warning<commit_after>package service

import (
	"errors"
	"fmt"
	"os"
	"regexp"
	"sort"
	"strconv"
	"time"

	"github.com/bborbe/backup/dto"
	"github.com/bborbe/backup/host"
	"github.com/bborbe/backup/rootdir"
	"github.com/bborbe/backup/util"
	"github.com/bborbe/log"
)

type BackupService interface {
	GetHost(host string) (dto.Host, error)
	ListHosts() ([]dto.Host, error)
	ListBackups(host dto.Host) ([]dto.Backup, error)
	ListOldBackups(host dto.Host) ([]dto.Backup, error)
	ListKeepBackups(host dto.Host) ([]dto.Backup, error)
	GetLatestBackup(host dto.Host) (dto.Backup, error)
	Cleanup(host dto.Host) error
	Resume(host dto.Host) error
}

type backupService struct {
	rootdir rootdir.Rootdir
}

var logger = log.DefaultLogger

func NewBackupService(rootdirectory string) *backupService {
	s := new(backupService)
	s.rootdir = rootdir.New(rootdirectory)
	return s
}

func (s *backupService) Resume(host dto.Host) error {
	return nil
}

func (s *backupService) ListHosts() ([]dto.Host, error) {
	hosts, err := host.All(s.rootdir)
	if err != nil {
		return nil, err
	}
	return s.createHosts(hosts)
}

func (s *backupService) createHosts(hosts []host.Host) ([]dto.Host, error) {
	result := []dto.Host{}
	for _, h := range hosts {
		dir := h.Path()
		isDir, err := isDir(dir)
		if err != nil {
			logger.Debugf("is dir failed: %v", err)
			return nil, err
		}
		if isDir {
			result = append(result, createHost(h.Name()))
		} else {
			logger.Debugf("createHost for %s failed, is not a directory", h)
		}
	}
	return result, nil
}

func isDir(dir string) (bool, error) {
	file, err := os.Open(dir)
	if err != nil {
		logger.Debugf("open dir %s failed: %v", dir, err)
		return false, nil
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		return false, err
	}
	return fileinfo.IsDir(), nil
}

func createHost(host string) dto.Host {
	h := dto.NewHost()
	h.SetName(host)
	return h
}

func (s *backupService) ListBackups(h dto.Host) ([]dto.Backup, error) {
	if h == nil {
		return nil, errors.New("parameter host missing")
	}
	dir := fmt.Sprintf("%s%c%s", s.rootdir.Path(), os.PathSeparator, h.GetName())
	file, err := os.Open(dir)
	if err != nil {
		logger.Debugf("open dir failed: %v", err)
		return nil, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		logger.Debugf("file stat failed: %v", err)
		return nil, err
	}
	if !fileinfo.IsDir() {
		return nil, fmt.Errorf("dir %s is not a directory", dir)
	}
	dirnames, err := file.Readdirnames(0)
	if err != nil {
		logger.Debugf("read dir names failed: %v", err)
		return nil, err
	}
	var names []string
	for _, name := range dirnames {
		if validBackupName(name) {
			names = append(names, name)
		}
	}
	backups := createBackups(names)
	return backups, nil
}

func validBackupName(name string) bool {
	re := regexp.MustCompile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}")
	return re.MatchString(name)
}

func createBackups(backups []string) []dto.Backup {
	result := make([]dto.Backup, len(backups))
	for i, backup := range backups {
		result[i] = createBackup(backup)
	}
	return result
}

func createBackup(backup string) dto.Backup {
	h := dto.NewBackup()
	h.SetName(backup)
	return h
}

func (s *backupService) GetHost(host string) (dto.Host, error) {
	dir := fmt.Sprintf("%s%c%s", s.rootdir.Path(), os.PathSeparator, host)
	file, err := os.Open(dir)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	fileinfo, err := file.Stat()
	if err != nil {
		return nil, err
	}
	if !fileinfo.IsDir() {
		return nil, fmt.Errorf("dir %s is not a directory", dir)
	}
	h := dto.NewHost()
	h.SetName(host)
	return h, nil
}

func (s *backupService) GetLatestBackup(host dto.Host) (dto.Backup, error) {
	list, err := s.ListBackups(host)
	if err != nil {
		logger.Debugf("list backups failed: %v", err)
		return nil, err
	}
	if len(list) == 0 {
		return nil, nil
	}
	var names []string
	backups := make(map[string]dto.Backup, 0)
	for _, backup := range list {
		backups[backup.GetName()] = backup
		names = append(names, backup.GetName())
	}
	sort.Strings(names)
	return backups[names[len(names)-1]], nil
}

func (s *backupService) ListOldBackups(host dto.Host) ([]dto.Backup, error) {
	backups, err := s.ListBackups(host)
	if err != nil {
		return nil, err
	}
	keep, err := getKeepBackups(backups)
	if err != nil {
		return nil, err
	}
	keepMap := make(map[string]bool)
	for _, b := range keep {
		keepMap[b.GetName()] = true
	}
	var result []dto.Backup
	for _, backup := range backups {
		if !keepMap[backup.GetName()] {
			result = append(result, backup)
		}
	}
	return result, nil
}

func getKeepBackups(backups []dto.Backup) ([]dto.Backup, error) {
	keep := make(map[string]dto.Backup, 0)
	now := time.Now()
	// keep all backups from today
	{
		b, err := getKeepToday(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per day if age <= 7 days
	{
		b, err := getKeepDay(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per week if age <= 28 days
	{
		b, err := getKeepWeek(backups, now)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep first backup per month
	{
		b, err := getKeepMonth(backups)
		if err != nil {
			return nil, err
		}
		for _, backup := range b {
			keep[backup.GetName()] = backup
		}
	}
	// keep latest backup / current
	{
		backup := latestBackup(backups)
		if backup != nil {
			keep[backup.GetName()] = backup
		}
	}

	var result []dto.Backup
	for _, backup := range keep {
		result = append(result, backup)
	}
	return result, nil
}

func getKeepToday(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	var result []dto.Backup
	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		if isToday(t, now) {
			result = append(result, backup)
		}
	}
	return result, nil
}

func isToday(t time.Time, now time.Time) bool {
	return t.Year() == now.Year() && t.Month() == now.Month() && t.Day() == now.Day()
}

func getKeepDay(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var result []dto.Backup

	var lastYear int = -1
	var lastMonth time.Month = -1
	var lastDay int = -1

	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		if ageLessThanDays(t, now, 7) && (t.Year() != lastYear || t.Month() != lastMonth || t.Day() != lastDay) {
			result = append(result, backup)
			lastYear = t.Year()
			lastMonth = t.Month()
			lastDay = t.Day()
		}
	}
	return result, nil
}

func ageLessThanDays(t time.Time, now time.Time, days int64) bool {
	diff := now.Unix() - t.Unix()
	return diff <= 24*60*60*days
}

func getKeepMonth(backups []dto.Backup) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var lastYear int64 = -1
	var lastMonth int64 = -1
	var result []dto.Backup
	for _, backup := range backups {
		name := backup.GetName()
		year, err := strconv.ParseInt(name[0:4], 10, 64)
		if err != nil {
			return nil, err
		}
		month, err := strconv.ParseInt(name[5:7], 10, 64)
		if err != nil {
			return nil, err
		}
		logger.Tracef("year %d month %d", year, month)

		if year != lastYear || month != lastMonth {
			result = append(result, backup)
		}
		lastYear = year
		lastMonth = month
	}
	return result, nil
}

func (s *backupService) Cleanup(hostDto dto.Host) error {
	if hostDto == nil {
		return errors.New("parameter host missing")
	}
	backups, err := s.ListOldBackups(hostDto)
	if err != nil {
		return err
	}
	logger.Debugf("found %d backup to delete for host %s", len(backups), hostDto.GetName())
	for _, backup := range backups {
		dir := fmt.Sprintf("%s%c%s%c%s", s.rootdir.Path(), os.PathSeparator, hostDto.GetName(), os.PathSeparator, backup.GetName())
		logger.Infof("delete %s started", dir)
		os.RemoveAll(dir)
		logger.Infof("delete %s finished", dir)
	}
	return nil
}

func (s *backupService) ListKeepBackups(host dto.Host) ([]dto.Backup, error) {
	backups, err := s.ListBackups(host)
	if err != nil {
		return nil, err
	}
	return getKeepBackups(backups)
}

func getTimeByName(backupName string) (time.Time, error) {
	return time.Parse("2006-01-02T15:04:05", backupName)
}

func getKeepWeek(backups []dto.Backup, now time.Time) ([]dto.Backup, error) {
	sort.Sort(util.BackupByDate(backups))
	var result []dto.Backup
	var lastWeek int = -1
	for _, backup := range backups {
		t, err := getTimeByName(backup.GetName())
		if err != nil {
			return nil, err
		}
		_, week := t.ISOWeek()
		if ageLessThanDays(t, now, 40) && week != lastWeek {
			result = append(result, backup)
			lastWeek = week
		}
	}
	return result, nil
}

func latestBackup(backups []dto.Backup) dto.Backup {
	if backups != nil && len(backups) > 0 {
		sort.Sort(util.BackupByDate(backups))
		return backups[len(backups)-1]
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package service_test

import (
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/juju/errors"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils/exec"
	"github.com/juju/utils/featureflag"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/feature"
	"github.com/juju/juju/juju/osenv"
	"github.com/juju/juju/service"
	"github.com/juju/juju/service/common"
	"github.com/juju/juju/service/systemd"
	"github.com/juju/juju/service/upstart"
	"github.com/juju/juju/service/windows"
	"github.com/juju/juju/version"
)

var maybeSystemd = service.InitSystemSystemd

func init() {
	if featureflag.Enabled(feature.LegacyUpstart) {
		maybeSystemd = service.InitSystemUpstart
	}
}

const unknownExecutable = "/sbin/unknown/init/system"

type discoveryTest struct {
	os       version.OSType
	series   string
	exec     string
	link     string
	expected string
}

func (dt discoveryTest) version() version.Binary {
	return version.Binary{
		OS:     dt.os,
		Series: dt.series,
	}
}

func (dt discoveryTest) goos() string {
	switch dt.os {
	case version.Windows:
		return "windows"
	default:
		return "non-windows"
	}
}

func (dt discoveryTest) executable(c *gc.C) string {
	if dt.exec != "" {
		return dt.exec
	}

	switch dt.expected {
	case service.InitSystemUpstart:
		return "/sbin/upstart"
	case service.InitSystemSystemd:
		return "/lib/systemd/systemd"
	case service.InitSystemWindows:
		return unknownExecutable
	case "":
		return unknownExecutable
	default:
		c.Errorf("unknown expected init system %q", dt.expected)
		return unknownExecutable
	}
}

func (dt discoveryTest) log(c *gc.C) {
	c.Logf(" - testing {%q, %q, %q, %q}...", dt.os, dt.series, dt.exec, dt.link)
}

func (dt discoveryTest) disableLocalDiscovery(c *gc.C, s *discoverySuite) {
	s.PatchGOOS("<another OS>")
	s.PatchPid1File(c, unknownExecutable, "")
	s.Patched.NotASymlink = unknownExecutable
}

func (dt discoveryTest) disableVersionDiscovery(s *discoverySuite) {
	s.PatchVersion(version.Binary{
		OS: version.Unknown,
	})
}

func (dt discoveryTest) setLocal(c *gc.C, s *discoverySuite) string {
	s.PatchGOOS(dt.goos())
	exec := dt.executable(c)
	if dt.link != "" {
		s.PatchLink(c, exec)
		exec = dt.link
	} else {
		s.Patched.NotASymlink = exec
	}
	verText := "..." + dt.expected + "..."
	return s.PatchPid1File(c, exec, verText)
}

func (dt discoveryTest) setVersion(s *discoverySuite) version.Binary {
	vers := dt.version()
	s.PatchVersion(vers)
	return vers
}

func (dt discoveryTest) checkService(c *gc.C, svc service.Service, err error, name string, conf common.Conf) {
	if dt.expected == "" {
		c.Check(err, jc.Satisfies, errors.IsNotFound)
		return
	}

	// Check the success case.
	if !c.Check(err, jc.ErrorIsNil) {
		return
	}
	switch dt.expected {
	case service.InitSystemUpstart:
		c.Check(svc, gc.FitsTypeOf, &upstart.Service{})
	case service.InitSystemSystemd:
		c.Check(svc, gc.FitsTypeOf, &systemd.Service{})
	case service.InitSystemWindows:
		c.Check(svc, gc.FitsTypeOf, &windows.Service{})
	default:
		c.Errorf("unknown expected init system %q", dt.expected)
		return
	}
	if svc == nil {
		return
	}

	c.Check(svc.Name(), gc.Equals, name)
	c.Check(svc.Conf(), jc.DeepEquals, conf)
}

func (dt discoveryTest) checkInitSystem(c *gc.C, name string, ok bool) {
	if dt.expected == "" {
		if !c.Check(ok, jc.IsFalse) {
			c.Logf("found init system %q", name)
		}
	} else {
		c.Check(ok, jc.IsTrue)
		c.Check(name, gc.Equals, dt.expected)
	}
}

var discoveryTests = []discoveryTest{{
	os:       version.Windows,
	expected: service.InitSystemWindows,
}, {
	os:       version.Ubuntu,
	series:   "oneiric",
	expected: "",
}, {
	os:       version.Ubuntu,
	series:   "precise",
	expected: service.InitSystemUpstart,
}, {
	os:       version.Ubuntu,
	series:   "precise",
	link:     "/sbin/init",
	expected: service.InitSystemUpstart,
}, {
	os:       version.Ubuntu,
	series:   "utopic",
	expected: service.InitSystemUpstart,
}, {
	os:       version.Ubuntu,
	series:   "vivid",
	expected: maybeSystemd,
}, {
	os:       version.Ubuntu,
	series:   "vivid",
	link:     "/sbin/init",
	expected: service.InitSystemSystemd,
}, {
	os:       version.CentOS,
	expected: "",
}, {
	os:       version.Unknown,
	expected: "",
}}

type discoverySuite struct {
	service.BaseSuite

	name string
	conf common.Conf
}

var _ = gc.Suite(&discoverySuite{})

func (s *discoverySuite) SetUpTest(c *gc.C) {
	s.BaseSuite.SetUpTest(c)

	s.name = "a-service"
	s.conf = common.Conf{
		Desc:      "some service",
		ExecStart: "/path/to/some-command",
	}
}

func (s *discoverySuite) unsetLegacyUpstart(c *gc.C) {
	err := os.Setenv(osenv.JujuFeatureFlagEnvKey, "")
	c.Assert(err, jc.ErrorIsNil)
	featureflag.SetFlagsFromEnvironment(osenv.JujuFeatureFlagEnvKey)
}

func (s *discoverySuite) setLegacyUpstart(c *gc.C) {
	err := os.Setenv(osenv.JujuFeatureFlagEnvKey, feature.LegacyUpstart)
	c.Assert(err, jc.ErrorIsNil)
	featureflag.SetFlagsFromEnvironment(osenv.JujuFeatureFlagEnvKey)
}

func (s *discoverySuite) TestDiscoverServiceLocalHost(c *gc.C) {
	var localInitSystem string
	switch runtime.GOOS {
	case "windows":
		localInitSystem = service.InitSystemWindows
	case "linux":
		// TODO(ericsnow) Drop the vivid special-case once systemd is
		// turned on there.
		if version.Current.Series == "vivid" {
			return
		}
		localInitSystem, _ = service.VersionInitSystem(version.Current)
	}
	test := discoveryTest{
		os:       version.Current.OS,
		series:   version.Current.Series,
		expected: localInitSystem,
	}
	test.disableVersionDiscovery(s)

	svc, err := service.DiscoverService(s.name, s.conf)
	c.Assert(err, jc.ErrorIsNil)

	test.checkService(c, svc, err, s.name, s.conf)
}

func (s *discoverySuite) TestDiscoverServiceGeneric(c *gc.C) {
	test := discoveryTest{
		os:       version.Ubuntu,
		series:   "trusty",
		link:     "/sbin/init",
		expected: service.InitSystemUpstart,
	}

	test.setLocal(c, s)
	test.disableVersionDiscovery(s)

	svc, err := service.DiscoverService(s.name, s.conf)

	test.checkService(c, svc, err, s.name, s.conf)
}

func (s *discoverySuite) TestDiscoverServiceLocalOnly(c *gc.C) {
	for _, test := range discoveryTests {
		test.log(c)

		test.setLocal(c, s)
		test.disableVersionDiscovery(s)

		svc, err := service.DiscoverService(s.name, s.conf)

		test.checkService(c, svc, err, s.name, s.conf)
	}
}

func (s *discoverySuite) TestDiscoverServiceVersionFallback(c *gc.C) {
	for _, test := range discoveryTests {
		test.log(c)

		test.disableLocalDiscovery(c, s)
		test.setVersion(s)

		svc, err := service.DiscoverService(s.name, s.conf)

		test.checkService(c, svc, err, s.name, s.conf)
	}
}

func (s *discoverySuite) TestVersionInitSystem(c *gc.C) {
	for _, test := range discoveryTests {
		test.log(c)

		vers := test.setVersion(s)

		initSystem, ok := service.VersionInitSystem(vers)

		test.checkInitSystem(c, initSystem, ok)
	}
}

func (s *discoverySuite) TestVersionInitSystemLegacyUpstart(c *gc.C) {
	s.setLegacyUpstart(c)
	test := discoveryTest{
		os:       version.Ubuntu,
		series:   "vivid",
		expected: service.InitSystemUpstart,
	}
	vers := test.setVersion(s)

	initSystem, ok := service.VersionInitSystem(vers)

	test.checkInitSystem(c, initSystem, ok)
}

func (s *discoverySuite) TestVersionInitSystemNoLegacyUpstart(c *gc.C) {
	s.unsetLegacyUpstart(c)
	test := discoveryTest{
		os:       version.Ubuntu,
		series:   "vivid",
		expected: service.InitSystemSystemd,
	}
	vers := test.setVersion(s)

	initSystem, ok := service.VersionInitSystem(vers)

	test.checkInitSystem(c, initSystem, ok)
}

func (s *discoverySuite) TestDiscoverInitSystemScript(c *gc.C) {
	if runtime.GOOS == "windows" {
		c.Skip("not supported on windows")
	}

	script, filename := s.newDiscoverInitSystemScript(c)
	script += filename
	response, err := exec.RunCommands(exec.RunParams{
		Commands: script,
	})
	c.Assert(err, jc.ErrorIsNil)

	initSystem, err := service.DiscoverInitSystem()
	c.Assert(err, jc.ErrorIsNil)
	c.Check(response.Code, gc.Equals, 0)
	c.Check(string(response.Stdout), gc.Equals, initSystem)
	c.Check(string(response.Stderr), gc.Equals, "")
}

func (s *discoverySuite) newDiscoverInitSystemScript(c *gc.C) (string, string) {
	filename := filepath.Join(c.MkDir(), "discover_init_system.sh")
	commands := service.WriteDiscoverInitSystemScript(filename)
	script := strings.Join(commands, "\n") + "\n"
	return script, filename
}

func (s *discoverySuite) TestNewShellSelectCommand(c *gc.C) {
	if runtime.GOOS == "windows" {
		c.Skip("not supported on windows")
	}

	script, filename := s.newDiscoverInitSystemScript(c)
	handler := func(initSystem string) (string, bool) {
		return "echo -n " + initSystem, true
	}
	script += "init_system=$(" + filename + ")\n"
	script += service.NewShellSelectCommand("init_system", handler)
	response, err := exec.RunCommands(exec.RunParams{
		Commands: script,
	})
	c.Assert(err, jc.ErrorIsNil)

	initSystem, err := service.DiscoverInitSystem()
	c.Assert(err, jc.ErrorIsNil)
	c.Check(response.Code, gc.Equals, 0)
	c.Check(string(response.Stdout), gc.Equals, initSystem)
	c.Check(string(response.Stderr), gc.Equals, "")
}
<commit_msg>Drop vivid special-case in tests.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package service_test

import (
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/juju/errors"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils/exec"
	"github.com/juju/utils/featureflag"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/feature"
	"github.com/juju/juju/juju/osenv"
	"github.com/juju/juju/service"
	"github.com/juju/juju/service/common"
	"github.com/juju/juju/service/systemd"
	"github.com/juju/juju/service/upstart"
	"github.com/juju/juju/service/windows"
	"github.com/juju/juju/version"
)

var maybeSystemd = service.InitSystemSystemd

func init() {
	if featureflag.Enabled(feature.LegacyUpstart) {
		maybeSystemd = service.InitSystemUpstart
	}
}

const unknownExecutable = "/sbin/unknown/init/system"

type discoveryTest struct {
	os       version.OSType
	series   string
	exec     string
	link     string
	expected string
}

func (dt discoveryTest) version() version.Binary {
	return version.Binary{
		OS:     dt.os,
		Series: dt.series,
	}
}

func (dt discoveryTest) goos() string {
	switch dt.os {
	case version.Windows:
		return "windows"
	default:
		return "non-windows"
	}
}

func (dt discoveryTest) executable(c *gc.C) string {
	if dt.exec != "" {
		return dt.exec
	}

	switch dt.expected {
	case service.InitSystemUpstart:
		return "/sbin/upstart"
	case service.InitSystemSystemd:
		return "/lib/systemd/systemd"
	case service.InitSystemWindows:
		return unknownExecutable
	case "":
		return unknownExecutable
	default:
		c.Errorf("unknown expected init system %q", dt.expected)
		return unknownExecutable
	}
}

func (dt discoveryTest) log(c *gc.C) {
	c.Logf(" - testing {%q, %q, %q, %q}...", dt.os, dt.series, dt.exec, dt.link)
}

func (dt discoveryTest) disableLocalDiscovery(c *gc.C, s *discoverySuite) {
	s.PatchGOOS("<another OS>")
	s.PatchPid1File(c, unknownExecutable, "")
	s.Patched.NotASymlink = unknownExecutable
}

func (dt discoveryTest) disableVersionDiscovery(s *discoverySuite) {
	s.PatchVersion(version.Binary{
		OS: version.Unknown,
	})
}

func (dt discoveryTest) setLocal(c *gc.C, s *discoverySuite) string {
	s.PatchGOOS(dt.goos())
	exec := dt.executable(c)
	if dt.link != "" {
		s.PatchLink(c, exec)
		exec = dt.link
	} else {
		s.Patched.NotASymlink = exec
	}
	verText := "..." + dt.expected + "..."
	return s.PatchPid1File(c, exec, verText)
}

func (dt discoveryTest) setVersion(s *discoverySuite) version.Binary {
	vers := dt.version()
	s.PatchVersion(vers)
	return vers
}

func (dt discoveryTest) checkService(c *gc.C, svc service.Service, err error, name string, conf common.Conf) {
	if dt.expected == "" {
		c.Check(err, jc.Satisfies, errors.IsNotFound)
		return
	}

	// Check the success case.
	if !c.Check(err, jc.ErrorIsNil) {
		return
	}
	switch dt.expected {
	case service.InitSystemUpstart:
		c.Check(svc, gc.FitsTypeOf, &upstart.Service{})
	case service.InitSystemSystemd:
		c.Check(svc, gc.FitsTypeOf, &systemd.Service{})
	case service.InitSystemWindows:
		c.Check(svc, gc.FitsTypeOf, &windows.Service{})
	default:
		c.Errorf("unknown expected init system %q", dt.expected)
		return
	}
	if svc == nil {
		return
	}

	c.Check(svc.Name(), gc.Equals, name)
	c.Check(svc.Conf(), jc.DeepEquals, conf)
}

func (dt discoveryTest) checkInitSystem(c *gc.C, name string, ok bool) {
	if dt.expected == "" {
		if !c.Check(ok, jc.IsFalse) {
			c.Logf("found init system %q", name)
		}
	} else {
		c.Check(ok, jc.IsTrue)
		c.Check(name, gc.Equals, dt.expected)
	}
}

var discoveryTests = []discoveryTest{{
	os:       version.Windows,
	expected: service.InitSystemWindows,
}, {
	os:       version.Ubuntu,
	series:   "oneiric",
	expected: "",
}, {
	os:       version.Ubuntu,
	series:   "precise",
	expected: service.InitSystemUpstart,
}, {
	os:       version.Ubuntu,
	series:   "precise",
	link:     "/sbin/init",
	expected: service.InitSystemUpstart,
}, {
	os:       version.Ubuntu,
	series:   "utopic",
	expected: service.InitSystemUpstart,
}, {
	os:       version.Ubuntu,
	series:   "vivid",
	expected: maybeSystemd,
}, {
	os:       version.Ubuntu,
	series:   "vivid",
	link:     "/sbin/init",
	expected: service.InitSystemSystemd,
}, {
	os:       version.CentOS,
	expected: "",
}, {
	os:       version.Unknown,
	expected: "",
}}

type discoverySuite struct {
	service.BaseSuite

	name string
	conf common.Conf
}

var _ = gc.Suite(&discoverySuite{})

func (s *discoverySuite) SetUpTest(c *gc.C) {
	s.BaseSuite.SetUpTest(c)

	s.name = "a-service"
	s.conf = common.Conf{
		Desc:      "some service",
		ExecStart: "/path/to/some-command",
	}
}

func (s *discoverySuite) unsetLegacyUpstart(c *gc.C) {
	err := os.Setenv(osenv.JujuFeatureFlagEnvKey, "")
	c.Assert(err, jc.ErrorIsNil)
	featureflag.SetFlagsFromEnvironment(osenv.JujuFeatureFlagEnvKey)
}

func (s *discoverySuite) setLegacyUpstart(c *gc.C) {
	err := os.Setenv(osenv.JujuFeatureFlagEnvKey, feature.LegacyUpstart)
	c.Assert(err, jc.ErrorIsNil)
	featureflag.SetFlagsFromEnvironment(osenv.JujuFeatureFlagEnvKey)
}

func (s *discoverySuite) TestDiscoverServiceLocalHost(c *gc.C) {
	var localInitSystem string
	switch runtime.GOOS {
	case "windows":
		localInitSystem = service.InitSystemWindows
	case "linux":
		localInitSystem, _ = service.VersionInitSystem(version.Current)
	}
	test := discoveryTest{
		os:       version.Current.OS,
		series:   version.Current.Series,
		expected: localInitSystem,
	}
	test.disableVersionDiscovery(s)

	svc, err := service.DiscoverService(s.name, s.conf)
	c.Assert(err, jc.ErrorIsNil)

	test.checkService(c, svc, err, s.name, s.conf)
}

func (s *discoverySuite) TestDiscoverServiceGeneric(c *gc.C) {
	test := discoveryTest{
		os:       version.Ubuntu,
		series:   "trusty",
		link:     "/sbin/init",
		expected: service.InitSystemUpstart,
	}

	test.setLocal(c, s)
	test.disableVersionDiscovery(s)

	svc, err := service.DiscoverService(s.name, s.conf)

	test.checkService(c, svc, err, s.name, s.conf)
}

func (s *discoverySuite) TestDiscoverServiceLocalOnly(c *gc.C) {
	for _, test := range discoveryTests {
		test.log(c)

		test.setLocal(c, s)
		test.disableVersionDiscovery(s)

		svc, err := service.DiscoverService(s.name, s.conf)

		test.checkService(c, svc, err, s.name, s.conf)
	}
}

func (s *discoverySuite) TestDiscoverServiceVersionFallback(c *gc.C) {
	for _, test := range discoveryTests {
		test.log(c)

		test.disableLocalDiscovery(c, s)
		test.setVersion(s)

		svc, err := service.DiscoverService(s.name, s.conf)

		test.checkService(c, svc, err, s.name, s.conf)
	}
}

func (s *discoverySuite) TestVersionInitSystem(c *gc.C) {
	for _, test := range discoveryTests {
		test.log(c)

		vers := test.setVersion(s)

		initSystem, ok := service.VersionInitSystem(vers)

		test.checkInitSystem(c, initSystem, ok)
	}
}

func (s *discoverySuite) TestVersionInitSystemLegacyUpstart(c *gc.C) {
	s.setLegacyUpstart(c)
	test := discoveryTest{
		os:       version.Ubuntu,
		series:   "vivid",
		expected: service.InitSystemUpstart,
	}
	vers := test.setVersion(s)

	initSystem, ok := service.VersionInitSystem(vers)

	test.checkInitSystem(c, initSystem, ok)
}

func (s *discoverySuite) TestVersionInitSystemNoLegacyUpstart(c *gc.C) {
	s.unsetLegacyUpstart(c)
	test := discoveryTest{
		os:       version.Ubuntu,
		series:   "vivid",
		expected: service.InitSystemSystemd,
	}
	vers := test.setVersion(s)

	initSystem, ok := service.VersionInitSystem(vers)

	test.checkInitSystem(c, initSystem, ok)
}

func (s *discoverySuite) TestDiscoverInitSystemScript(c *gc.C) {
	if runtime.GOOS == "windows" {
		c.Skip("not supported on windows")
	}

	script, filename := s.newDiscoverInitSystemScript(c)
	script += filename
	response, err := exec.RunCommands(exec.RunParams{
		Commands: script,
	})
	c.Assert(err, jc.ErrorIsNil)

	initSystem, err := service.DiscoverInitSystem()
	c.Assert(err, jc.ErrorIsNil)
	c.Check(response.Code, gc.Equals, 0)
	c.Check(string(response.Stdout), gc.Equals, initSystem)
	c.Check(string(response.Stderr), gc.Equals, "")
}

func (s *discoverySuite) newDiscoverInitSystemScript(c *gc.C) (string, string) {
	filename := filepath.Join(c.MkDir(), "discover_init_system.sh")
	commands := service.WriteDiscoverInitSystemScript(filename)
	script := strings.Join(commands, "\n") + "\n"
	return script, filename
}

func (s *discoverySuite) TestNewShellSelectCommand(c *gc.C) {
	if runtime.GOOS == "windows" {
		c.Skip("not supported on windows")
	}

	script, filename := s.newDiscoverInitSystemScript(c)
	handler := func(initSystem string) (string, bool) {
		return "echo -n " + initSystem, true
	}
	script += "init_system=$(" + filename + ")\n"
	script += service.NewShellSelectCommand("init_system", handler)
	response, err := exec.RunCommands(exec.RunParams{
		Commands: script,
	})
	c.Assert(err, jc.ErrorIsNil)

	initSystem, err := service.DiscoverInitSystem()
	c.Assert(err, jc.ErrorIsNil)
	c.Check(response.Code, gc.Equals, 0)
	c.Check(string(response.Stdout), gc.Equals, initSystem)
	c.Check(string(response.Stderr), gc.Equals, "")
}
<|endoftext|>


<commit_before>package httpd

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/pprof"
	"os"
	"strings"

	"github.com/bmizerany/pat"
	"github.com/gocql/gocql"
	"github.com/mattbaird/elastigo/lib"
	"github.com/rcrowley/go-metrics"
	"github.com/vatcinc/bio/models"
	"github.com/vatcinc/bio/schema"
	"gopkg.in/mgo.v2"
	// "github.com/gorilla/websocket"
)

const (
	// DefaultChunkSize specifies the amount of data mappers will read
	// up to, before sending results back to the engine. This is the
	// default size in the number of values returned in a raw query.
	//
	// Could be many more bytes depending on fields returned.
	DefaultChunkSize = 10000
)

// TODO: Standard response headers (see: HeaderHandler)
// TODO: Compression (see: CompressionHeaderHandler)

// TODO: Check HTTP response codes: 400, 401, 403, 409.

type route struct {
	name        string
	method      string
	pattern     string
	gzipped     bool
	log         bool
	handlerFunc interface{}
}

// Handler represents an HTTP handler for the InfluxDB server.
type Handler struct {
	mux                   *pat.PatternServeMux
	requireAuthentication bool
	Version               string

	Logger         *log.Logger
	loggingEnabled bool // Log every HTTP access.
	WriteTrace     bool // Detailed logging of write path
	Mongo          *mgo.Database
	Elasticsearch  *elastigo.Conn
	Cassandra      *gocql.ClusterConfig
	statMap        metrics.Registry
	//statMap        *expvar.Map
}

// NewHandler returns a new instance of handler with routes.
func NewHandler(requireAuthentication, loggingEnabled, writeTrace bool) *Handler {
	// c := metrics.NewCounter()
	// metrics.Register(statRequest, c)
	// c1 := metrics.NewCounter()
	// metrics.Register(statAuthFail, c1)
	h := &Handler{
		mux: pat.New(),
		requireAuthentication: requireAuthentication,
		Logger:                log.New(os.Stderr, "[http] ", log.LstdFlags),
		loggingEnabled:        loggingEnabled,
		WriteTrace:            writeTrace,
		statMap:               metrics.DefaultRegistry,
	}

	h.SetRoutes([]route{
		route{
			"index_bios",
			"GET", "/profiles/api/bios", true, true, h.indexBiographies,
		},
		route{
			"create_biography",
			"POST", "/profiles/api/bios", true, true, h.createBiography,
		},
		route{
			"create_biography_opts",
			"OPTIONS", "/profiles/api/bios", true, true, h.serveOptions,
		},
		route{
			"update_biography_opts",
			"OPTIONS", "/profiles/api/bio", true, true, h.serveOptions,
		},
		route{
			"show_bio",
			"GET", "/profiles/api/bio", true, true, h.showBiography,
		},
		route{
			"show_bio",
			"DELETE", "/profiles/api/bio", true, true, h.deleteBiography,
		},
		route{
			"update_bio",
			"PUT", "/profiles/api/bio", true, true, h.updateBiography,
		},
		route{
			"manifest_appcache",
			"GET", "/profiles/biography/:manifest.appcache", true, true, h.serveAppCache,
		},
		route{
			"manifest_json",
			"GET", "/profiles/biography/:manifest.json", true, true, h.serveAppJson,
		},
		route{
			"serviceworker_js",
			"GET", "/profiles/biography/serviceworker/:version.js", true, true, h.serveServiceWorker,
		},
		route{
			"js",
			"GET", "/profiles/biography/:version.js", true, true, h.serveBundleJs,
		},
		route{
			"css",
			"GET", "/profiles/biography/:version.css", true, true, h.serveMainCss,
		},
		route{
			"biography",
			"GET", "/profiles/biography", true, true, h.serveBiographyResult,
		},
		route{
			"biography",
			"GET", "/profiles/biography/:name", true, true, h.serveBiographyResult,
		},
		route{
			"biography",
			"GET", "/profiles/biography/:name/:action", true, true, h.serveBiographyResult,
		},
		route{
			"healthcheck",
			"GET", "/healthcheck", true, true, h.serveHealthcheck,
		},
		route{
			"status", // Query serving route.
			"GET", "/status", true, true, h.serveHealthcheck,
		},
	})

	return h
}

func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	//h.statMap.Add(statRequest, 1)

	counter := metrics.GetOrRegisterCounter(statRequest, h.statMap)
	counter.Inc(1)

	meter := metrics.GetOrRegisterMeter(statRequestNew, h.statMap)
	meter.Mark(1)

	// FIXME(benbjohnson): Add pprof enabled flag.
	if strings.HasPrefix(r.URL.Path, "/debug/pprof") {
		switch r.URL.Path {
		case "/debug/pprof/cmdline":
			pprof.Cmdline(w, r)
		case "/debug/pprof/profile":
			pprof.Profile(w, r)
		case "/debug/pprof/symbol":
			pprof.Symbol(w, r)
		default:
			pprof.Index(w, r)
		}
	} else {
		h.mux.ServeHTTP(w, r)
		return
	}

}

// SetRoutes sets the provided routes on the handler.
func (h *Handler) SetRoutes(routes []route) {
	for _, r := range routes {
		var handler http.Handler

		// If it's a handler func that requires a domain, wrap it in a domain :lol:
		if hf, ok := r.handlerFunc.(func(http.ResponseWriter, *http.Request, *models.Domain)); ok {
			handler = materializeDomain(hf, h)
		}

		// If it's a handler func that requires authorization, wrap it in authorization
		if hf, ok := r.handlerFunc.(func(http.ResponseWriter, *http.Request, *bio.Users)); ok {
			handler = authenticate(hf, h, h.requireAuthentication)
		}
		// This is a normal handler signature and does not require authorization
		if hf, ok := r.handlerFunc.(func(http.ResponseWriter, *http.Request)); ok {
			handler = http.HandlerFunc(hf)
		}

		if r.gzipped {
			handler = gzipFilter(handler)
		}
		handler = versionHeader(handler, h)
		handler = cors(handler)
		handler = requestID(handler)

		if h.loggingEnabled && r.log {
			handler = logging(handler, r.name, h.Logger)
		}

		handler = recovery(handler, r.name, h.Logger) // make sure recovery is always last

		h.mux.Add(r.method, r.pattern, handler)
	}
}

func (h *Handler) serveOptions(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusNoContent)
}
func (h *Handler) serveServiceWorker(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("content-type", "application/javascript")
	w.WriteHeader(http.StatusOK)
	dat, err := ioutil.ReadFile("build/serviceworker.js")
	if err != nil {
		httpError(w, "No id given", false, http.StatusNotFound)
		return
	}
	w.Write(dat)
}
func (h *Handler) serveBundleJs(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("content-type", "application/javascript")
	w.WriteHeader(http.StatusOK)
	dat, err := ioutil.ReadFile("build/assets/javascript/bundle.js")
	if err != nil {
		httpError(w, err.Error(), false, http.StatusNotFound)
		return
	}
	w.Write(dat)
}
func (h *Handler) serveMainCss(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("content-type", "text/css")
	w.WriteHeader(http.StatusOK)
	dat, err := ioutil.ReadFile("build/assets/css/main.css")
	if err != nil {
		httpError(w, err.Error(), false, http.StatusNotFound)
		return
	}
	w.Write(dat)
}
func (h *Handler) serveAppCache(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("content-type", "text/cache-manifest")
	w.WriteHeader(http.StatusOK)
	w.Write([]byte(`CACHE MANIFEST
# ef14122730dc075fa895

/profiles/biography/manifest.json
/profiles/biography/serviceworker.js
/profiles/biography/bundle.js
/profiles/biography/main.css
/profiles/biography

NETWORK:
*
  `))
}
func (h *Handler) serveAppJson(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("content-type", "application/json")
	w.WriteHeader(http.StatusOK)
	w.Write([]byte(`{
    "name": "SITREP Profiles",
    "start_url": "/profiles/biography",
    "display": "standalone",
    "orientation": "portrait",
    "background_color": "#FFFFFF"
  }`))
}

// RootAPIResult describes the API Result of the Root Document
type RootAPIResult struct {
	AppName      string                   `json:"app"`
	Version      string                   `json:"version"`
	AllowedPaths []map[string]interface{} `json:"paths"`
}

func (h *Handler) serveHealthcheck(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("content-type", "application/json")

	res := map[string]string{"status": "ok"}
	w.Write(MarshalJSON(res, false))
}

// MarshalJSON will marshal v to JSON. Pretty prints if pretty is true.
func MarshalJSON(v interface{}, pretty bool) []byte {
	var b []byte
	var err error
	if pretty {
		b, err = json.MarshalIndent(v, "", "    ")
	} else {
		b, err = json.Marshal(v)
	}

	if err != nil {
		return []byte(err.Error())
	}
	return b
}

// Filters and filter helpers

// Response represents a list of statement results.
type Response struct {
	Results []interface{} `json:"results"`
	Err     string        `json:"error"`
}

// MarshalJSON encodes a Response struct into JSON.
func (r Response) MarshalJSON() ([]byte, error) {
	// Define a struct that outputs "error" as a string.
	var o struct {
		Results []interface{} `json:"results,omitempty"`
		Err     string        `json:"error,omitempty"`
	}

	// Copy fields to output struct.
	o.Results = r.Results
	// if r.Err != nil {
	// 	o.Err = r.Err.Error()
	// }

	return json.Marshal(&o)
}

// UnmarshalJSON decodes the data into the Response struct
func (r *Response) UnmarshalJSON(b []byte) error {
	var o struct {
		Results []interface{} `json:"results,omitempty"`
		Err     string        `json:"error,omitempty"`
	}

	err := json.Unmarshal(b, &o)
	if err != nil {
		return err
	}
	r.Results = o.Results
	if o.Err != "" {
		r.Err = o.Err
	}
	return nil
}

// Error returns the first error from any statement.
// Returns nil if no errors occurred on any statements.
func (r *Response) Error() error {
	// if r.Err != nil {
	// 	return r.Err
	// }
	// for _, rr := range r.Results {
	// 	// if rr.Err != nil {
	// 	// 	return rr.Err
	// 	// }
	// }
	return nil
}

// Helpers

// MarshalEmber wraps a document in an ember package
func MarshalEmber(w http.ResponseWriter, id gocql.UUID, data interface{}, typeString string, pretty bool) {
	w.Header().Add("content-type", "application/json")
	ember := &models.EmberData{
		Data: models.EmberDataObj{
			Type:       typeString,
			ID:         id.String(),
			Attributes: MarshalJSON(data, pretty),
		},
	}
	w.Write(MarshalJSON(ember, pretty))
}

// MarshalMultiEmber wraps a document in an ember package
func MarshalMultiEmber(w http.ResponseWriter, data *models.EmberMultiData, pretty bool) {
	w.Header().Add("content-type", "application/json")
	w.Write(MarshalJSON(data, pretty))
}

// EmberDataObj defines a inner-response object that follows embers standards
type EmberDataObj struct {
	Type       string          `json:"type"`
	ID         string          `json:"id,omitempty"`
	Attributes json.RawMessage `json:"attributes"`
}

// EmberData is the wrapper for EmberDataObj.
type EmberData struct {
	Data   EmberDataObj `json:"data"`
	Errors interface{}  `json:"errors,omitempty"`
	Meta   interface{}  `json:"meta,omitempty"`
}

// EmberMultiData defines multiple occurrencies of EmberData's Data
type EmberMultiData struct {
	Data   interface{} `json:"data"`
	Errors interface{} `json:"errors,omitempty"`
	Meta   interface{} `json:"meta,omitempty"`
}
<commit_msg>updated application<commit_after>package httpd

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/pprof"
	"os"
	"strings"

	"github.com/bmizerany/pat"
	"github.com/gocql/gocql"
	"github.com/mattbaird/elastigo/lib"
	"github.com/rcrowley/go-metrics"
	"github.com/vatcinc/bio/models"
	"github.com/vatcinc/bio/schema"
	"gopkg.in/mgo.v2"
	// "github.com/gorilla/websocket"
)

const (
	// DefaultChunkSize specifies the amount of data mappers will read
	// up to, before sending results back to the engine. This is the
	// default size in the number of values returned in a raw query.
	//
	// Could be many more bytes depending on fields returned.
	DefaultChunkSize = 10000
)

// TODO: Standard response headers (see: HeaderHandler)
// TODO: Compression (see: CompressionHeaderHandler)

// TODO: Check HTTP response codes: 400, 401, 403, 409.

type route struct {
	name        string
	method      string
	pattern     string
	gzipped     bool
	log         bool
	handlerFunc interface{}
}

// Handler represents an HTTP handler for the InfluxDB server.
type Handler struct {
	mux                   *pat.PatternServeMux
	requireAuthentication bool
	Version               string

	Logger         *log.Logger
	loggingEnabled bool // Log every HTTP access.
	WriteTrace     bool // Detailed logging of write path
	Mongo          *mgo.Database
	Elasticsearch  *elastigo.Conn
	Cassandra      *gocql.ClusterConfig
	statMap        metrics.Registry
	//statMap        *expvar.Map
}

// NewHandler returns a new instance of handler with routes.
func NewHandler(requireAuthentication, loggingEnabled, writeTrace bool) *Handler {
	// c := metrics.NewCounter()
	// metrics.Register(statRequest, c)
	// c1 := metrics.NewCounter()
	// metrics.Register(statAuthFail, c1)
	h := &Handler{
		mux: pat.New(),
		requireAuthentication: requireAuthentication,
		Logger:                log.New(os.Stderr, "[http] ", log.LstdFlags),
		loggingEnabled:        loggingEnabled,
		WriteTrace:            writeTrace,
		statMap:               metrics.DefaultRegistry,
	}

	h.SetRoutes([]route{
		route{
			"index_bios",
			"GET", "/profiles/api/bios", true, true, h.indexBiographies,
		},
		route{
			"create_biography",
			"POST", "/profiles/api/bios", true, true, h.createBiography,
		},
		route{
			"create_biography_opts",
			"OPTIONS", "/profiles/api/bios", true, true, h.serveOptions,
		},
		route{
			"update_biography_opts",
			"OPTIONS", "/profiles/api/bio", true, true, h.serveOptions,
		},
		route{
			"show_bio",
			"GET", "/profiles/api/bio", true, true, h.showBiography,
		},
		route{
			"show_bio",
			"DELETE", "/profiles/api/bio", true, true, h.deleteBiography,
		},
		route{
			"update_bio",
			"PUT", "/profiles/api/bio", true, true, h.updateBiography,
		},
		route{
			"manifest_appcache",
			"GET", "/profiles/biography/:manifest.appcache", true, true, h.serveAppCache,
		},
		route{
			"manifest_json",
			"GET", "/profiles/biography/:manifest.json", true, true, h.serveAppJson,
		},
		route{
			"serviceworker_js",
			"GET", "/profiles/biography/serviceworker/:version.js", true, true, h.serveServiceWorker,
		},
		route{
			"js",
			"GET", "/profiles/biography/:version.js", true, true, h.serveBundleJs,
		},
		route{
			"css",
			"GET", "/profiles/biography/:version.css", true, true, h.serveMainCss,
		},
		route{
			"biography",
			"GET", "/profiles/biography", true, true, h.serveBiographyResult,
		},
		route{
			"biography",
			"GET", "/profiles/biography/:name", true, true, h.serveBiographyResult,
		},
		route{
			"biography",
			"GET", "/profiles/biography/:name/:action", true, true, h.serveBiographyResult,
		},
		route{
			"healthcheck",
			"GET", "/healthcheck", true, true, h.serveHealthcheck,
		},
		route{
			"status", // Query serving route.
			"GET", "/status", true, true, h.serveHealthcheck,
		},
	})

	return h
}

func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	//h.statMap.Add(statRequest, 1)

	counter := metrics.GetOrRegisterCounter(statRequest, h.statMap)
	counter.Inc(1)

	meter := metrics.GetOrRegisterMeter(statRequestNew, h.statMap)
	meter.Mark(1)

	// FIXME(benbjohnson): Add pprof enabled flag.
	if strings.HasPrefix(r.URL.Path, "/debug/pprof") {
		switch r.URL.Path {
		case "/debug/pprof/cmdline":
			pprof.Cmdline(w, r)
		case "/debug/pprof/profile":
			pprof.Profile(w, r)
		case "/debug/pprof/symbol":
			pprof.Symbol(w, r)
		default:
			pprof.Index(w, r)
		}
	} else {
		h.mux.ServeHTTP(w, r)
		return
	}

}

// SetRoutes sets the provided routes on the handler.
func (h *Handler) SetRoutes(routes []route) {
	for _, r := range routes {
		var handler http.Handler

		// If it's a handler func that requires a domain, wrap it in a domain :lol:
		if hf, ok := r.handlerFunc.(func(http.ResponseWriter, *http.Request, *models.Domain)); ok {
			handler = materializeDomain(hf, h)
		}

		// If it's a handler func that requires authorization, wrap it in authorization
		if hf, ok := r.handlerFunc.(func(http.ResponseWriter, *http.Request, *bio.Users)); ok {
			handler = authenticate(hf, h, h.requireAuthentication)
		}
		// This is a normal handler signature and does not require authorization
		if hf, ok := r.handlerFunc.(func(http.ResponseWriter, *http.Request)); ok {
			handler = http.HandlerFunc(hf)
		}

		if r.gzipped {
			handler = gzipFilter(handler)
		}
		handler = versionHeader(handler, h)
		handler = cors(handler)
		handler = requestID(handler)

		if h.loggingEnabled && r.log {
			handler = logging(handler, r.name, h.Logger)
		}

		handler = recovery(handler, r.name, h.Logger) // make sure recovery is always last

		h.mux.Add(r.method, r.pattern, handler)
	}
}

func (h *Handler) serveOptions(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusNoContent)
}
func (h *Handler) serveServiceWorker(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("content-type", "application/javascript")
	w.WriteHeader(http.StatusOK)
	dat, err := ioutil.ReadFile("build/serviceworker.js")
	if err != nil {
		httpError(w, "No id given", false, http.StatusNotFound)
		return
	}
	w.Write(dat)
}
func (h *Handler) serveBundleJs(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("content-type", "application/javascript")
	w.WriteHeader(http.StatusOK)
	dat, err := ioutil.ReadFile("build/assets/javascript/bundle.js")
	if err != nil {
		httpError(w, err.Error(), false, http.StatusNotFound)
		return
	}
	w.Write(dat)
}
func (h *Handler) serveMainCss(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("content-type", "text/css")
	w.WriteHeader(http.StatusOK)
	dat, err := ioutil.ReadFile("build/assets/css/main.css")
	if err != nil {
		httpError(w, err.Error(), false, http.StatusNotFound)
		return
	}
	w.Write(dat)
}
func (h *Handler) serveAppCache(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("content-type", "text/cache-manifest")
	w.WriteHeader(http.StatusOK)
	w.Write([]byte(`CACHE MANIFEST
# `+h.Version+`

/profiles/biography/` + h.Version + `.json
/profiles/biography/serviceworker/` + h.Version + `.js
/profiles/biography/` + h.Version + `.js
/profiles/biography/` + h.Version + `.css
/profiles/biography

NETWORK:
*
  `))
}
func (h *Handler) serveAppJson(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("content-type", "application/json")
	w.WriteHeader(http.StatusOK)
	w.Write([]byte(`{
    "name": "SITREP Profiles",
    "start_url": "/profiles/biography",
    "display": "standalone",
    "orientation": "portrait",
    "background_color": "#FFFFFF"
  }`))
}

// RootAPIResult describes the API Result of the Root Document
type RootAPIResult struct {
	AppName      string                   `json:"app"`
	Version      string                   `json:"version"`
	AllowedPaths []map[string]interface{} `json:"paths"`
}

func (h *Handler) serveHealthcheck(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("content-type", "application/json")

	res := map[string]string{"status": "ok"}
	w.Write(MarshalJSON(res, false))
}

// MarshalJSON will marshal v to JSON. Pretty prints if pretty is true.
func MarshalJSON(v interface{}, pretty bool) []byte {
	var b []byte
	var err error
	if pretty {
		b, err = json.MarshalIndent(v, "", "    ")
	} else {
		b, err = json.Marshal(v)
	}

	if err != nil {
		return []byte(err.Error())
	}
	return b
}

// Filters and filter helpers

// Response represents a list of statement results.
type Response struct {
	Results []interface{} `json:"results"`
	Err     string        `json:"error"`
}

// MarshalJSON encodes a Response struct into JSON.
func (r Response) MarshalJSON() ([]byte, error) {
	// Define a struct that outputs "error" as a string.
	var o struct {
		Results []interface{} `json:"results,omitempty"`
		Err     string        `json:"error,omitempty"`
	}

	// Copy fields to output struct.
	o.Results = r.Results
	// if r.Err != nil {
	// 	o.Err = r.Err.Error()
	// }

	return json.Marshal(&o)
}

// UnmarshalJSON decodes the data into the Response struct
func (r *Response) UnmarshalJSON(b []byte) error {
	var o struct {
		Results []interface{} `json:"results,omitempty"`
		Err     string        `json:"error,omitempty"`
	}

	err := json.Unmarshal(b, &o)
	if err != nil {
		return err
	}
	r.Results = o.Results
	if o.Err != "" {
		r.Err = o.Err
	}
	return nil
}

// Error returns the first error from any statement.
// Returns nil if no errors occurred on any statements.
func (r *Response) Error() error {
	// if r.Err != nil {
	// 	return r.Err
	// }
	// for _, rr := range r.Results {
	// 	// if rr.Err != nil {
	// 	// 	return rr.Err
	// 	// }
	// }
	return nil
}

// Helpers

// MarshalEmber wraps a document in an ember package
func MarshalEmber(w http.ResponseWriter, id gocql.UUID, data interface{}, typeString string, pretty bool) {
	w.Header().Add("content-type", "application/json")
	ember := &models.EmberData{
		Data: models.EmberDataObj{
			Type:       typeString,
			ID:         id.String(),
			Attributes: MarshalJSON(data, pretty),
		},
	}
	w.Write(MarshalJSON(ember, pretty))
}

// MarshalMultiEmber wraps a document in an ember package
func MarshalMultiEmber(w http.ResponseWriter, data *models.EmberMultiData, pretty bool) {
	w.Header().Add("content-type", "application/json")
	w.Write(MarshalJSON(data, pretty))
}

// EmberDataObj defines a inner-response object that follows embers standards
type EmberDataObj struct {
	Type       string          `json:"type"`
	ID         string          `json:"id,omitempty"`
	Attributes json.RawMessage `json:"attributes"`
}

// EmberData is the wrapper for EmberDataObj.
type EmberData struct {
	Data   EmberDataObj `json:"data"`
	Errors interface{}  `json:"errors,omitempty"`
	Meta   interface{}  `json:"meta,omitempty"`
}

// EmberMultiData defines multiple occurrencies of EmberData's Data
type EmberMultiData struct {
	Data   interface{} `json:"data"`
	Errors interface{} `json:"errors,omitempty"`
	Meta   interface{} `json:"meta,omitempty"`
}
<|endoftext|>


<commit_before>package scipipe

import (
	"time"
)

// Sink is a simple component that just receives InformationPacket on its In-port
// without doing anything with them
type Sink struct {
	Process
	name    string
	inPorts []*FilePort
}

// Instantiate a Sink component
func NewSink(name string) (s *Sink) {
	return &Sink{
		name:    name,
		inPorts: []*FilePort{},
	}
}

func (p *Sink) IsConnected() bool {
	if len(p.inPorts) > 0 {
		return true
	} else {
		return false
	}
}

func (p *Sink) Connect(outPort *FilePort) {
	newInPort := NewFilePort()
	newInPort.Connect(outPort)
	p.inPorts = append(p.inPorts, newInPort)
}

func (p *Sink) Name() string {
	return p.name
}

// Execute the Sink component
func (p *Sink) Run() {
	ok := true
	var ft *InformationPacket
	for len(p.inPorts) > 0 {
	loop:
		for i, inp := range p.inPorts {
			select {
			case ft, ok = <-inp.Chan:
				if !ok {
					Debug.Println(p.name+": Channel closed, so deleting, and restarting loop", ft.GetPath())
					p.deleteInPortAtKey(i)
					break loop
				}
				Debug.Println(p.name+": Received file in sink: ", ft.GetPath())
			default:
				Debug.Printf(p.name+": No receive on inport %d, so continuing ...\n", i)
				time.Sleep(100 * time.Millisecond)
			}
		}
	}
}
<commit_msg>Fixing bug in sink: Can't access nonexistent IP<commit_after>package scipipe

import (
	"time"
)

// Sink is a simple component that just receives InformationPacket on its In-port
// without doing anything with them
type Sink struct {
	Process
	name    string
	inPorts []*FilePort
}

// Instantiate a Sink component
func NewSink(name string) (s *Sink) {
	return &Sink{
		name:    name,
		inPorts: []*FilePort{},
	}
}

func (p *Sink) IsConnected() bool {
	if len(p.inPorts) > 0 {
		return true
	} else {
		return false
	}
}

func (p *Sink) Connect(outPort *FilePort) {
	newInPort := NewFilePort()
	newInPort.Connect(outPort)
	p.inPorts = append(p.inPorts, newInPort)
}

func (p *Sink) Name() string {
	return p.name
}

// Execute the Sink component
func (p *Sink) Run() {
	ok := true
	var ft *InformationPacket
	for len(p.inPorts) > 0 {
	loop:
		for i, inp := range p.inPorts {
			select {
			case ft, ok = <-inp.Chan:
				if !ok {
					Debug.Println(p.name + ": Channel closed, so deleting, and restarting loop")
					p.deleteInPortAtKey(i)
					break loop
				}
				Debug.Println(p.name+": Received file in sink: ", ft.GetPath())
			default:
				Debug.Printf(p.name+": No receive on inport %d, so continuing ...\n", i)
				time.Sleep(100 * time.Millisecond)
			}
		}
	}
}
<|endoftext|>


<commit_before>package middleware

import (
	"fmt"
	"io"
	"net"
	"os"
	"strconv"
	"time"

	"github.com/labstack/echo"
	"github.com/labstack/gommon/color"
	isatty "github.com/mattn/go-isatty"
	"github.com/valyala/fasttemplate"
)

type (
	// LoggerConfig defines the config for logger middleware.
	LoggerConfig struct {
		// Format is the log format which can be constructed using the following tags:
		//
		// - time_rfc3339
		// - remote_ip
		// - uri
		// - method
		// - path
		// - status
		// - response_time
		// - response_size
		//
		// Example "${remote_id} ${status}"
		//
		// Optional with default value as `DefaultLoggerConfig.Format`.
		Format string

		// Output is the writer where logs are written.
		// Optional with default value as os.Stdout.
		Output io.Writer

		template *fasttemplate.Template
		color    *color.Color
	}
)

var (
	// DefaultLoggerConfig is the default logger middleware config.
	DefaultLoggerConfig = LoggerConfig{
		Format: "time=${time_rfc3339}, remote_ip=${remote_ip}, method=${method}, " +
			"uri=${uri}, status=${status}, took=${response_time}, sent=${response_size} bytes\n",
		color:  color.New(),
		Output: os.Stdout,
	}
)

// Logger returns a middleware that logs HTTP requests.
func Logger() echo.MiddlewareFunc {
	return LoggerWithConfig(DefaultLoggerConfig)
}

// LoggerWithConfig returns a logger middleware from config.
// See `Logger()`.
func LoggerWithConfig(config LoggerConfig) echo.MiddlewareFunc {
	// Defaults
	if config.Format == "" {
		config.Format = DefaultLoggerConfig.Format
	}
	if config.Output == nil {
		config.Output = DefaultLoggerConfig.Output
	}

	config.template = fasttemplate.New(config.Format, "${", "}")
	config.color = color.New()
	if w, ok := config.Output.(*os.File); ok && !isatty.IsTerminal(w.Fd()) {
		config.color.Disable()
	}

	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c echo.Context) (err error) {
			rq := c.Request()
			rs := c.Response()
			start := time.Now()
			if err = next(c); err != nil {
				c.Error(err)
			}
			stop := time.Now()

			_, err = config.template.ExecuteFunc(config.Output, func(w io.Writer, tag string) (int, error) {
				switch tag {
				case "time_rfc3339":
					return w.Write([]byte(time.Now().Format(time.RFC3339)))
				case "remote_ip":
					ra := rq.RemoteAddress()
					if ip := rq.Header().Get(echo.HeaderXRealIP); ip != "" {
						ra = ip
					} else if ip = rq.Header().Get(echo.HeaderXForwardedFor); ip != "" {
						ra = ip
					} else {
						ra, _, _ = net.SplitHostPort(ra)
					}
					return w.Write([]byte(ra))
				case "uri":
					return w.Write([]byte(rq.URI()))
				case "method":
					return w.Write([]byte(rq.Method()))
				case "path":
					p := rq.URL().Path()
					if p == "" {
						p = "/"
					}
					return w.Write([]byte(p))
				case "status":
					n := rs.Status()
					s := color.Green(n)
					switch {
					case n >= 500:
						s = color.Red(n)
					case n >= 400:
						s = color.Yellow(n)
					case n >= 300:
						s = color.Cyan(n)
					}
					return w.Write([]byte(s))
				case "response_time":
					return w.Write([]byte(stop.Sub(start).String()))
				case "response_size":
					return w.Write([]byte(strconv.FormatInt(rs.Size(), 10)))
				default:
					return w.Write([]byte(fmt.Sprintf("[unknown tag %s]", tag)))
				}
			})
			return
		}
	}
}
<commit_msg>Properly disable colors in Logger middleware<commit_after>package middleware

import (
	"fmt"
	"io"
	"net"
	"os"
	"strconv"
	"time"

	"github.com/labstack/echo"
	"github.com/labstack/gommon/color"
	isatty "github.com/mattn/go-isatty"
	"github.com/valyala/fasttemplate"
)

type (
	// LoggerConfig defines the config for logger middleware.
	LoggerConfig struct {
		// Format is the log format which can be constructed using the following tags:
		//
		// - time_rfc3339
		// - remote_ip
		// - uri
		// - method
		// - path
		// - status
		// - response_time
		// - response_size
		//
		// Example "${remote_id} ${status}"
		//
		// Optional with default value as `DefaultLoggerConfig.Format`.
		Format string

		// Output is the writer where logs are written.
		// Optional with default value as os.Stdout.
		Output io.Writer

		template *fasttemplate.Template
		color    *color.Color
	}
)

var (
	// DefaultLoggerConfig is the default logger middleware config.
	DefaultLoggerConfig = LoggerConfig{
		Format: "time=${time_rfc3339}, remote_ip=${remote_ip}, method=${method}, " +
			"uri=${uri}, status=${status}, took=${response_time}, sent=${response_size} bytes\n",
		color:  color.New(),
		Output: os.Stdout,
	}
)

// Logger returns a middleware that logs HTTP requests.
func Logger() echo.MiddlewareFunc {
	return LoggerWithConfig(DefaultLoggerConfig)
}

// LoggerWithConfig returns a logger middleware from config.
// See `Logger()`.
func LoggerWithConfig(config LoggerConfig) echo.MiddlewareFunc {
	// Defaults
	if config.Format == "" {
		config.Format = DefaultLoggerConfig.Format
	}
	if config.Output == nil {
		config.Output = DefaultLoggerConfig.Output
	}

	config.template = fasttemplate.New(config.Format, "${", "}")
	config.color = color.New()
	if w, ok := config.Output.(*os.File); !ok || !isatty.IsTerminal(w.Fd()) {
		config.color.Disable()
	}

	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c echo.Context) (err error) {
			rq := c.Request()
			rs := c.Response()
			start := time.Now()
			if err = next(c); err != nil {
				c.Error(err)
			}
			stop := time.Now()

			_, err = config.template.ExecuteFunc(config.Output, func(w io.Writer, tag string) (int, error) {
				switch tag {
				case "time_rfc3339":
					return w.Write([]byte(time.Now().Format(time.RFC3339)))
				case "remote_ip":
					ra := rq.RemoteAddress()
					if ip := rq.Header().Get(echo.HeaderXRealIP); ip != "" {
						ra = ip
					} else if ip = rq.Header().Get(echo.HeaderXForwardedFor); ip != "" {
						ra = ip
					} else {
						ra, _, _ = net.SplitHostPort(ra)
					}
					return w.Write([]byte(ra))
				case "uri":
					return w.Write([]byte(rq.URI()))
				case "method":
					return w.Write([]byte(rq.Method()))
				case "path":
					p := rq.URL().Path()
					if p == "" {
						p = "/"
					}
					return w.Write([]byte(p))
				case "status":
					n := rs.Status()
					s := config.color.Green(n)
					switch {
					case n >= 500:
						s = config.color.Red(n)
					case n >= 400:
						s = config.color.Yellow(n)
					case n >= 300:
						s = config.color.Cyan(n)
					}
					return w.Write([]byte(s))
				case "response_time":
					return w.Write([]byte(stop.Sub(start).String()))
				case "response_size":
					return w.Write([]byte(strconv.FormatInt(rs.Size(), 10)))
				default:
					return w.Write([]byte(fmt.Sprintf("[unknown tag %s]", tag)))
				}
			})
			return
		}
	}
}
<|endoftext|>