<commit_before>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)
// Author: Matthew O'Connor (matthew.t.oconnor@gmail.com)
// Author: Zach Brock (zbrock@gmail.com)

package storage

import (
	"bytes"
	"fmt"
	"sort"
	"testing"
	"time"

	"github.com/cockroachdb/cockroach/proto"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/util/hlc"
)

var testIdent = proto.StoreIdent{
	ClusterID: "cluster",
	NodeID:    1,
	StoreID:   1,
}

// TestStoreInitAndBootstrap verifies store initialization and
// bootstrap.
func TestStoreInitAndBootstrap(t *testing.T) {
	manual := hlc.ManualClock(0)
	clock := hlc.NewClock(manual.UnixNano)
	eng := engine.NewInMem(proto.Attributes{}, 1<<20)
	store := NewStore(clock, eng, nil, nil)
	defer store.Close()

	// Can't init as haven't bootstrapped.
	if err := store.Init(); err == nil {
		t.Error("expected failure init'ing un-bootstrapped store")
	}

	// Bootstrap with a fake ident.
	if err := store.Bootstrap(testIdent); err != nil {
		t.Errorf("error bootstrapping store: %v", err)
	}

	// Try to get 1st range--non-existent.
	if _, err := store.GetRange(1); err == nil {
		t.Error("expected error fetching non-existent range")
	}

	// Create range and fetch.
	if _, err := store.CreateRange(engine.KeyMin, engine.KeyMax, []proto.Replica{}); err != nil {
		t.Errorf("failure to create first range: %v", err)
	}
	if _, err := store.GetRange(1); err != nil {
		t.Errorf("failure fetching 1st range: %v", err)
	}

	// Now, attempt to initialize a store with a now-bootstrapped engine.
	store = NewStore(clock, eng, nil, nil)
	if err := store.Init(); err != nil {
		t.Errorf("failure initializing bootstrapped store: %v", err)
	}
	// 1st range should be available.
	if _, err := store.GetRange(1); err != nil {
		t.Errorf("failure fetching 1st range: %v", err)
	}
}

// TestBootstrapOfNonEmptyStore verifies bootstrap failure if engine
// is not empty.
func TestBootstrapOfNonEmptyStore(t *testing.T) {
	eng := engine.NewInMem(proto.Attributes{}, 1<<20)

	// Put some random garbage into the engine.
	if err := eng.Put(engine.Key("foo"), []byte("bar")); err != nil {
		t.Errorf("failure putting key foo into engine: %v", err)
	}
	manual := hlc.ManualClock(0)
	clock := hlc.NewClock(manual.UnixNano)
	store := NewStore(clock, eng, nil, nil)
	defer store.Close()

	// Can't init as haven't bootstrapped.
	if err := store.Init(); err == nil {
		t.Error("expected failure init'ing un-bootstrapped store")
	}

	// Bootstrap should fail on non-empty engine.
	if err := store.Bootstrap(testIdent); err == nil {
		t.Error("expected bootstrap error on non-empty store")
	}
}

func TestRangeSliceSort(t *testing.T) {
	var rs RangeSlice
	for i := 4; i >= 0; i-- {
		key := engine.Key(fmt.Sprintf("foo%d", i))
		rs = append(rs, &Range{
			Meta: &proto.RangeMetadata{
				RangeDescriptor: proto.RangeDescriptor{StartKey: key},
			},
		})
	}

	sort.Sort(rs)
	for i := 0; i < 5; i++ {
		expectedKey := engine.Key(fmt.Sprintf("foo%d", i))
		if !bytes.Equal(rs[i].Meta.StartKey, expectedKey) {
			t.Errorf("Expected %s, got %s", expectedKey, rs[i].Meta.StartKey)
		}
	}
}

// createTestStore creates a test store using an in-memory
// engine. Returns the store clock's manual unix nanos time and the
// store. If createDefaultRange is true, creates a single range from
// key "a" to key "z" with a default replica descriptor (i.e. StoreID
// = 0, RangeID = 1, etc.). The caller is responsible for closing the
// store on exit.
func createTestStore(createDefaultRange bool, t *testing.T) (*Store, *hlc.ManualClock) {
	manual := hlc.ManualClock(0)
	clock := hlc.NewClock(manual.UnixNano)
	eng := engine.NewInMem(proto.Attributes{}, 1<<20)
	store := NewStore(clock, eng, nil, nil)
	store.Ident.StoreID = 1
	replica := proto.Replica{StoreID: 1}
	// Create system key range for allocations.
	_, err := store.CreateRange(engine.KeySystemPrefix, engine.PrefixEndKey(engine.KeySystemPrefix), []proto.Replica{replica})
	if err != nil {
		t.Fatal(err)
	}
	// Now that the system key range is available, set store DB so new
	// ranges can be allocated as needed for tests.
	db, _ := newTestDB(store)
	store.db = db
	// If requested, create a default range for tests from "a"-"z".
	if createDefaultRange {
		_, err := store.CreateRange(engine.Key("a"), engine.Key("z"), []proto.Replica{replica})
		if err != nil {
			t.Fatal(err)
		}
	}
	return store, &manual
}

// TestStoreExecuteCmd verifies straightforward command execution
// of both a read-only and a read-write command.
func TestStoreExecuteCmd(t *testing.T) {
	store, _ := createTestStore(true, t)
	defer store.Close()
	args, reply := getArgs([]byte("a"), 2)

	// Try a successful get request.
	err := store.ExecuteCmd("Get", args, reply)
	if err != nil {
		t.Fatal(err)
	}
}

// TestStoreExecuteCmdUpdateTime verifies that the node clock is updated.
func TestStoreExecuteCmdUpdateTime(t *testing.T) {
	store, _ := createTestStore(true, t)
	defer store.Close()
	args, reply := getArgs([]byte("a"), 2)
	args.Timestamp = store.clock.Now()
	args.Timestamp.WallTime += (100 * time.Millisecond).Nanoseconds()
	err := store.ExecuteCmd("Get", args, reply)
	if err != nil {
		t.Fatal(err)
	}
	ts := store.clock.Timestamp()
	if ts.WallTime != args.Timestamp.WallTime || ts.Logical <= args.Timestamp.Logical {
		t.Errorf("expected store clock to advance to %+v; got %+v", args.Timestamp, ts)
	}
}

// TestStoreExecuteCmdWithZeroTime verifies that no timestamp causes
// the command to assume the node's wall time.
func TestStoreExecuteCmdWithZeroTime(t *testing.T) {
	store, mc := createTestStore(true, t)
	defer store.Close()
	args, reply := getArgs([]byte("a"), 2)

	// Set clock to time 1.
	*mc = hlc.ManualClock(1)
	err := store.ExecuteCmd("Get", args, reply)
	if err != nil {
		t.Fatal(err)
	}
	// The Logical time will increase over the course of the command
	// execution so we can only rely on comparing the WallTime.
	if reply.Timestamp.WallTime != store.clock.Timestamp().WallTime {
		t.Errorf("expected reply to have store clock time %+v; got %+v",
			store.clock.Timestamp(), reply.Timestamp)
	}
}

// TestStoreExecuteCmdWithClockDrift verifies that if the request
// specifies a timestamp further into the future than the node's
// maximum allowed clock drift, the cmd fails with an error.
func TestStoreExecuteCmdWithClockDrift(t *testing.T) {
	store, mc := createTestStore(true, t)
	defer store.Close()
	args, reply := getArgs([]byte("a"), 2)

	// Set clock to time 1.
	*mc = hlc.ManualClock(1)
	// Set clock max drift to 250ms.
	maxDrift := 250 * time.Millisecond
	store.clock.SetMaxDrift(maxDrift)
	// Set args timestamp to exceed max drift.
	args.Timestamp = store.clock.Now()
	args.Timestamp.WallTime += maxDrift.Nanoseconds() + 1
	err := store.ExecuteCmd("Get", args, reply)
	if err == nil {
		t.Error("expected max drift clock error")
	}
}

// TestStoreExecuteCmdBadRange passes a bad range.
func TestStoreExecuteCmdBadRange(t *testing.T) {
	store, _ := createTestStore(true, t)
	defer store.Close()
	// Range is from "a" to "z", so this value should fail.
	args, reply := getArgs([]byte("0"), 2)
	args.RangeID = 2
	err := store.ExecuteCmd("Get", args, reply)
	if err == nil {
		t.Error("expected invalid range")
	}
}

// TestStoreExecuteCmdOutOfRange passes a key not contained
// within the range's key range.
func TestStoreExecuteCmdOutOfRange(t *testing.T) {
	store, _ := createTestStore(true, t)
	defer store.Close()
	// Range is from "a" to "z", so this value should fail.
	args, reply := getArgs([]byte("0"), 2)
	err := store.ExecuteCmd("Get", args, reply)
	if err == nil {
		t.Error("expected key to be out of range")
	}
}

func addTestRange(store *Store, start, end engine.Key, t *testing.T) *Range {
	replicas := []proto.Replica{
		proto.Replica{StoreID: store.Ident.StoreID},
	}
	r, err := store.CreateRange(start, end, replicas)
	if err != nil {
		t.Fatal(err)
	}
	return r
}

// TestStoreRangesByKey verifies we can lookup ranges by key using
// the sorted rangesByKey slice.
func TestStoreRangesByKey(t *testing.T) {
	store, _ := createTestStore(false, t)
	defer store.Close()

	r1 := addTestRange(store, engine.Key("A"), engine.Key("C"), t)
	r2 := addTestRange(store, engine.Key("C"), engine.Key("X"), t)
	r3 := addTestRange(store, engine.Key("X"), engine.Key("ZZ"), t)

	if store.LookupRange(engine.Key("a"), nil) != nil {
		t.Errorf("expected \"a\" to not have an associated range")
	}
	if r := store.LookupRange(engine.Key("B"), nil); r != r1 {
		t.Errorf("mismatched range %+v != %+v", r, r1.Meta)
	}
	if r := store.LookupRange(engine.Key("C"), nil); r != r2 {
		t.Errorf("mismatched range %+v != %+v", r, r2.Meta)
	}
	if r := store.LookupRange(engine.Key("M"), nil); r != r2 {
		t.Errorf("mismatched range %+v != %+v", r, r2.Meta)
	}
	if r := store.LookupRange(engine.Key("X"), nil); r != r3 {
		t.Errorf("mismatched range %+v != %+v", r, r3.Meta)
	}
	if r := store.LookupRange(engine.Key("Z"), nil); r != r3 {
		t.Errorf("mismatched range %+v != %+v", r, r3.Meta)
	}
	if store.LookupRange(engine.KeyMax, nil) != nil {
		t.Errorf("expected engine.KeyMax to not have an associated range")
	}
}
<commit_msg>Added a test for allocation of successive blocks of range IDs.<commit_after>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)
// Author: Matthew O'Connor (matthew.t.oconnor@gmail.com)
// Author: Zach Brock (zbrock@gmail.com)

package storage

import (
	"bytes"
	"fmt"
	"sort"
	"testing"
	"time"

	"github.com/cockroachdb/cockroach/proto"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/util/hlc"
)

var testIdent = proto.StoreIdent{
	ClusterID: "cluster",
	NodeID:    1,
	StoreID:   1,
}

// TestStoreInitAndBootstrap verifies store initialization and
// bootstrap.
func TestStoreInitAndBootstrap(t *testing.T) {
	manual := hlc.ManualClock(0)
	clock := hlc.NewClock(manual.UnixNano)
	eng := engine.NewInMem(proto.Attributes{}, 1<<20)
	store := NewStore(clock, eng, nil, nil)
	defer store.Close()

	// Can't init as haven't bootstrapped.
	if err := store.Init(); err == nil {
		t.Error("expected failure init'ing un-bootstrapped store")
	}

	// Bootstrap with a fake ident.
	if err := store.Bootstrap(testIdent); err != nil {
		t.Errorf("error bootstrapping store: %v", err)
	}

	// Try to get 1st range--non-existent.
	if _, err := store.GetRange(1); err == nil {
		t.Error("expected error fetching non-existent range")
	}

	// Create range and fetch.
	if _, err := store.CreateRange(engine.KeyMin, engine.KeyMax, []proto.Replica{}); err != nil {
		t.Errorf("failure to create first range: %v", err)
	}
	if _, err := store.GetRange(1); err != nil {
		t.Errorf("failure fetching 1st range: %v", err)
	}

	// Now, attempt to initialize a store with a now-bootstrapped engine.
	store = NewStore(clock, eng, nil, nil)
	if err := store.Init(); err != nil {
		t.Errorf("failure initializing bootstrapped store: %v", err)
	}
	// 1st range should be available.
	if _, err := store.GetRange(1); err != nil {
		t.Errorf("failure fetching 1st range: %v", err)
	}
}

// TestBootstrapOfNonEmptyStore verifies bootstrap failure if engine
// is not empty.
func TestBootstrapOfNonEmptyStore(t *testing.T) {
	eng := engine.NewInMem(proto.Attributes{}, 1<<20)

	// Put some random garbage into the engine.
	if err := eng.Put(engine.Key("foo"), []byte("bar")); err != nil {
		t.Errorf("failure putting key foo into engine: %v", err)
	}
	manual := hlc.ManualClock(0)
	clock := hlc.NewClock(manual.UnixNano)
	store := NewStore(clock, eng, nil, nil)
	defer store.Close()

	// Can't init as haven't bootstrapped.
	if err := store.Init(); err == nil {
		t.Error("expected failure init'ing un-bootstrapped store")
	}

	// Bootstrap should fail on non-empty engine.
	if err := store.Bootstrap(testIdent); err == nil {
		t.Error("expected bootstrap error on non-empty store")
	}
}

func TestRangeSliceSort(t *testing.T) {
	var rs RangeSlice
	for i := 4; i >= 0; i-- {
		key := engine.Key(fmt.Sprintf("foo%d", i))
		rs = append(rs, &Range{
			Meta: &proto.RangeMetadata{
				RangeDescriptor: proto.RangeDescriptor{StartKey: key},
			},
		})
	}

	sort.Sort(rs)
	for i := 0; i < 5; i++ {
		expectedKey := engine.Key(fmt.Sprintf("foo%d", i))
		if !bytes.Equal(rs[i].Meta.StartKey, expectedKey) {
			t.Errorf("Expected %s, got %s", expectedKey, rs[i].Meta.StartKey)
		}
	}
}

// createTestStore creates a test store using an in-memory
// engine. Returns the store clock's manual unix nanos time and the
// store. If createDefaultRange is true, creates a single range from
// key "a" to key "z" with a default replica descriptor (i.e. StoreID
// = 0, RangeID = 1, etc.). The caller is responsible for closing the
// store on exit.
func createTestStore(createDefaultRange bool, t *testing.T) (*Store, *hlc.ManualClock) {
	manual := hlc.ManualClock(0)
	clock := hlc.NewClock(manual.UnixNano)
	eng := engine.NewInMem(proto.Attributes{}, 1<<20)
	store := NewStore(clock, eng, nil, nil)
	store.Ident.StoreID = 1
	replica := proto.Replica{StoreID: 1}
	// Create system key range for allocations.
	_, err := store.CreateRange(engine.KeySystemPrefix, engine.PrefixEndKey(engine.KeySystemPrefix), []proto.Replica{replica})
	if err != nil {
		t.Fatal(err)
	}
	// Now that the system key range is available, set store DB so new
	// ranges can be allocated as needed for tests.
	db, _ := newTestDB(store)
	store.db = db
	// If requested, create a default range for tests from "a"-"z".
	if createDefaultRange {
		_, err := store.CreateRange(engine.Key("a"), engine.Key("z"), []proto.Replica{replica})
		if err != nil {
			t.Fatal(err)
		}
	}
	return store, &manual
}

// TestStoreExecuteCmd verifies straightforward command execution
// of both a read-only and a read-write command.
func TestStoreExecuteCmd(t *testing.T) {
	store, _ := createTestStore(true, t)
	defer store.Close()
	args, reply := getArgs([]byte("a"), 2)

	// Try a successful get request.
	err := store.ExecuteCmd("Get", args, reply)
	if err != nil {
		t.Fatal(err)
	}
}

// TestStoreExecuteCmdUpdateTime verifies that the node clock is updated.
func TestStoreExecuteCmdUpdateTime(t *testing.T) {
	store, _ := createTestStore(true, t)
	defer store.Close()
	args, reply := getArgs([]byte("a"), 2)
	args.Timestamp = store.clock.Now()
	args.Timestamp.WallTime += (100 * time.Millisecond).Nanoseconds()
	err := store.ExecuteCmd("Get", args, reply)
	if err != nil {
		t.Fatal(err)
	}
	ts := store.clock.Timestamp()
	if ts.WallTime != args.Timestamp.WallTime || ts.Logical <= args.Timestamp.Logical {
		t.Errorf("expected store clock to advance to %+v; got %+v", args.Timestamp, ts)
	}
}

// TestStoreExecuteCmdWithZeroTime verifies that no timestamp causes
// the command to assume the node's wall time.
func TestStoreExecuteCmdWithZeroTime(t *testing.T) {
	store, mc := createTestStore(true, t)
	defer store.Close()
	args, reply := getArgs([]byte("a"), 2)

	// Set clock to time 1.
	*mc = hlc.ManualClock(1)
	err := store.ExecuteCmd("Get", args, reply)
	if err != nil {
		t.Fatal(err)
	}
	// The Logical time will increase over the course of the command
	// execution so we can only rely on comparing the WallTime.
	if reply.Timestamp.WallTime != store.clock.Timestamp().WallTime {
		t.Errorf("expected reply to have store clock time %+v; got %+v",
			store.clock.Timestamp(), reply.Timestamp)
	}
}

// TestStoreExecuteCmdWithClockDrift verifies that if the request
// specifies a timestamp further into the future than the node's
// maximum allowed clock drift, the cmd fails with an error.
func TestStoreExecuteCmdWithClockDrift(t *testing.T) {
	store, mc := createTestStore(true, t)
	defer store.Close()
	args, reply := getArgs([]byte("a"), 2)

	// Set clock to time 1.
	*mc = hlc.ManualClock(1)
	// Set clock max drift to 250ms.
	maxDrift := 250 * time.Millisecond
	store.clock.SetMaxDrift(maxDrift)
	// Set args timestamp to exceed max drift.
	args.Timestamp = store.clock.Now()
	args.Timestamp.WallTime += maxDrift.Nanoseconds() + 1
	err := store.ExecuteCmd("Get", args, reply)
	if err == nil {
		t.Error("expected max drift clock error")
	}
}

// TestStoreExecuteCmdBadRange passes a bad range.
func TestStoreExecuteCmdBadRange(t *testing.T) {
	store, _ := createTestStore(true, t)
	defer store.Close()
	// Range is from "a" to "z", so this value should fail.
	args, reply := getArgs([]byte("0"), 2)
	args.RangeID = 2
	err := store.ExecuteCmd("Get", args, reply)
	if err == nil {
		t.Error("expected invalid range")
	}
}

// TestStoreExecuteCmdOutOfRange passes a key not contained
// within the range's key range.
func TestStoreExecuteCmdOutOfRange(t *testing.T) {
	store, _ := createTestStore(true, t)
	defer store.Close()
	// Range is from "a" to "z", so this value should fail.
	args, reply := getArgs([]byte("0"), 2)
	err := store.ExecuteCmd("Get", args, reply)
	if err == nil {
		t.Error("expected key to be out of range")
	}
}

// TestStoreRangeIDAllocation verifies that range IDs are
// allocated in successive blocks.
func TestStoreRangeIDAllocation(t *testing.T) {
	store, _ := createTestStore(false, t)
	defer store.Close()

	// Range IDs should be allocated from ID 2 (first alloc'd range)
	// to rangeIDCount * 3 + 1.
	for i := 0; i < rangeIDAllocCount*3; i++ {
		r := addTestRange(store, engine.Key(fmt.Sprintf("%03d", i)), engine.Key(fmt.Sprintf("%03d", i+1)), t)
		if r.Meta.RangeID != int64(2+i) {
			t.Error("expected range id %d; got %d", 2+i, r.Meta.RangeID)
		}
	}
}

func addTestRange(store *Store, start, end engine.Key, t *testing.T) *Range {
	replicas := []proto.Replica{
		proto.Replica{StoreID: store.Ident.StoreID},
	}
	r, err := store.CreateRange(start, end, replicas)
	if err != nil {
		t.Fatal(err)
	}
	return r
}

// TestStoreRangesByKey verifies we can lookup ranges by key using
// the sorted rangesByKey slice.
func TestStoreRangesByKey(t *testing.T) {
	store, _ := createTestStore(false, t)
	defer store.Close()

	r1 := addTestRange(store, engine.Key("A"), engine.Key("C"), t)
	r2 := addTestRange(store, engine.Key("C"), engine.Key("X"), t)
	r3 := addTestRange(store, engine.Key("X"), engine.Key("ZZ"), t)

	if store.LookupRange(engine.Key("a"), nil) != nil {
		t.Errorf("expected \"a\" to not have an associated range")
	}
	if r := store.LookupRange(engine.Key("B"), nil); r != r1 {
		t.Errorf("mismatched range %+v != %+v", r, r1.Meta)
	}
	if r := store.LookupRange(engine.Key("C"), nil); r != r2 {
		t.Errorf("mismatched range %+v != %+v", r, r2.Meta)
	}
	if r := store.LookupRange(engine.Key("M"), nil); r != r2 {
		t.Errorf("mismatched range %+v != %+v", r, r2.Meta)
	}
	if r := store.LookupRange(engine.Key("X"), nil); r != r3 {
		t.Errorf("mismatched range %+v != %+v", r, r3.Meta)
	}
	if r := store.LookupRange(engine.Key("Z"), nil); r != r3 {
		t.Errorf("mismatched range %+v != %+v", r, r3.Meta)
	}
	if store.LookupRange(engine.KeyMax, nil) != nil {
		t.Errorf("expected engine.KeyMax to not have an associated range")
	}
}
<|endoftext|>


<commit_before>package api

import (
	k8score "k8s.io/api/core/v1"
	"gopkg.in/yaml.v2"
	"github.com/golang/glog"
)

type PrometheusAlertGroup struct {
	Name  string
	Rules []PrometheusAlertRule
}

type PrometheusAlertRule struct {
	Alert       string
	Expr        string
	For         string
	Labels      map[string]string
	Annotations map[string]string
}

func addRulesToConfigMap(configMap *k8score.ConfigMap, deploymentRequest NaisDeploymentRequest, manifest NaisManifest) (*k8score.ConfigMap, error) {
	ruleGroupName := deploymentRequest.Namespace + deploymentRequest.Application
	alertGroup := PrometheusAlertGroup{Name: ruleGroupName, Rules: manifest.Alerts}
	alertGroupYamlBytes, err := yaml.Marshal(alertGroup)
	if err != nil {
		glog.Errorf("Failed to marshal %v to yaml\n", alertGroup)
		return nil, err
	}

	if configMap.Data == nil {
		configMap.Data = make(map[string]string)
	}

	configMap.Data[ruleGroupName + ".yaml"] = string(alertGroupYamlBytes)

	return configMap, nil
}

func validateAlertRules(manifest NaisManifest) *ValidationError {
	for _, alertRule := range manifest.Alerts {
		if alertRule.Alert == "" {
			return &ValidationError{
				"Alert must be specified",
				map[string]string{"Alert": alertRule.Alert},
			}
		}
		if alertRule.Expr == "" {
			return &ValidationError{
				"Expr must be specified",
				map[string]string{"Expr": alertRule.Expr},
			}
		}
		if alertRule.For == "" {
			return &ValidationError{
				"For must be specified",
				map[string]string{"For": alertRule.For},
			}
		}
		if action, exists := alertRule.Annotations["action"]; !exists {
			return &ValidationError{
				"An annotation named action must be specified",
				map[string]string{"annotations[action]": action},
			}
		}
	}

	return nil
}
<commit_msg>Filename for prometheus alert has to be .yml<commit_after>package api

import (
	k8score "k8s.io/api/core/v1"
	"gopkg.in/yaml.v2"
	"github.com/golang/glog"
)

type PrometheusAlertGroup struct {
	Name  string
	Rules []PrometheusAlertRule
}

type PrometheusAlertRule struct {
	Alert       string
	Expr        string
	For         string
	Labels      map[string]string
	Annotations map[string]string
}

func addRulesToConfigMap(configMap *k8score.ConfigMap, deploymentRequest NaisDeploymentRequest, manifest NaisManifest) (*k8score.ConfigMap, error) {
	ruleGroupName := deploymentRequest.Namespace + deploymentRequest.Application
	alertGroup := PrometheusAlertGroup{Name: ruleGroupName, Rules: manifest.Alerts}
	alertGroupYamlBytes, err := yaml.Marshal(alertGroup)
	if err != nil {
		glog.Errorf("Failed to marshal %v to yaml\n", alertGroup)
		return nil, err
	}

	if configMap.Data == nil {
		configMap.Data = make(map[string]string)
	}

	configMap.Data[ruleGroupName + ".yml"] = string(alertGroupYamlBytes)

	return configMap, nil
}

func validateAlertRules(manifest NaisManifest) *ValidationError {
	for _, alertRule := range manifest.Alerts {
		if alertRule.Alert == "" {
			return &ValidationError{
				"Alert must be specified",
				map[string]string{"Alert": alertRule.Alert},
			}
		}
		if alertRule.Expr == "" {
			return &ValidationError{
				"Expr must be specified",
				map[string]string{"Expr": alertRule.Expr},
			}
		}
		if alertRule.For == "" {
			return &ValidationError{
				"For must be specified",
				map[string]string{"For": alertRule.For},
			}
		}
		if action, exists := alertRule.Annotations["action"]; !exists {
			return &ValidationError{
				"An annotation named action must be specified",
				map[string]string{"annotations[action]": action},
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package filer2

import (
	"bytes"
	"io"
	"math"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
)

func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {

	chunkViews := ViewFromChunks(chunks, offset, size)

	fileId2Url := make(map[string]string)

	for _, chunkView := range chunkViews {

		urlString, err := masterClient.LookupFileId(chunkView.FileId)
		if err != nil {
			glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
			return err
		}
		fileId2Url[chunkView.FileId] = urlString
	}

	for _, chunkView := range chunkViews {

		urlString := fileId2Url[chunkView.FileId]
		err := util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
			w.Write(data)
		})
		if err != nil {
			glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err)
			return err
		}
	}

	return nil

}

// ----------------  ReadAllReader ----------------------------------

func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) ([]byte, error) {

	buffer := bytes.Buffer{}

	chunkViews := ViewFromChunks(chunks, 0, math.MaxInt32)

	lookupFileId := func(fileId string) (targetUrl string, err error) {
		return masterClient.LookupFileId(fileId)
	}

	for _, chunkView := range chunkViews {
		urlString, err := lookupFileId(chunkView.FileId)
		if err != nil {
			glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
			return nil, err
		}
		err = util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
			buffer.Write(data)
		})
		if err != nil {
			glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err)
			return nil, err
		}
	}
	return buffer.Bytes(), nil
}

// ----------------  ChunkStreamReader ----------------------------------
type ChunkStreamReader struct {
	chunkViews   []*ChunkView
	logicOffset  int64
	buffer       []byte
	bufferOffset int64
	bufferPos    int
	chunkIndex   int
	lookupFileId func(fileId string) (targetUrl string, err error)
}

var _ = io.ReadSeeker(&ChunkStreamReader{})

func NewChunkStreamReaderFromFiler(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader {

	chunkViews := ViewFromChunks(chunks, 0, math.MaxInt32)

	return &ChunkStreamReader{
		chunkViews: chunkViews,
		lookupFileId: func(fileId string) (targetUrl string, err error) {
			return masterClient.LookupFileId(fileId)
		},
	}
}

func (c *ChunkStreamReader) Read(p []byte) (n int, err error) {
	if c.isBufferEmpty() {
		if c.chunkIndex >= len(c.chunkViews) {
			return 0, io.EOF
		}
		chunkView := c.chunkViews[c.chunkIndex]
		c.fetchChunkToBuffer(chunkView)
		c.chunkIndex++
	}
	n = copy(p, c.buffer[c.bufferPos:])
	c.bufferPos += n
	return
}

func (c *ChunkStreamReader) isBufferEmpty() bool {
	return len(c.buffer) <= c.bufferPos
}

func (c *ChunkStreamReader) Seek(offset int64, whence int) (int64, error) {

	var totalSize int64
	for _, chunk := range c.chunkViews {
		totalSize += int64(chunk.Size)
	}

	var err error
	switch whence {
	case io.SeekStart:
	case io.SeekCurrent:
		offset += c.bufferOffset + int64(c.bufferPos)
	case io.SeekEnd:
		offset = totalSize + offset
	}
	if offset > totalSize {
		err = io.ErrUnexpectedEOF
	}

	for i, chunk := range c.chunkViews {
		if chunk.LogicOffset <= offset && offset < chunk.LogicOffset+int64(chunk.Size) {
			if c.isBufferEmpty() || c.bufferOffset != chunk.LogicOffset {
				c.fetchChunkToBuffer(chunk)
				c.chunkIndex = i + 1
				break
			}
		}
	}
	c.bufferPos = int(offset - c.bufferOffset)

	return offset, err

}

func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
	urlString, err := c.lookupFileId(chunkView.FileId)
	if err != nil {
		glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
		return err
	}
	var buffer bytes.Buffer
	err = util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
		buffer.Write(data)
	})
	if err != nil {
		glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err)
		return err
	}
	c.buffer = buffer.Bytes()
	c.bufferPos = 0
	c.bufferOffset = chunkView.LogicOffset

	// glog.V(0).Infof("read %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size))

	return nil
}

func (c *ChunkStreamReader) Close() {
	// TODO try to release and reuse buffer
}

func VolumeId(fileId string) string {
	lastCommaIndex := strings.LastIndex(fileId, ",")
	if lastCommaIndex > 0 {
		return fileId[:lastCommaIndex]
	}
	return fileId
}
<commit_msg>read in case cross chunks<commit_after>package filer2

import (
	"bytes"
	"io"
	"math"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
)

func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {

	chunkViews := ViewFromChunks(chunks, offset, size)

	fileId2Url := make(map[string]string)

	for _, chunkView := range chunkViews {

		urlString, err := masterClient.LookupFileId(chunkView.FileId)
		if err != nil {
			glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
			return err
		}
		fileId2Url[chunkView.FileId] = urlString
	}

	for _, chunkView := range chunkViews {

		urlString := fileId2Url[chunkView.FileId]
		err := util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
			w.Write(data)
		})
		if err != nil {
			glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err)
			return err
		}
	}

	return nil

}

// ----------------  ReadAllReader ----------------------------------

func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) ([]byte, error) {

	buffer := bytes.Buffer{}

	chunkViews := ViewFromChunks(chunks, 0, math.MaxInt32)

	lookupFileId := func(fileId string) (targetUrl string, err error) {
		return masterClient.LookupFileId(fileId)
	}

	for _, chunkView := range chunkViews {
		urlString, err := lookupFileId(chunkView.FileId)
		if err != nil {
			glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
			return nil, err
		}
		err = util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
			buffer.Write(data)
		})
		if err != nil {
			glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err)
			return nil, err
		}
	}
	return buffer.Bytes(), nil
}

// ----------------  ChunkStreamReader ----------------------------------
type ChunkStreamReader struct {
	chunkViews   []*ChunkView
	logicOffset  int64
	buffer       []byte
	bufferOffset int64
	bufferPos    int
	chunkIndex   int
	lookupFileId func(fileId string) (targetUrl string, err error)
}

var _ = io.ReadSeeker(&ChunkStreamReader{})

func NewChunkStreamReaderFromFiler(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader {

	chunkViews := ViewFromChunks(chunks, 0, math.MaxInt32)

	return &ChunkStreamReader{
		chunkViews: chunkViews,
		lookupFileId: func(fileId string) (targetUrl string, err error) {
			return masterClient.LookupFileId(fileId)
		},
	}
}

func (c *ChunkStreamReader) Read(p []byte) (n int, err error) {
	for n < len(p) {
		if c.isBufferEmpty() {
			if c.chunkIndex >= len(c.chunkViews) {
				return n, io.EOF
			}
			chunkView := c.chunkViews[c.chunkIndex]
			c.fetchChunkToBuffer(chunkView)
			c.chunkIndex++
		}
		t := copy(p[n:], c.buffer[c.bufferPos:])
		c.bufferPos += t
		n += t
	}
	return
}

func (c *ChunkStreamReader) isBufferEmpty() bool {
	return len(c.buffer) <= c.bufferPos
}

func (c *ChunkStreamReader) Seek(offset int64, whence int) (int64, error) {

	var totalSize int64
	for _, chunk := range c.chunkViews {
		totalSize += int64(chunk.Size)
	}

	var err error
	switch whence {
	case io.SeekStart:
	case io.SeekCurrent:
		offset += c.bufferOffset + int64(c.bufferPos)
	case io.SeekEnd:
		offset = totalSize + offset
	}
	if offset > totalSize {
		err = io.ErrUnexpectedEOF
	}

	for i, chunk := range c.chunkViews {
		if chunk.LogicOffset <= offset && offset < chunk.LogicOffset+int64(chunk.Size) {
			if c.isBufferEmpty() || c.bufferOffset != chunk.LogicOffset {
				c.fetchChunkToBuffer(chunk)
				c.chunkIndex = i + 1
				break
			}
		}
	}
	c.bufferPos = int(offset - c.bufferOffset)

	return offset, err

}

func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
	urlString, err := c.lookupFileId(chunkView.FileId)
	if err != nil {
		glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
		return err
	}
	var buffer bytes.Buffer
	err = util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
		buffer.Write(data)
	})
	if err != nil {
		glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err)
		return err
	}
	c.buffer = buffer.Bytes()
	c.bufferPos = 0
	c.bufferOffset = chunkView.LogicOffset

	// glog.V(0).Infof("read %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size))

	return nil
}

func (c *ChunkStreamReader) Close() {
	// TODO try to release and reuse buffer
}

func VolumeId(fileId string) string {
	lastCommaIndex := strings.LastIndex(fileId, ",")
	if lastCommaIndex > 0 {
		return fileId[:lastCommaIndex]
	}
	return fileId
}
<|endoftext|>


<commit_before>package weed_server

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
	"io"
	"io/fs"
	"mime/multipart"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/util"

	"github.com/gorilla/mux"
)

var serverStats *stats.ServerStats
var startTime = time.Now()

func init() {
	serverStats = stats.NewServerStats()
	go serverStats.Start()
}

// bodyAllowedForStatus is a copy of http.bodyAllowedForStatus non-exported function.
func bodyAllowedForStatus(status int) bool {
	switch {
	case status >= 100 && status <= 199:
		return false
	case status == http.StatusNoContent:
		return false
	case status == http.StatusNotModified:
		return false
	}
	return true
}

func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) (err error) {
	if !bodyAllowedForStatus(httpStatus) {
		return
	}

	var bytes []byte
	if obj != nil {
		if r.FormValue("pretty") != "" {
			bytes, err = json.MarshalIndent(obj, "", "  ")
		} else {
			bytes, err = json.Marshal(obj)
		}
	}
	if err != nil {
		return
	}

	if httpStatus >= 400 {
		glog.V(0).Infof("response method:%s URL:%s with httpStatus:%d and JSON:%s",
			r.Method, r.URL.String(), httpStatus, string(bytes))
	}

	callback := r.FormValue("callback")
	if callback == "" {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(httpStatus)
		if httpStatus == http.StatusNotModified {
			return
		}
		_, err = w.Write(bytes)
	} else {
		w.Header().Set("Content-Type", "application/javascript")
		w.WriteHeader(httpStatus)
		if httpStatus == http.StatusNotModified {
			return
		}
		if _, err = w.Write([]uint8(callback)); err != nil {
			return
		}
		if _, err = w.Write([]uint8("(")); err != nil {
			return
		}
		fmt.Fprint(w, string(bytes))
		if _, err = w.Write([]uint8(")")); err != nil {
			return
		}
	}

	return
}

// wrapper for writeJson - just logs errors
func writeJsonQuiet(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) {
	if err := writeJson(w, r, httpStatus, obj); err != nil {
		glog.V(0).Infof("error writing JSON status %d: %v", httpStatus, err)
		glog.V(1).Infof("JSON content: %+v", obj)
	}
}
func writeJsonError(w http.ResponseWriter, r *http.Request, httpStatus int, err error) {
	m := make(map[string]interface{})
	m["error"] = err.Error()
	writeJsonQuiet(w, r, httpStatus, m)
}

func debug(params ...interface{}) {
	glog.V(4).Infoln(params...)
}

func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption) {
	m := make(map[string]interface{})
	if r.Method != "POST" {
		writeJsonError(w, r, http.StatusMethodNotAllowed, errors.New("Only submit via POST!"))
		return
	}

	debug("parsing upload file...")
	bytesBuffer := bufPool.Get().(*bytes.Buffer)
	defer bufPool.Put(bytesBuffer)
	pu, pe := needle.ParseUpload(r, 256*1024*1024, bytesBuffer)
	if pe != nil {
		writeJsonError(w, r, http.StatusBadRequest, pe)
		return
	}

	debug("assigning file id for", pu.FileName)
	r.ParseForm()
	count := uint64(1)
	if r.FormValue("count") != "" {
		count, pe = strconv.ParseUint(r.FormValue("count"), 10, 32)
		if pe != nil {
			writeJsonError(w, r, http.StatusBadRequest, pe)
			return
		}
	}
	ar := &operation.VolumeAssignRequest{
		Count:       count,
		DataCenter:  r.FormValue("dataCenter"),
		Rack:        r.FormValue("rack"),
		Replication: r.FormValue("replication"),
		Collection:  r.FormValue("collection"),
		Ttl:         r.FormValue("ttl"),
		DiskType:    r.FormValue("disk"),
	}
	assignResult, ae := operation.Assign(masterFn, grpcDialOption, ar)
	if ae != nil {
		writeJsonError(w, r, http.StatusInternalServerError, ae)
		return
	}

	url := "http://" + assignResult.Url + "/" + assignResult.Fid
	if pu.ModifiedTime != 0 {
		url = url + "?ts=" + strconv.FormatUint(pu.ModifiedTime, 10)
	}

	debug("upload file to store", url)
	uploadOption := &operation.UploadOption{
		UploadUrl:         url,
		Filename:          pu.FileName,
		Cipher:            false,
		IsInputCompressed: pu.IsGzipped,
		MimeType:          pu.MimeType,
		PairMap:           pu.PairMap,
		Jwt:               assignResult.Auth,
	}
	uploadResult, err := operation.UploadData(pu.Data, uploadOption)
	if err != nil {
		writeJsonError(w, r, http.StatusInternalServerError, err)
		return
	}

	m["fileName"] = pu.FileName
	m["fid"] = assignResult.Fid
	m["fileUrl"] = assignResult.PublicUrl + "/" + assignResult.Fid
	m["size"] = pu.OriginalDataSize
	m["eTag"] = uploadResult.ETag
	writeJsonQuiet(w, r, http.StatusCreated, m)
	return
}

func parseURLPath(path string) (vid, fid, filename, ext string, isVolumeIdOnly bool) {
	switch strings.Count(path, "/") {
	case 3:
		parts := strings.Split(path, "/")
		vid, fid, filename = parts[1], parts[2], parts[3]
		ext = filepath.Ext(filename)
	case 2:
		parts := strings.Split(path, "/")
		vid, fid = parts[1], parts[2]
		dotIndex := strings.LastIndex(fid, ".")
		if dotIndex > 0 {
			ext = fid[dotIndex:]
			fid = fid[0:dotIndex]
		}
	default:
		sepIndex := strings.LastIndex(path, "/")
		commaIndex := strings.LastIndex(path[sepIndex:], ",")
		if commaIndex <= 0 {
			vid, isVolumeIdOnly = path[sepIndex+1:], true
			return
		}
		dotIndex := strings.LastIndex(path[sepIndex:], ".")
		vid = path[sepIndex+1 : commaIndex]
		fid = path[commaIndex+1:]
		ext = ""
		if dotIndex > 0 {
			fid = path[commaIndex+1 : dotIndex]
			ext = path[dotIndex:]
		}
	}
	return
}

func statsHealthHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	writeJsonQuiet(w, r, http.StatusOK, m)
}
func statsCounterHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	m["Counters"] = serverStats
	writeJsonQuiet(w, r, http.StatusOK, m)
}

func statsMemoryHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	m["Memory"] = stats.MemStat()
	writeJsonQuiet(w, r, http.StatusOK, m)
}

var StaticFS fs.FS

func handleStaticResources(defaultMux *http.ServeMux) {
	defaultMux.Handle("/favicon.ico", http.FileServer(http.FS(StaticFS)))
	defaultMux.Handle("/seaweedfsstatic/", http.StripPrefix("/seaweedfsstatic", http.FileServer(http.FS(StaticFS))))
}

func handleStaticResources2(r *mux.Router) {
	r.Handle("/favicon.ico", http.FileServer(http.FS(StaticFS)))
	r.PathPrefix("/seaweedfsstatic/").Handler(http.StripPrefix("/seaweedfsstatic", http.FileServer(http.FS(StaticFS))))
}

func adjustPassthroughHeaders(w http.ResponseWriter, r *http.Request, filename string) {
	for header, values := range r.Header {
		if normalizedHeader, ok := s3_constants.PassThroughHeaders[strings.ToLower(header)]; ok {
			w.Header()[normalizedHeader] = values
		}
	}
	adjustHeaderContentDisposition(w, r, filename)
}
func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, filename string) {
	if contentDisposition := w.Header().Get("Content-Disposition"); contentDisposition != "" {
		return
	}
	if filename != "" {
		filename = url.QueryEscape(filename)
		contentDisposition := "inline"
		if r.FormValue("dl") != "" {
			if dl, _ := strconv.ParseBool(r.FormValue("dl")); dl {
				contentDisposition = "attachment"
			}
		}
		w.Header().Set("Content-Disposition", contentDisposition+`; filename="`+fileNameEscaper.Replace(filename)+`"`)
	}
}

func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64) error) {
	rangeReq := r.Header.Get("Range")
	bufferedWriter := bufio.NewWriterSize(w, 128*1024)
	defer bufferedWriter.Flush()

	if rangeReq == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
		if err := writeFn(bufferedWriter, 0, totalSize); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	}

	//the rest is dealing with partial content request
	//mostly copy from src/pkg/net/http/fs.go
	ranges, err := parseRange(rangeReq, totalSize)
	if err != nil {
		http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
		return
	}
	if sumRangesSize(ranges) > totalSize {
		// The total number of bytes in all the ranges
		// is larger than the size of the file by
		// itself, so this is probably an attack, or a
		// dumb client.  Ignore the range request.
		return
	}
	if len(ranges) == 0 {
		return
	}
	if len(ranges) == 1 {
		// RFC 2616, Section 14.16:
		// "When an HTTP message includes the content of a single
		// range (for example, a response to a request for a
		// single range, or to a request for a set of ranges
		// that overlap without any holes), this content is
		// transmitted with a Content-Range header, and a
		// Content-Length header showing the number of bytes
		// actually transferred.
		// ...
		// A response to a request for a single range MUST NOT
		// be sent using the multipart/byteranges media type."
		ra := ranges[0]
		w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
		w.Header().Set("Content-Range", ra.contentRange(totalSize))

		w.WriteHeader(http.StatusPartialContent)
		err = writeFn(bufferedWriter, ra.start, ra.length)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	}

	// process multiple ranges
	for _, ra := range ranges {
		if ra.start > totalSize {
			http.Error(w, "Out of Range", http.StatusRequestedRangeNotSatisfiable)
			return
		}
	}
	sendSize := rangesMIMESize(ranges, mimeType, totalSize)
	pr, pw := io.Pipe()
	mw := multipart.NewWriter(pw)
	w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary())
	sendContent := pr
	defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish.
	go func() {
		for _, ra := range ranges {
			part, e := mw.CreatePart(ra.mimeHeader(mimeType, totalSize))
			if e != nil {
				pw.CloseWithError(e)
				return
			}
			if e = writeFn(part, ra.start, ra.length); e != nil {
				pw.CloseWithError(e)
				return
			}
		}
		mw.Close()
		pw.Close()
	}()
	if w.Header().Get("Content-Encoding") == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10))
	}
	w.WriteHeader(http.StatusPartialContent)
	if _, err := io.CopyN(bufferedWriter, sendContent, sendSize); err != nil {
		http.Error(w, "Internal Error", http.StatusInternalServerError)
		return
	}
}
<commit_msg>logging processRangeRequest errors<commit_after>package weed_server

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
	"io"
	"io/fs"
	"mime/multipart"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/util"

	"github.com/gorilla/mux"
)

var serverStats *stats.ServerStats
var startTime = time.Now()

func init() {
	serverStats = stats.NewServerStats()
	go serverStats.Start()
}

// bodyAllowedForStatus is a copy of http.bodyAllowedForStatus non-exported function.
func bodyAllowedForStatus(status int) bool {
	switch {
	case status >= 100 && status <= 199:
		return false
	case status == http.StatusNoContent:
		return false
	case status == http.StatusNotModified:
		return false
	}
	return true
}

func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) (err error) {
	if !bodyAllowedForStatus(httpStatus) {
		return
	}

	var bytes []byte
	if obj != nil {
		if r.FormValue("pretty") != "" {
			bytes, err = json.MarshalIndent(obj, "", "  ")
		} else {
			bytes, err = json.Marshal(obj)
		}
	}
	if err != nil {
		return
	}

	if httpStatus >= 400 {
		glog.V(0).Infof("response method:%s URL:%s with httpStatus:%d and JSON:%s",
			r.Method, r.URL.String(), httpStatus, string(bytes))
	}

	callback := r.FormValue("callback")
	if callback == "" {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(httpStatus)
		if httpStatus == http.StatusNotModified {
			return
		}
		_, err = w.Write(bytes)
	} else {
		w.Header().Set("Content-Type", "application/javascript")
		w.WriteHeader(httpStatus)
		if httpStatus == http.StatusNotModified {
			return
		}
		if _, err = w.Write([]uint8(callback)); err != nil {
			return
		}
		if _, err = w.Write([]uint8("(")); err != nil {
			return
		}
		fmt.Fprint(w, string(bytes))
		if _, err = w.Write([]uint8(")")); err != nil {
			return
		}
	}

	return
}

// wrapper for writeJson - just logs errors
func writeJsonQuiet(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) {
	if err := writeJson(w, r, httpStatus, obj); err != nil {
		glog.V(0).Infof("error writing JSON status %d: %v", httpStatus, err)
		glog.V(1).Infof("JSON content: %+v", obj)
	}
}
func writeJsonError(w http.ResponseWriter, r *http.Request, httpStatus int, err error) {
	m := make(map[string]interface{})
	m["error"] = err.Error()
	writeJsonQuiet(w, r, httpStatus, m)
}

func debug(params ...interface{}) {
	glog.V(4).Infoln(params...)
}

func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption) {
	m := make(map[string]interface{})
	if r.Method != "POST" {
		writeJsonError(w, r, http.StatusMethodNotAllowed, errors.New("Only submit via POST!"))
		return
	}

	debug("parsing upload file...")
	bytesBuffer := bufPool.Get().(*bytes.Buffer)
	defer bufPool.Put(bytesBuffer)
	pu, pe := needle.ParseUpload(r, 256*1024*1024, bytesBuffer)
	if pe != nil {
		writeJsonError(w, r, http.StatusBadRequest, pe)
		return
	}

	debug("assigning file id for", pu.FileName)
	r.ParseForm()
	count := uint64(1)
	if r.FormValue("count") != "" {
		count, pe = strconv.ParseUint(r.FormValue("count"), 10, 32)
		if pe != nil {
			writeJsonError(w, r, http.StatusBadRequest, pe)
			return
		}
	}
	ar := &operation.VolumeAssignRequest{
		Count:       count,
		DataCenter:  r.FormValue("dataCenter"),
		Rack:        r.FormValue("rack"),
		Replication: r.FormValue("replication"),
		Collection:  r.FormValue("collection"),
		Ttl:         r.FormValue("ttl"),
		DiskType:    r.FormValue("disk"),
	}
	assignResult, ae := operation.Assign(masterFn, grpcDialOption, ar)
	if ae != nil {
		writeJsonError(w, r, http.StatusInternalServerError, ae)
		return
	}

	url := "http://" + assignResult.Url + "/" + assignResult.Fid
	if pu.ModifiedTime != 0 {
		url = url + "?ts=" + strconv.FormatUint(pu.ModifiedTime, 10)
	}

	debug("upload file to store", url)
	uploadOption := &operation.UploadOption{
		UploadUrl:         url,
		Filename:          pu.FileName,
		Cipher:            false,
		IsInputCompressed: pu.IsGzipped,
		MimeType:          pu.MimeType,
		PairMap:           pu.PairMap,
		Jwt:               assignResult.Auth,
	}
	uploadResult, err := operation.UploadData(pu.Data, uploadOption)
	if err != nil {
		writeJsonError(w, r, http.StatusInternalServerError, err)
		return
	}

	m["fileName"] = pu.FileName
	m["fid"] = assignResult.Fid
	m["fileUrl"] = assignResult.PublicUrl + "/" + assignResult.Fid
	m["size"] = pu.OriginalDataSize
	m["eTag"] = uploadResult.ETag
	writeJsonQuiet(w, r, http.StatusCreated, m)
	return
}

func parseURLPath(path string) (vid, fid, filename, ext string, isVolumeIdOnly bool) {
	switch strings.Count(path, "/") {
	case 3:
		parts := strings.Split(path, "/")
		vid, fid, filename = parts[1], parts[2], parts[3]
		ext = filepath.Ext(filename)
	case 2:
		parts := strings.Split(path, "/")
		vid, fid = parts[1], parts[2]
		dotIndex := strings.LastIndex(fid, ".")
		if dotIndex > 0 {
			ext = fid[dotIndex:]
			fid = fid[0:dotIndex]
		}
	default:
		sepIndex := strings.LastIndex(path, "/")
		commaIndex := strings.LastIndex(path[sepIndex:], ",")
		if commaIndex <= 0 {
			vid, isVolumeIdOnly = path[sepIndex+1:], true
			return
		}
		dotIndex := strings.LastIndex(path[sepIndex:], ".")
		vid = path[sepIndex+1 : commaIndex]
		fid = path[commaIndex+1:]
		ext = ""
		if dotIndex > 0 {
			fid = path[commaIndex+1 : dotIndex]
			ext = path[dotIndex:]
		}
	}
	return
}

func statsHealthHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	writeJsonQuiet(w, r, http.StatusOK, m)
}
func statsCounterHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	m["Counters"] = serverStats
	writeJsonQuiet(w, r, http.StatusOK, m)
}

func statsMemoryHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	m["Memory"] = stats.MemStat()
	writeJsonQuiet(w, r, http.StatusOK, m)
}

var StaticFS fs.FS

func handleStaticResources(defaultMux *http.ServeMux) {
	defaultMux.Handle("/favicon.ico", http.FileServer(http.FS(StaticFS)))
	defaultMux.Handle("/seaweedfsstatic/", http.StripPrefix("/seaweedfsstatic", http.FileServer(http.FS(StaticFS))))
}

func handleStaticResources2(r *mux.Router) {
	r.Handle("/favicon.ico", http.FileServer(http.FS(StaticFS)))
	r.PathPrefix("/seaweedfsstatic/").Handler(http.StripPrefix("/seaweedfsstatic", http.FileServer(http.FS(StaticFS))))
}

func adjustPassthroughHeaders(w http.ResponseWriter, r *http.Request, filename string) {
	for header, values := range r.Header {
		if normalizedHeader, ok := s3_constants.PassThroughHeaders[strings.ToLower(header)]; ok {
			w.Header()[normalizedHeader] = values
		}
	}
	adjustHeaderContentDisposition(w, r, filename)
}
func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, filename string) {
	if contentDisposition := w.Header().Get("Content-Disposition"); contentDisposition != "" {
		return
	}
	if filename != "" {
		filename = url.QueryEscape(filename)
		contentDisposition := "inline"
		if r.FormValue("dl") != "" {
			if dl, _ := strconv.ParseBool(r.FormValue("dl")); dl {
				contentDisposition = "attachment"
			}
		}
		w.Header().Set("Content-Disposition", contentDisposition+`; filename="`+fileNameEscaper.Replace(filename)+`"`)
	}
}

func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64) error) {
	rangeReq := r.Header.Get("Range")
	bufferedWriter := bufio.NewWriterSize(w, 128*1024)
	defer bufferedWriter.Flush()

	if rangeReq == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
		if err := writeFn(bufferedWriter, 0, totalSize); err != nil {
			glog.Errorf("processRangeRequest headers: %+v err: %v", w.Header(), err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	}

	//the rest is dealing with partial content request
	//mostly copy from src/pkg/net/http/fs.go
	ranges, err := parseRange(rangeReq, totalSize)
	if err != nil {
		glog.Errorf("processRangeRequest headers: %+v err: %v", w.Header(), err)
		http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
		return
	}
	if sumRangesSize(ranges) > totalSize {
		// The total number of bytes in all the ranges
		// is larger than the size of the file by
		// itself, so this is probably an attack, or a
		// dumb client.  Ignore the range request.
		return
	}
	if len(ranges) == 0 {
		return
	}
	if len(ranges) == 1 {
		// RFC 2616, Section 14.16:
		// "When an HTTP message includes the content of a single
		// range (for example, a response to a request for a
		// single range, or to a request for a set of ranges
		// that overlap without any holes), this content is
		// transmitted with a Content-Range header, and a
		// Content-Length header showing the number of bytes
		// actually transferred.
		// ...
		// A response to a request for a single range MUST NOT
		// be sent using the multipart/byteranges media type."
		ra := ranges[0]
		w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
		w.Header().Set("Content-Range", ra.contentRange(totalSize))

		w.WriteHeader(http.StatusPartialContent)
		err = writeFn(bufferedWriter, ra.start, ra.length)
		if err != nil {
			glog.Errorf("processRangeRequest headers: %+v err: %v", w.Header(), err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	}

	// process multiple ranges
	for _, ra := range ranges {
		if ra.start > totalSize {
			http.Error(w, "Out of Range", http.StatusRequestedRangeNotSatisfiable)
			return
		}
	}
	sendSize := rangesMIMESize(ranges, mimeType, totalSize)
	pr, pw := io.Pipe()
	mw := multipart.NewWriter(pw)
	w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary())
	sendContent := pr
	defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish.
	go func() {
		for _, ra := range ranges {
			part, e := mw.CreatePart(ra.mimeHeader(mimeType, totalSize))
			if e != nil {
				pw.CloseWithError(e)
				return
			}
			if e = writeFn(part, ra.start, ra.length); e != nil {
				pw.CloseWithError(e)
				return
			}
		}
		mw.Close()
		pw.Close()
	}()
	if w.Header().Get("Content-Encoding") == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10))
	}
	w.WriteHeader(http.StatusPartialContent)
	if _, err := io.CopyN(bufferedWriter, sendContent, sendSize); err != nil {
		glog.Errorf("processRangeRequest err: %v", err)
		http.Error(w, "Internal Error", http.StatusInternalServerError)
		return
	}
}
<|endoftext|>


<commit_before>package types

import (
	"time"
)

const (
	AuditActionCreated = "created"
	AuditActionUpdated = "updated"
	AuditActionDeleted = "deleted"

	// Approval specific actions
	AuditActionApprovalApproved = "approved"
	AuditActionApprovalRejected = "rejected"
	AuditActionApprovalExpired  = "expired"
	AuditActionApprovalArchived = "archived"

	// audit specific resource kinds (others are set by
	// providers, ie: deployment, daemonset, helm chart)
	AuditResourceKindApproval = "approval"
	AuditResourceKindWebhook  = "webhook"
)

// AuditLog - audit logs lets users basic things happening in keel such as
// deployment updates and approval actions
type AuditLog struct {
	ID        string    `json:"id" gorm:"primary_key;type:varchar(36)"`
	CreatedAt time.Time `json:"createdAt"`
	UpdatedAt time.Time `json:"updatedAt"`

	AccountID string `json:"accountId"`
	Username  string `json:"username"`
	Email     string `json:"email"`

	// create/delete/update
	Action       string `json:"action"`
	ResourceKind string `json:"resourceKind"` // approval/deployment/daemonset/statefulset/etc...
	Identifier   string `json:"identifier"`

	Message     string `json:"message"`
	Payload     string `json:"payload"` // can be used for bigger messages such as webhook payload
	PayloadType string `json:"payloadType"`

	Metadata JSONB `json:"metadata" gorm:"type:json"`
}

// SetMetadata - set audit log metadata (providers, namespaces)
func (l *AuditLog) SetMetadata(m map[string]string) {
	meta := make(map[string]interface{})
	for key, value := range m {
		meta[key] = value
	}

	l.Metadata = meta
}

// AuditLogQuery - struct used to query audit logs
type AuditLogQuery struct {
	Email    string `json:"email"`
	Username string `json:"username"`
	Order    string `json:"order"` // empty or "desc"
	Limit    int    `json:"limit"`
	Offset   int    `json:"offset"`

	ResourceKindFilter []string `json:"resourceKindFilter"`
}

type AuditLogStatsQuery struct {
}

type AuditLogStats struct {
	Date     string `json:"date"`
	Webhooks int    `json:"webhooks"`
	Approved int    `json:"approved"`
	Rejected int    `json:"rejected"`
	Updates  int    `json:"updates"`
}
<commit_msg>number of days to get<commit_after>package types

import (
	"time"
)

const (
	AuditActionCreated = "created"
	AuditActionUpdated = "updated"
	AuditActionDeleted = "deleted"

	// Approval specific actions
	AuditActionApprovalApproved = "approved"
	AuditActionApprovalRejected = "rejected"
	AuditActionApprovalExpired  = "expired"
	AuditActionApprovalArchived = "archived"

	// audit specific resource kinds (others are set by
	// providers, ie: deployment, daemonset, helm chart)
	AuditResourceKindApproval = "approval"
	AuditResourceKindWebhook  = "webhook"
)

// AuditLog - audit logs lets users basic things happening in keel such as
// deployment updates and approval actions
type AuditLog struct {
	ID        string    `json:"id" gorm:"primary_key;type:varchar(36)"`
	CreatedAt time.Time `json:"createdAt"`
	UpdatedAt time.Time `json:"updatedAt"`

	AccountID string `json:"accountId"`
	Username  string `json:"username"`
	Email     string `json:"email"`

	// create/delete/update
	Action       string `json:"action"`
	ResourceKind string `json:"resourceKind"` // approval/deployment/daemonset/statefulset/etc...
	Identifier   string `json:"identifier"`

	Message     string `json:"message"`
	Payload     string `json:"payload"` // can be used for bigger messages such as webhook payload
	PayloadType string `json:"payloadType"`

	Metadata JSONB `json:"metadata" gorm:"type:json"`
}

// SetMetadata - set audit log metadata (providers, namespaces)
func (l *AuditLog) SetMetadata(m map[string]string) {
	meta := make(map[string]interface{})
	for key, value := range m {
		meta[key] = value
	}

	l.Metadata = meta
}

// AuditLogQuery - struct used to query audit logs
type AuditLogQuery struct {
	Email    string `json:"email"`
	Username string `json:"username"`
	Order    string `json:"order"` // empty or "desc"
	Limit    int    `json:"limit"`
	Offset   int    `json:"offset"`

	ResourceKindFilter []string `json:"resourceKindFilter"`
}

type AuditLogStatsQuery struct {
	Days int
}

type AuditLogStats struct {
	Date     string `json:"date"`
	Webhooks int    `json:"webhooks"`
	Approved int    `json:"approved"`
	Rejected int    `json:"rejected"`
	Updates  int    `json:"updates"`
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package s390x

import (
	"cmd/compile/internal/gc"
	"cmd/internal/obj/s390x"
)

func betypeinit() {
	gc.Widthptr = 8
	gc.Widthint = 8
	gc.Widthreg = 8
}

func Main() {
	gc.Thearch.LinkArch = &s390x.Links390x
	gc.Thearch.REGSP = s390x.REGSP
	gc.Thearch.REGCTXT = s390x.REGCTXT
	gc.Thearch.REGCALLX = s390x.REG_R3
	gc.Thearch.REGCALLX2 = s390x.REG_R4
	gc.Thearch.REGRETURN = s390x.REG_R3
	gc.Thearch.REGMIN = s390x.REG_R0
	gc.Thearch.REGMAX = s390x.REG_R15
	gc.Thearch.FREGMIN = s390x.REG_F0
	gc.Thearch.FREGMAX = s390x.REG_F15
	gc.Thearch.MAXWIDTH = 1 << 50
	gc.Thearch.ReservedRegs = resvd

	gc.Thearch.Betypeinit = betypeinit
	gc.Thearch.Cgen_hmul = cgen_hmul
	gc.Thearch.Cgen_shift = cgen_shift
	gc.Thearch.Clearfat = clearfat
	gc.Thearch.Defframe = defframe
	gc.Thearch.Dodiv = dodiv
	gc.Thearch.Excise = excise
	gc.Thearch.Expandchecks = expandchecks
	gc.Thearch.Getg = getg
	gc.Thearch.Gins = gins
	gc.Thearch.Ginscmp = ginscmp
	gc.Thearch.Ginscon = ginscon
	gc.Thearch.Ginsnop = ginsnop
	gc.Thearch.Gmove = gmove
	gc.Thearch.Peep = peep
	gc.Thearch.Proginfo = proginfo
	gc.Thearch.Regtyp = isReg
	gc.Thearch.Sameaddr = sameaddr
	gc.Thearch.Smallindir = smallindir
	gc.Thearch.Stackaddr = stackaddr
	gc.Thearch.Blockcopy = blockcopy
	gc.Thearch.Sudoaddable = sudoaddable
	gc.Thearch.Sudoclean = sudoclean
	gc.Thearch.Excludedregs = excludedregs
	gc.Thearch.RtoB = RtoB
	gc.Thearch.FtoB = RtoB
	gc.Thearch.BtoR = BtoR
	gc.Thearch.BtoF = BtoF
	gc.Thearch.Optoas = optoas
	gc.Thearch.Doregbits = doregbits
	gc.Thearch.Regnames = regnames

	gc.Main()
	gc.Exit(0)
}
<commit_msg>cmd/compile/internal/s390x: cleanup betypeinit<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package s390x

import (
	"cmd/compile/internal/gc"
	"cmd/internal/obj/s390x"
)

func betypeinit() {
}

func Main() {
	gc.Thearch.LinkArch = &s390x.Links390x
	gc.Thearch.REGSP = s390x.REGSP
	gc.Thearch.REGCTXT = s390x.REGCTXT
	gc.Thearch.REGCALLX = s390x.REG_R3
	gc.Thearch.REGCALLX2 = s390x.REG_R4
	gc.Thearch.REGRETURN = s390x.REG_R3
	gc.Thearch.REGMIN = s390x.REG_R0
	gc.Thearch.REGMAX = s390x.REG_R15
	gc.Thearch.FREGMIN = s390x.REG_F0
	gc.Thearch.FREGMAX = s390x.REG_F15
	gc.Thearch.MAXWIDTH = 1 << 50
	gc.Thearch.ReservedRegs = resvd

	gc.Thearch.Betypeinit = betypeinit
	gc.Thearch.Cgen_hmul = cgen_hmul
	gc.Thearch.Cgen_shift = cgen_shift
	gc.Thearch.Clearfat = clearfat
	gc.Thearch.Defframe = defframe
	gc.Thearch.Dodiv = dodiv
	gc.Thearch.Excise = excise
	gc.Thearch.Expandchecks = expandchecks
	gc.Thearch.Getg = getg
	gc.Thearch.Gins = gins
	gc.Thearch.Ginscmp = ginscmp
	gc.Thearch.Ginscon = ginscon
	gc.Thearch.Ginsnop = ginsnop
	gc.Thearch.Gmove = gmove
	gc.Thearch.Peep = peep
	gc.Thearch.Proginfo = proginfo
	gc.Thearch.Regtyp = isReg
	gc.Thearch.Sameaddr = sameaddr
	gc.Thearch.Smallindir = smallindir
	gc.Thearch.Stackaddr = stackaddr
	gc.Thearch.Blockcopy = blockcopy
	gc.Thearch.Sudoaddable = sudoaddable
	gc.Thearch.Sudoclean = sudoclean
	gc.Thearch.Excludedregs = excludedregs
	gc.Thearch.RtoB = RtoB
	gc.Thearch.FtoB = RtoB
	gc.Thearch.BtoR = BtoR
	gc.Thearch.BtoF = BtoF
	gc.Thearch.Optoas = optoas
	gc.Thearch.Doregbits = doregbits
	gc.Thearch.Regnames = regnames

	gc.Main()
	gc.Exit(0)
}
<|endoftext|>


<commit_before>// Copyright 2016 EF CTX. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"regexp"
	"strconv"

	"github.com/cezarsa/form"
	"github.com/gorilla/mux"
	"github.com/tsuru/tsuru/api"
)

var tsuruServer = newFakeTsuruServer()

// fakeTsuruServer provides a non-thread-safe, partial implementation of the
// tsuru API.
type fakeTsuruServer struct {
	apps    []app
	envVars map[string][]envVar
	deploys map[string][]deploy
	server  *httptest.Server
	router  *mux.Router
}

func newFakeTsuruServer() *fakeTsuruServer {
	var s fakeTsuruServer
	s.buildRouter()
	s.server = httptest.NewServer(s.router)
	s.reset()
	return &s
}

func (s *fakeTsuruServer) buildRouter() {
	s.router = mux.NewRouter()
	r := s.router.PathPrefix("/1.0").Subrouter()
	r.HandleFunc("/apps", func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "POST":
			s.createApp(w, r)
		case "GET":
			s.listApps(w, r)
		default:
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		}
	})
	r.HandleFunc("/apps/{appname}", func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "PUT":
			s.updateApp(w, r)
		case "GET":
			s.getApp(w, r)
		case "DELETE":
			s.deleteApp(w, r)
		default:
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		}
	})
	r.HandleFunc("/apps/{appname}/env", func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "POST":
			s.setEnvs(w, r)
		case "GET":
			s.getEnvs(w, r)
		case "DELETE":
			s.unsetEnvs(w, r)
		default:
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		}
	})
	r.HandleFunc("/deploys", s.listDeploys)
	r.HandleFunc("/apps/{appname}/cname", s.addCName)
}

func (s *fakeTsuruServer) createApp(w http.ResponseWriter, r *http.Request) {
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	var opts createAppOptions
	form.DecodeValues(&opts, r.Form)
	if opts.Name == "" || opts.Platform == "" {
		http.Error(w, "invalid params", http.StatusBadRequest)
		return
	}
	_, index := s.findApp(opts.Name)
	if index > -1 {
		http.Error(w, "app already exists", http.StatusConflict)
		return
	}
	repositoryURL := fmt.Sprintf("git@gandalf.example.com:%s.git", opts.Name)
	s.apps = append(s.apps, app{
		Name:        opts.Name,
		Platform:    opts.Platform,
		Description: opts.Description,
		TeamOwner:   opts.Team,
		Teams:       []string{opts.Team},
		Plan: struct {
			Name string `json:"name"`
		}{Name: opts.Plan},
		Pool:          opts.Pool,
		Owner:         "user@example.com",
		RepositoryURL: repositoryURL,
		Addr:          opts.Name + ".tsuru.example.com",
	})
	s.writeJSON(w, map[string]string{"repository_url": repositoryURL})
}

func (s *fakeTsuruServer) listApps(w http.ResponseWriter, r *http.Request) {
	nameRegexp, err := regexp.Compile(r.URL.Query().Get("name"))
	if err != nil {
		http.Error(w, "invalid name regexp", http.StatusBadRequest)
		return
	}
	var apps []app
	for _, a := range s.apps {
		if nameRegexp.MatchString(a.Name) {
			apps = append(apps, a)
		}
	}
	if len(apps) == 0 {
		w.WriteHeader(http.StatusNoContent)
		return
	}
	s.writeJSON(w, apps)
}

func (s *fakeTsuruServer) updateApp(w http.ResponseWriter, r *http.Request) {
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	var opts createAppOptions
	form.DecodeValues(&opts, r.Form)
	a, index := s.findApp(mux.Vars(r)["appname"])
	if index < 0 {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	if opts.Team != "" {
		a.TeamOwner = opts.Team
	}
	if opts.Description != "" {
		a.Description = opts.Description
	}
	if opts.Plan != "" {
		a.Plan.Name = opts.Plan
	}
	if opts.Pool != "" {
		a.Plan.Name = opts.Pool
	}
	s.apps[index] = a
}

func (s *fakeTsuruServer) getApp(w http.ResponseWriter, r *http.Request) {
	a, index := s.findApp(mux.Vars(r)["appname"])
	if index < 0 {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	s.writeJSON(w, a)
}

func (s *fakeTsuruServer) deleteApp(w http.ResponseWriter, r *http.Request) {
	_, index := s.findApp(mux.Vars(r)["appname"])
	if index < 0 {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	s.apps[index] = s.apps[len(s.apps)-1]
	s.apps = s.apps[:len(s.apps)-1]
	w.WriteHeader(http.StatusOK)
}

func (s *fakeTsuruServer) setEnvs(w http.ResponseWriter, r *http.Request) {
	appName := mux.Vars(r)["appname"]
	envs, ok := s.envVars[appName]
	if !ok {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	var evars api.Envs
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	form.DecodeValues(&evars, r.Form)
	for _, e := range evars.Envs {
		envs = append(envs, envVar{
			Name:   e.Name,
			Value:  e.Value,
			Public: !evars.Private,
		})
	}
	s.envVars[appName] = envs
	w.WriteHeader(http.StatusOK)
}

func (s *fakeTsuruServer) getEnvs(w http.ResponseWriter, r *http.Request) {
	appName := mux.Vars(r)["appname"]
	envs, ok := s.envVars[appName]
	if !ok {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	s.writeJSON(w, envs)
}

func (s *fakeTsuruServer) unsetEnvs(w http.ResponseWriter, r *http.Request) {
	appName := mux.Vars(r)["appname"]
	envs, ok := s.envVars[appName]
	if !ok {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	envNames := r.URL.Query()["env"]
	var newEnvs []envVar
	for _, e := range envs {
		var exclude bool
		for _, envName := range envNames {
			if e.Name == envName && e.Public {
				exclude = true
				break
			}
		}
		if !exclude {
			newEnvs = append(newEnvs, e)
		}
	}
	s.envVars[appName] = newEnvs
	w.WriteHeader(http.StatusOK)
}

func (s *fakeTsuruServer) listDeploys(w http.ResponseWriter, r *http.Request) {
	appName := r.URL.Query().Get("app")
	limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
	if appName == "" {
		http.Error(w, "missing app name in querystring", http.StatusBadRequest)
		return
	}
	deployList, ok := s.deploys[appName]
	if !ok {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	if len(deployList) == 0 {
		w.WriteHeader(http.StatusNoContent)
		return
	}
	if limit == 0 || limit > len(deployList) {
		limit = len(deployList)
	}
	var deploys []deploy
	for i := len(deployList) - 1; i >= len(deployList)-limit; i-- {
		deploys = append(deploys, deployList[i])
	}
	s.writeJSON(w, deploys)
}

func (s *fakeTsuruServer) addCName(w http.ResponseWriter, r *http.Request) {
	cName := r.FormValue("cname")
	if cName == "" {
		http.Error(w, "missing param", http.StatusBadRequest)
		return
	}
	a, index := s.findApp(mux.Vars(r)["appname"])
	if index < 0 {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	for _, name := range a.CName {
		if name == cName {
			http.Error(w, "duplicate cname", http.StatusConflict)
			return
		}
	}
	a.CName = append(a.CName, cName)
	s.apps[index] = a
}

func (s *fakeTsuruServer) writeJSON(w http.ResponseWriter, data interface{}) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)
	json.NewEncoder(w).Encode(data)
}

func (s *fakeTsuruServer) findApp(name string) (a app, index int) {
	index = -1
	for i := range s.apps {
		if s.apps[i].Name == name {
			a = s.apps[i]
			index = i
			break
		}
	}
	return a, index
}

func (s *fakeTsuruServer) stop() {
	s.server.Close()
}

func (s *fakeTsuruServer) url() string {
	return s.server.URL
}

func (s *fakeTsuruServer) reset() {
	s.apps = nil
	s.envVars = make(map[string][]envVar)
	s.deploys = make(map[string][]deploy)
}
<commit_msg>Remove unused stop method from fake tsuru server<commit_after>// Copyright 2016 EF CTX. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"regexp"
	"strconv"

	"github.com/cezarsa/form"
	"github.com/gorilla/mux"
	"github.com/tsuru/tsuru/api"
)

var tsuruServer = newFakeTsuruServer()

// fakeTsuruServer provides a non-thread-safe, partial implementation of the
// tsuru API.
type fakeTsuruServer struct {
	apps    []app
	envVars map[string][]envVar
	deploys map[string][]deploy
	server  *httptest.Server
	router  *mux.Router
}

func newFakeTsuruServer() *fakeTsuruServer {
	var s fakeTsuruServer
	s.buildRouter()
	s.server = httptest.NewServer(s.router)
	s.reset()
	return &s
}

func (s *fakeTsuruServer) buildRouter() {
	s.router = mux.NewRouter()
	r := s.router.PathPrefix("/1.0").Subrouter()
	r.HandleFunc("/apps", func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "POST":
			s.createApp(w, r)
		case "GET":
			s.listApps(w, r)
		default:
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		}
	})
	r.HandleFunc("/apps/{appname}", func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "PUT":
			s.updateApp(w, r)
		case "GET":
			s.getApp(w, r)
		case "DELETE":
			s.deleteApp(w, r)
		default:
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		}
	})
	r.HandleFunc("/apps/{appname}/env", func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "POST":
			s.setEnvs(w, r)
		case "GET":
			s.getEnvs(w, r)
		case "DELETE":
			s.unsetEnvs(w, r)
		default:
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		}
	})
	r.HandleFunc("/deploys", s.listDeploys)
	r.HandleFunc("/apps/{appname}/cname", s.addCName)
}

func (s *fakeTsuruServer) createApp(w http.ResponseWriter, r *http.Request) {
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	var opts createAppOptions
	form.DecodeValues(&opts, r.Form)
	if opts.Name == "" || opts.Platform == "" {
		http.Error(w, "invalid params", http.StatusBadRequest)
		return
	}
	_, index := s.findApp(opts.Name)
	if index > -1 {
		http.Error(w, "app already exists", http.StatusConflict)
		return
	}
	repositoryURL := fmt.Sprintf("git@gandalf.example.com:%s.git", opts.Name)
	s.apps = append(s.apps, app{
		Name:        opts.Name,
		Platform:    opts.Platform,
		Description: opts.Description,
		TeamOwner:   opts.Team,
		Teams:       []string{opts.Team},
		Plan: struct {
			Name string `json:"name"`
		}{Name: opts.Plan},
		Pool:          opts.Pool,
		Owner:         "user@example.com",
		RepositoryURL: repositoryURL,
		Addr:          opts.Name + ".tsuru.example.com",
	})
	s.writeJSON(w, map[string]string{"repository_url": repositoryURL})
}

func (s *fakeTsuruServer) listApps(w http.ResponseWriter, r *http.Request) {
	nameRegexp, err := regexp.Compile(r.URL.Query().Get("name"))
	if err != nil {
		http.Error(w, "invalid name regexp", http.StatusBadRequest)
		return
	}
	var apps []app
	for _, a := range s.apps {
		if nameRegexp.MatchString(a.Name) {
			apps = append(apps, a)
		}
	}
	if len(apps) == 0 {
		w.WriteHeader(http.StatusNoContent)
		return
	}
	s.writeJSON(w, apps)
}

func (s *fakeTsuruServer) updateApp(w http.ResponseWriter, r *http.Request) {
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	var opts createAppOptions
	form.DecodeValues(&opts, r.Form)
	a, index := s.findApp(mux.Vars(r)["appname"])
	if index < 0 {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	if opts.Team != "" {
		a.TeamOwner = opts.Team
	}
	if opts.Description != "" {
		a.Description = opts.Description
	}
	if opts.Plan != "" {
		a.Plan.Name = opts.Plan
	}
	if opts.Pool != "" {
		a.Plan.Name = opts.Pool
	}
	s.apps[index] = a
}

func (s *fakeTsuruServer) getApp(w http.ResponseWriter, r *http.Request) {
	a, index := s.findApp(mux.Vars(r)["appname"])
	if index < 0 {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	s.writeJSON(w, a)
}

func (s *fakeTsuruServer) deleteApp(w http.ResponseWriter, r *http.Request) {
	_, index := s.findApp(mux.Vars(r)["appname"])
	if index < 0 {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	s.apps[index] = s.apps[len(s.apps)-1]
	s.apps = s.apps[:len(s.apps)-1]
	w.WriteHeader(http.StatusOK)
}

func (s *fakeTsuruServer) setEnvs(w http.ResponseWriter, r *http.Request) {
	appName := mux.Vars(r)["appname"]
	envs, ok := s.envVars[appName]
	if !ok {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	var evars api.Envs
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	form.DecodeValues(&evars, r.Form)
	for _, e := range evars.Envs {
		envs = append(envs, envVar{
			Name:   e.Name,
			Value:  e.Value,
			Public: !evars.Private,
		})
	}
	s.envVars[appName] = envs
	w.WriteHeader(http.StatusOK)
}

func (s *fakeTsuruServer) getEnvs(w http.ResponseWriter, r *http.Request) {
	appName := mux.Vars(r)["appname"]
	envs, ok := s.envVars[appName]
	if !ok {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	s.writeJSON(w, envs)
}

func (s *fakeTsuruServer) unsetEnvs(w http.ResponseWriter, r *http.Request) {
	appName := mux.Vars(r)["appname"]
	envs, ok := s.envVars[appName]
	if !ok {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	envNames := r.URL.Query()["env"]
	var newEnvs []envVar
	for _, e := range envs {
		var exclude bool
		for _, envName := range envNames {
			if e.Name == envName && e.Public {
				exclude = true
				break
			}
		}
		if !exclude {
			newEnvs = append(newEnvs, e)
		}
	}
	s.envVars[appName] = newEnvs
	w.WriteHeader(http.StatusOK)
}

func (s *fakeTsuruServer) listDeploys(w http.ResponseWriter, r *http.Request) {
	appName := r.URL.Query().Get("app")
	limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
	if appName == "" {
		http.Error(w, "missing app name in querystring", http.StatusBadRequest)
		return
	}
	deployList, ok := s.deploys[appName]
	if !ok {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	if len(deployList) == 0 {
		w.WriteHeader(http.StatusNoContent)
		return
	}
	if limit == 0 || limit > len(deployList) {
		limit = len(deployList)
	}
	var deploys []deploy
	for i := len(deployList) - 1; i >= len(deployList)-limit; i-- {
		deploys = append(deploys, deployList[i])
	}
	s.writeJSON(w, deploys)
}

func (s *fakeTsuruServer) addCName(w http.ResponseWriter, r *http.Request) {
	cName := r.FormValue("cname")
	if cName == "" {
		http.Error(w, "missing param", http.StatusBadRequest)
		return
	}
	a, index := s.findApp(mux.Vars(r)["appname"])
	if index < 0 {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	for _, name := range a.CName {
		if name == cName {
			http.Error(w, "duplicate cname", http.StatusConflict)
			return
		}
	}
	a.CName = append(a.CName, cName)
	s.apps[index] = a
}

func (s *fakeTsuruServer) writeJSON(w http.ResponseWriter, data interface{}) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)
	json.NewEncoder(w).Encode(data)
}

func (s *fakeTsuruServer) findApp(name string) (a app, index int) {
	index = -1
	for i := range s.apps {
		if s.apps[i].Name == name {
			a = s.apps[i]
			index = i
			break
		}
	}
	return a, index
}

func (s *fakeTsuruServer) url() string {
	return s.server.URL
}

func (s *fakeTsuruServer) reset() {
	s.apps = nil
	s.envVars = make(map[string][]envVar)
	s.deploys = make(map[string][]deploy)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io/ioutil"
	"path/filepath"
)

const NUMSTEPS = 16

// RESOURCES:
// read file examples: https://gobyexample.com/reading-files

func checkError(err error) {
	if err != nil {
		fmt.Println("error: ", err)
	}
}

func main() {
	// get list of file names at target directory
	inDataDirectory := "fixtures"
	files, err := ioutil.ReadDir(inDataDirectory)
	checkError(err)

	// clean list names
	// - remove .DS_Store
	var fileList []string
	for _, file := range files {
		if file.Name() != ".DS_Store" {
			fileList = append(fileList, file.Name())
		}
	}

	var fileLen int
	var spliceHeader [6]byte   // 6
	var trackSize int64        // 8
	var versionString [32]byte // 32
	var tempo float32          // 4

	// inspect data contents
	var id uint8
	var nameLength int32
	for _, fileName := range fileList {
		// open file
		fullPath := filepath.Join(inDataDirectory, fileName)
		fileContents, err := ioutil.ReadFile(fullPath)
		checkError(err)
		//fmt.Printf("%s\n", hex.Dump(fileContents))
		buf := bytes.NewReader(fileContents)
		fileLen = len(fileContents)

		// Header: SPLICE
		err = binary.Read(buf, binary.BigEndian, &spliceHeader)
		checkError(err)
		fileLen -= binary.Size(spliceHeader)

		// Header: track size is big endian
		err = binary.Read(buf, binary.BigEndian, &trackSize)
		checkError(err)
		fileLen -= binary.Size(trackSize)

		// Header: version
		err = binary.Read(buf, binary.BigEndian, &versionString)
		checkError(err)
		fileLen -= binary.Size(versionString)

		// Header: tempo
		// NOTE: tempo is little Endian?
		err = binary.Read(buf, binary.LittleEndian, &tempo)
		checkError(err)
		fileLen -= binary.Size(tempo)

		// Read in body. id+name + 16 steps
		// TODO: Issue is with pattern 5...
		for fileLen > 0 {
			// ID
			err = binary.Read(buf, binary.BigEndian, &id)
			checkError(err)
			fileLen -= binary.Size(id)

			// Length of instrument name
			err = binary.Read(buf, binary.BigEndian, &nameLength)
			checkError(err)
			fileLen -= binary.Size(nameLength)

			// name of instrument
			nameBuf := make([]byte, nameLength)
			err = binary.Read(buf, binary.LittleEndian, &nameBuf)
			checkError(err)
			fileLen -= binary.Size(nameBuf)

			// steps
			stepBuf := make([]byte, NUMSTEPS)
			err = binary.Read(buf, binary.LittleEndian, &stepBuf)
			checkError(err)
			fileLen -= binary.Size(stepBuf)

		}

	}

}
<commit_msg>trying to use structs<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io/ioutil"
	"path/filepath"
)

const NUMSTEPS = 16

type instrument struct {
	instrumentName []byte
	instrumentID   uint8
	steps          []byte
}

type track struct {
	fileLen       int
	spliceHeader  [6]byte  // 6
	trackSize     int64    // 8
	versionString [32]byte // 32
	tempo         float32  // 4
	instruments   []instrument
}

// RESOURCES:
// read file examples: https://gobyexample.com/reading-files

// func printTrackFormat(curTrack track) {
// 	fmt.Println("%v\n", track.spliceHeader)
// 	fmt.Println("%v\n", track.versionString)
// }

func checkError(err error) {
	if err != nil {
		fmt.Println("error: ", err)
	}
}

func main() {
	var tracks []track
	// get list of file names at target directory
	inDataDirectory := "fixtures"
	files, err := ioutil.ReadDir(inDataDirectory)
	checkError(err)

	// clean list names
	// - remove .DS_Store
	var fileList []string
	for _, file := range files {
		if file.Name() != ".DS_Store" {
			fileList = append(fileList, file.Name())
		}
	}

	var fileLen int
	var spliceHeader [6]byte   // 6
	var trackSize int64        // 8
	var versionString [32]byte // 32
	var tempo float32          // 4

	// inspect data contents
	var id uint8
	var nameLength int32
	for _, fileName := range fileList {
		// open file
		fullPath := filepath.Join(inDataDirectory, fileName)
		fileContents, err := ioutil.ReadFile(fullPath)
		checkError(err)
		newTrack := track{}
		//fmt.Printf("%s\n", hex.Dump(fileContents))
		buf := bytes.NewReader(fileContents)
		fileLen = len(fileContents)
		// NOTE: this will need to be looked at
		newTrack.trackSize = int64(fileLen)

		// Header: SPLICE
		err = binary.Read(buf, binary.BigEndian, &spliceHeader)
		checkError(err)
		fileLen -= binary.Size(spliceHeader)
		newTrack.spliceHeader = spliceHeader

		// Header: track size is big endian
		err = binary.Read(buf, binary.BigEndian, &trackSize)
		checkError(err)
		fileLen -= binary.Size(trackSize)
		newTrack.trackSize = trackSize

		// Header: version
		err = binary.Read(buf, binary.BigEndian, &versionString)
		checkError(err)
		fileLen -= binary.Size(versionString)
		newTrack.versionString = versionString

		// Header: tempo
		// NOTE: tempo is little Endian?
		err = binary.Read(buf, binary.LittleEndian, &tempo)
		checkError(err)
		fileLen -= binary.Size(tempo)
		newTrack.tempo = tempo

		// Read in body. id+name + 16 steps
		// TODO: Issue is with pattern 5...
		for fileLen > 0 {
			curInstrument := instrument{}
			// ID
			err = binary.Read(buf, binary.BigEndian, &id)
			checkError(err)
			fileLen -= binary.Size(id)
			curInstrument.instrumentID = id

			// Length of instrument name
			err = binary.Read(buf, binary.BigEndian, &nameLength)
			checkError(err)
			fileLen -= binary.Size(nameLength)

			// name of instrument
			nameBuf := make([]byte, nameLength)
			err = binary.Read(buf, binary.LittleEndian, &nameBuf)
			checkError(err)
			fileLen -= binary.Size(nameBuf)
			curInstrument.instrumentName = nameBuf

			// steps
			stepBuf := make([]byte, NUMSTEPS)
			err = binary.Read(buf, binary.LittleEndian, &stepBuf)
			checkError(err)
			fileLen -= binary.Size(stepBuf)
			curInstrument.steps = stepBuf
			newTrack.instruments = append(newTrack.instruments, curInstrument)
		}
		tracks = append(tracks, newTrack)

	}
	fmt.Println(tracks)

}
<|endoftext|>


<commit_before>// Package api provides rest-like server
package api

import (
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/didip/tollbooth"
	"github.com/didip/tollbooth_chi"
	"github.com/go-chi/chi/v5"
	"github.com/go-chi/chi/v5/middleware"
	"github.com/go-chi/render"
	"github.com/go-pkgz/lcw"
	log "github.com/go-pkgz/lgr"
	"github.com/go-pkgz/rest"
	"github.com/go-pkgz/rest/logger"

	"github.com/umputun/feed-master/app/feed"
	"github.com/umputun/feed-master/app/proc"
)

// Server provides HTTP API
type Server struct {
	Version string
	Conf    proc.Conf
	Store   *proc.BoltDB

	httpServer *http.Server
	cache      lcw.LoadingCache
}

// Run starts http server for API with all routes
func (s *Server) Run(port int) {
	var err error
	if s.cache, err = lcw.NewExpirableCache(lcw.TTL(time.Minute*5), lcw.MaxCacheSize(10*1024*1024)); err != nil {
		log.Printf("[PANIC] failed to make loading cache, %v", err)
		return
	}

	router := chi.NewRouter()
	router.Use(middleware.RealIP, rest.Recoverer(log.Default()))
	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
	router.Use(rest.AppInfo("feed-master", "umputun", s.Version), rest.Ping)
	router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))

	s.httpServer = &http.Server{
		Addr:              fmt.Sprintf(":%d", port),
		Handler:           router,
		ReadHeaderTimeout: 5 * time.Second,
		WriteTimeout:      30 * time.Second,
		IdleTimeout:       30 * time.Second,
	}

	router.Group(func(rimg chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"))
		rimg.Use(l.Handler)
		rimg.Get("/images/{name}", s.getImageCtrl)
		rimg.Get("/image/{name}", s.getImageCtrl)
		rimg.Head("/image/{name}", s.getImageHeadCtrl)
		rimg.Head("/images/{name}", s.getImageHeadCtrl)
	})

	router.Group(func(rrss chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"))
		rrss.Use(l.Handler)
		rrss.Get("/rss/{name}", s.getFeedCtrl)
		rrss.Get("/list", s.getListCtrl)
		rrss.Get("/feed/{name}", s.getFeedPageCtrl)
	})

	s.addFileServer(router, "/static", http.Dir(filepath.Join("webapp", "static")))
	err = s.httpServer.ListenAndServe()
	log.Printf("[WARN] http server terminated, %s", err)
}

// GET /rss/{name} - returns rss for given feeds set
func (s *Server) getFeedCtrl(w http.ResponseWriter, r *http.Request) {
	feedName := chi.URLParam(r, "name")
	items, err := s.Store.Load(feedName, s.Conf.System.MaxTotal, true)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest, err, "failed to get feed")
		return
	}

	rss := feed.Rss2{
		Version:       "2.0",
		ItemList:      items,
		Title:         s.Conf.Feeds[feedName].Title,
		Description:   s.Conf.Feeds[feedName].Description,
		Language:      s.Conf.Feeds[feedName].Language,
		Link:          s.Conf.Feeds[feedName].Link,
		PubDate:       items[0].PubDate,
		LastBuildDate: time.Now().Format(time.RFC822Z),
	}

	// replace link to UI page
	if s.Conf.System.BaseURL != "" {
		rss.Link = s.Conf.System.BaseURL + "/feed/" + feedName
	}

	b, err := xml.MarshalIndent(&rss, "", "  ")
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to marshal rss")
		return
	}
	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	_, _ = fmt.Fprintf(w, "%s", string(b))
}

// GET /image/{name}
func (s *Server) getImageCtrl(w http.ResponseWriter, r *http.Request) {
	fm := chi.URLParam(r, "name")
	fm = strings.TrimRight(fm, ".png")
	feedConf, found := s.Conf.Feeds[fm]
	if !found {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			errors.New("image "+chi.URLParam(r, "name")+" not found"), "failed to load image")
		return
	}

	b, err := ioutil.ReadFile(feedConf.Image)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			errors.New("can't read  "+chi.URLParam(r, "name")), "failed to read image")
		return
	}
	w.Header().Set("Content-Type", "image/png")
	if _, err := w.Write(b); err != nil {
		log.Printf("[WARN] failed to send image, %s", err)
	}
}

// HEAD /image/{name}
func (s *Server) getImageHeadCtrl(w http.ResponseWriter, r *http.Request) {
	fm := chi.URLParam(r, "name")
	fm = strings.TrimRight(fm, ".png")
	feedConf, found := s.Conf.Feeds[fm]
	if !found {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	info, err := os.Stat(feedConf.Image)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	w.Header().Set("Content-Type", "image/png")
	w.Header().Set("Content-Length", strconv.Itoa(int(info.Size())))
	w.WriteHeader(http.StatusOK)
}

// GET /list - returns feed's image
func (s *Server) getListCtrl(w http.ResponseWriter, r *http.Request) {
	buckets, err := s.Store.Buckets()
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to read list")
		return
	}
	render.JSON(w, r, buckets)
}
<commit_msg>fix trim logic<commit_after>// Package api provides rest-like server
package api

import (
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/didip/tollbooth"
	"github.com/didip/tollbooth_chi"
	"github.com/go-chi/chi/v5"
	"github.com/go-chi/chi/v5/middleware"
	"github.com/go-chi/render"
	"github.com/go-pkgz/lcw"
	log "github.com/go-pkgz/lgr"
	"github.com/go-pkgz/rest"
	"github.com/go-pkgz/rest/logger"

	"github.com/umputun/feed-master/app/feed"
	"github.com/umputun/feed-master/app/proc"
)

// Server provides HTTP API
type Server struct {
	Version string
	Conf    proc.Conf
	Store   *proc.BoltDB

	httpServer *http.Server
	cache      lcw.LoadingCache
}

// Run starts http server for API with all routes
func (s *Server) Run(port int) {
	var err error
	if s.cache, err = lcw.NewExpirableCache(lcw.TTL(time.Minute*5), lcw.MaxCacheSize(10*1024*1024)); err != nil {
		log.Printf("[PANIC] failed to make loading cache, %v", err)
		return
	}

	router := chi.NewRouter()
	router.Use(middleware.RealIP, rest.Recoverer(log.Default()))
	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
	router.Use(rest.AppInfo("feed-master", "umputun", s.Version), rest.Ping)
	router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))

	s.httpServer = &http.Server{
		Addr:              fmt.Sprintf(":%d", port),
		Handler:           router,
		ReadHeaderTimeout: 5 * time.Second,
		WriteTimeout:      30 * time.Second,
		IdleTimeout:       30 * time.Second,
	}

	router.Group(func(rimg chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"))
		rimg.Use(l.Handler)
		rimg.Get("/images/{name}", s.getImageCtrl)
		rimg.Get("/image/{name}", s.getImageCtrl)
		rimg.Head("/image/{name}", s.getImageHeadCtrl)
		rimg.Head("/images/{name}", s.getImageHeadCtrl)
	})

	router.Group(func(rrss chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"))
		rrss.Use(l.Handler)
		rrss.Get("/rss/{name}", s.getFeedCtrl)
		rrss.Get("/list", s.getListCtrl)
		rrss.Get("/feed/{name}", s.getFeedPageCtrl)
	})

	s.addFileServer(router, "/static", http.Dir(filepath.Join("webapp", "static")))
	err = s.httpServer.ListenAndServe()
	log.Printf("[WARN] http server terminated, %s", err)
}

// GET /rss/{name} - returns rss for given feeds set
func (s *Server) getFeedCtrl(w http.ResponseWriter, r *http.Request) {
	feedName := chi.URLParam(r, "name")
	items, err := s.Store.Load(feedName, s.Conf.System.MaxTotal, true)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest, err, "failed to get feed")
		return
	}

	rss := feed.Rss2{
		Version:       "2.0",
		ItemList:      items,
		Title:         s.Conf.Feeds[feedName].Title,
		Description:   s.Conf.Feeds[feedName].Description,
		Language:      s.Conf.Feeds[feedName].Language,
		Link:          s.Conf.Feeds[feedName].Link,
		PubDate:       items[0].PubDate,
		LastBuildDate: time.Now().Format(time.RFC822Z),
	}

	// replace link to UI page
	if s.Conf.System.BaseURL != "" {
		rss.Link = s.Conf.System.BaseURL + "/feed/" + feedName
	}

	b, err := xml.MarshalIndent(&rss, "", "  ")
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to marshal rss")
		return
	}
	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	_, _ = fmt.Fprintf(w, "%s", string(b))
}

// GET /image/{name}
func (s *Server) getImageCtrl(w http.ResponseWriter, r *http.Request) {
	fm := chi.URLParam(r, "name")
	fm = strings.TrimSuffix(fm, ".png")
	feedConf, found := s.Conf.Feeds[fm]
	if !found {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			fmt.Errorf("image %s not found", fm), "failed to load image")
		return
	}

	b, err := ioutil.ReadFile(feedConf.Image)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			errors.New("can't read  "+chi.URLParam(r, "name")), "failed to read image")
		return
	}
	w.Header().Set("Content-Type", "image/png")
	if _, err := w.Write(b); err != nil {
		log.Printf("[WARN] failed to send image, %s", err)
	}
}

// HEAD /image/{name}
func (s *Server) getImageHeadCtrl(w http.ResponseWriter, r *http.Request) {
	fm := chi.URLParam(r, "name")
	fm = strings.TrimSuffix(fm, ".png")
	feedConf, found := s.Conf.Feeds[fm]
	if !found {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	info, err := os.Stat(feedConf.Image)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	w.Header().Set("Content-Type", "image/png")
	w.Header().Set("Content-Length", strconv.Itoa(int(info.Size())))
	w.WriteHeader(http.StatusOK)
}

// GET /list - returns feed's image
func (s *Server) getListCtrl(w http.ResponseWriter, r *http.Request) {
	buckets, err := s.Store.Buckets()
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to read list")
		return
	}
	render.JSON(w, r, buckets)
}
<|endoftext|>