<commit_before>package mdb

import (
	"fmt"
	"log"
	"testing"
	"time"

	"gopkg.in/mgo.v2"
	"gopkg.in/ory-am/dockertest.v2"
)

func TestNew(t *testing.T) {
	log.SetFlags(log.Lshortfile)

	// COMMON ERRORS SCENARIO
	// prefix session empty
	{
		expected := "prefix value is empty"
		err := New("", nil, -1, -1)
		if err == nil {
			t.Fail()
		}
		if err.Error() != expected {
			log.Printf("err [%s]", err)
			t.Fail()
		}
	}
	// dial info is nil
	{
		expected := "dial info is empty"
		err := New("x", nil, -1, -1)
		if err == nil {
			t.Fail()
		}
		if err.Error() != expected {
			log.Printf("err [%s]", err)
			t.Fail()
		}
	}
	// fail connection.
	{
		expected := "no reachable servers"
		di := &mgo.DialInfo{
			Addrs:    []string{"localhost:27017"},
			Database: "test",
			Timeout:  time.Millisecond,
		}
		err := New("x", di, 1, 1)
		if err.Error() != expected {
			log.Printf("err [%s]", err)
			t.Fail()
		}
	}
	// invalid timeout
	{
		expected := "invalid timeout duration"
		di := &mgo.DialInfo{
			Addrs:    []string{},
			Database: "test",
			Timeout:  11 * time.Second,
		}
		err := New("yy", di, 1, 1)
		if err.Error() != expected {
			log.Printf("err [%s]", err)
			t.Fail()
		}
	}
	// already inited
	{
		c, ip, port, err := dockertest.SetupMongoContainer()
		if err != nil {
			log.Printf("err [%s]", err)
			t.Fail()
		}

		expected := "session already done"
		di := &mgo.DialInfo{
			Addrs:    []string{fmt.Sprintf("%v:%v", ip, port)},
			Database: "test",
			Timeout:  time.Second,
		}
		_ = New("yy", di, 1, 1)
		err = New("yy", di, 1, 1)
		if err.Error() != expected {
			log.Printf("err [%s]", err)
			t.Fail()
		}

		c.KillRemove()
	}
	// bad jobq workers size
	{
		c, ip, port, err := dockertest.SetupMongoContainer()
		if err != nil {
			log.Printf("err [%s]", err)
			t.Fail()
		}

		expected := "invalid worker size"
		di := &mgo.DialInfo{
			Addrs:    []string{fmt.Sprintf("%v:%v", ip, port)},
			Database: "test",
			Timeout:  time.Second,
		}
		err = New("x100", di, -1, 1)
		if err.Error() != expected {
			log.Printf("err [%s]", err)
			t.Fail()
		}
		c.KillRemove()
	}
	// bad jobq queue size
	{
		c, ip, port, err := dockertest.SetupMongoContainer()
		if err != nil {
			log.Printf("err [%s]", err)
			t.Fail()
		}

		expected := "invalid queue size"
		di := &mgo.DialInfo{
			Addrs:    []string{fmt.Sprintf("%v:%v", ip, port)},
			Database: "test",
			Timeout:  time.Second,
		}
		err = New("x101", di, 1, -1)
		if err.Error() != expected {
			log.Printf("err [%s]", err)
			t.Fail()
		}
		c.KillRemove()
	}

	// WORKING SCENARIO
	// execute
	{
		c, ip, port, err := dockertest.SetupMongoContainer()
		if err != nil {
			log.Printf("err [%s]", err)
			t.Fail()
		}

		di := &mgo.DialInfo{
			Addrs:    []string{fmt.Sprintf("%v:%v", ip, port)},
			Database: "test",
			Timeout:  time.Second,
		}
		err = New("x", di, 1, 1)
		if err != nil {
			log.Printf("err [%s]", err)
			t.Fail()
		}

		// ok Run
		err = Run("x", "coltest", func(c *mgo.Collection) error {
			x := struct {
				Message string `bson:"msg"`
			}{"hello world"}
			return c.Insert(x)
		})
		if err != nil {
			log.Printf("insert err : [%s]", err)
			t.Fail()
		}

		// ok RunWithDB
		err = RunWithDB("x", func(db *mgo.Database) error {
			x := struct {
				Message string `bson:"msg"`
			}{"hello world"}
			return db.C("ottos").Insert(x)
		})
		if err != nil {
			log.Printf("insert with DB err : [%s]", err)
			t.Fail()
		}

		expected := "session not found"
		// prefix not found
		err = Run("x1", "coltest", func(c *mgo.Collection) error {
			x := struct {
				Message string `bson:"msg"`
			}{"hello world"}
			return c.Insert(x)
		})
		if err.Error() != expected {
			log.Printf("insert err : [%s]", err)
			t.Fail()
		}
		err = RunWithDB("x1", func(db *mgo.Database) error {
			x := struct {
				Message string `bson:"msg"`
			}{"hello world"}
			return db.C("ottos").Insert(x)
		})
		if err.Error() != expected {
			log.Printf("insert with DB err : [%s]", err)
			t.Fail()
		}

		expected = "operation timeout"
		// timeout Run
		err = Run("x", "coltest", func(c *mgo.Collection) error {
			time.Sleep(2 * time.Second)
			x := struct {
				Message string `bson:"msg"`
			}{"hello world"}
			return c.Insert(x)
		})
		if err.Error() != expected {
			log.Printf("insert err : [%s]", err)
			t.Fail()
		}

		// timeout RunWithDB
		err = RunWithDB("x", func(db *mgo.Database) error {
			time.Sleep(2 * time.Second)
			x := struct {
				Message string `bson:"msg"`
			}{"hello world"}
			return db.C("ottos").Insert(x)
		})
		if err.Error() != expected {
			log.Printf("insert with DB err : [%s]", err)
			t.Fail()
		}

		// close prefix not found
		expected = "prefix not found"
		err = Close("x3")
		if err.Error() != expected {
			log.Printf("Close x3 err : [%s]", err)
			t.Fail()
		}

		// end work and close
		err = Close("x")
		if err != nil {
			log.Printf("Close x err : [%s]", err)
			t.Fail()
		}

		c.KillRemove()
	}
}
<commit_msg>fix go 1.6<commit_after>package mdb

import (
	"fmt"
	"log"
	"testing"
	"time"

	"gopkg.in/mgo.v2"
	"gopkg.in/ory-am/dockertest.v2"
)

func TestNew(t *testing.T) {
	log.SetFlags(log.Lshortfile)

	// COMMON ERRORS SCENARIO
	// prefix session empty
	{
		expected := "prefix value is empty"
		err := New("", nil, -1, -1)
		if err == nil {
			t.Fail()
		}
		if err.Error() != expected {
			log.Printf("err [%s]", err)
			t.Fail()
		}
	}
	// dial info is nil
	{
		expected := "dial info is empty"
		err := New("x", nil, -1, -1)
		if err == nil {
			t.Fail()
		}
		if err.Error() != expected {
			log.Printf("err [%s]", err)
			t.Fail()
		}
	}
	// fail connection.
	{
		expected := "no reachable servers"
		di := &mgo.DialInfo{
			Addrs:    []string{"localhost:27017"},
			Database: "test",
			Timeout:  time.Millisecond,
		}
		err := New("x", di, 1, 1)
		if err.Error() != expected {
			log.Printf("err [%s]", err)
			t.Fail()
		}
	}
	// invalid timeout
	{
		expected := "invalid timeout duration"
		di := &mgo.DialInfo{
			Addrs:    []string{},
			Database: "test",
			Timeout:  11 * time.Second,
		}
		err := New("yy", di, 1, 1)
		if err.Error() != expected {
			log.Printf("err [%s]", err)
			t.Fail()
		}
	}
	// already inited
	{
		c, ip, port, err := dockertest.SetupMongoContainer()
		if err != nil {
			log.Printf("err [%s]", err)
			t.Fail()
		}

		expected := "session already done"
		di := &mgo.DialInfo{
			Addrs:    []string{fmt.Sprintf("%v:%v", ip, port)},
			Database: "test",
			Timeout:  time.Second,
		}
		_ = New("yy", di, 1, 1)
		err = New("yy", di, 1, 1)
		if err == nil {
			log.Printf("err [%s]", err)
			t.Fail()
		}
		if err.Error() != expected {
			log.Printf("err [%s]", err)
			t.Fail()
		}

		c.KillRemove()
	}
	// bad jobq workers size
	{
		c, ip, port, err := dockertest.SetupMongoContainer()
		if err != nil {
			log.Printf("err [%s]", err)
			t.Fail()
		}

		expected := "invalid worker size"
		di := &mgo.DialInfo{
			Addrs:    []string{fmt.Sprintf("%v:%v", ip, port)},
			Database: "test",
			Timeout:  time.Second,
		}
		err = New("x100", di, -1, 1)
		if err.Error() != expected {
			log.Printf("err [%s]", err)
			t.Fail()
		}
		c.KillRemove()
	}
	// bad jobq queue size
	{
		c, ip, port, err := dockertest.SetupMongoContainer()
		if err != nil {
			log.Printf("err [%s]", err)
			t.Fail()
		}

		expected := "invalid queue size"
		di := &mgo.DialInfo{
			Addrs:    []string{fmt.Sprintf("%v:%v", ip, port)},
			Database: "test",
			Timeout:  time.Second,
		}
		err = New("x101", di, 1, -1)
		if err.Error() != expected {
			log.Printf("err [%s]", err)
			t.Fail()
		}
		c.KillRemove()
	}

	// WORKING SCENARIO
	// execute
	{
		c, ip, port, err := dockertest.SetupMongoContainer()
		if err != nil {
			log.Printf("err [%s]", err)
			t.Fail()
		}

		di := &mgo.DialInfo{
			Addrs:    []string{fmt.Sprintf("%v:%v", ip, port)},
			Database: "test",
			Timeout:  time.Second,
		}
		err = New("x", di, 1, 1)
		if err != nil {
			log.Printf("err [%s]", err)
			t.Fail()
		}

		// ok Run
		err = Run("x", "coltest", func(c *mgo.Collection) error {
			x := struct {
				Message string `bson:"msg"`
			}{"hello world"}
			return c.Insert(x)
		})
		if err != nil {
			log.Printf("insert err : [%s]", err)
			t.Fail()
		}

		// ok RunWithDB
		err = RunWithDB("x", func(db *mgo.Database) error {
			x := struct {
				Message string `bson:"msg"`
			}{"hello world"}
			return db.C("ottos").Insert(x)
		})
		if err != nil {
			log.Printf("insert with DB err : [%s]", err)
			t.Fail()
		}

		expected := "session not found"
		// prefix not found
		err = Run("x1", "coltest", func(c *mgo.Collection) error {
			x := struct {
				Message string `bson:"msg"`
			}{"hello world"}
			return c.Insert(x)
		})
		if err.Error() != expected {
			log.Printf("insert err : [%s]", err)
			t.Fail()
		}
		err = RunWithDB("x1", func(db *mgo.Database) error {
			x := struct {
				Message string `bson:"msg"`
			}{"hello world"}
			return db.C("ottos").Insert(x)
		})
		if err.Error() != expected {
			log.Printf("insert with DB err : [%s]", err)
			t.Fail()
		}

		expected = "operation timeout"
		// timeout Run
		err = Run("x", "coltest", func(c *mgo.Collection) error {
			time.Sleep(2 * time.Second)
			x := struct {
				Message string `bson:"msg"`
			}{"hello world"}
			return c.Insert(x)
		})
		if err.Error() != expected {
			log.Printf("insert err : [%s]", err)
			t.Fail()
		}

		// timeout RunWithDB
		err = RunWithDB("x", func(db *mgo.Database) error {
			time.Sleep(2 * time.Second)
			x := struct {
				Message string `bson:"msg"`
			}{"hello world"}
			return db.C("ottos").Insert(x)
		})
		if err.Error() != expected {
			log.Printf("insert with DB err : [%s]", err)
			t.Fail()
		}

		// close prefix not found
		expected = "prefix not found"
		err = Close("x3")
		if err.Error() != expected {
			log.Printf("Close x3 err : [%s]", err)
			t.Fail()
		}

		// end work and close
		err = Close("x")
		if err != nil {
			log.Printf("Close x err : [%s]", err)
			t.Fail()
		}

		c.KillRemove()
	}
}
<|endoftext|>


<commit_before>// Copyright © 2015 Steve Francia <spf@spf13.com>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helpers

import (
	"fmt"
	"io/ioutil"
	"path/filepath"

	"github.com/spf13/hugo/hugofs"
	"github.com/spf13/hugo/parser"
)

// this should be the only one
const hugoVersionMain = 0.14
const hugoVersionSuffix = "" // blank this when doing a release

// HugoVersion returns the current Hugo version. It will include
// a suffix, typically '-DEV', if it's development version.
func HugoVersion() string {
	return hugoVersion(hugoVersionMain, hugoVersionSuffix)
}

// HugoReleaseVersion is same as HugoVersion, but no suffix.
func HugoReleaseVersion() string {
	return hugoVersionNoSuffix(hugoVersionMain)
}

// NextHugoReleaseVersion returns the next Hugo release version.
func NextHugoReleaseVersion() string {
	return hugoVersionNoSuffix(hugoVersionMain + 0.01)
}

func hugoVersion(version float32, suffix string) string {
	return fmt.Sprintf("%.2g%s", version, suffix)
}

func hugoVersionNoSuffix(version float32) string {
	return fmt.Sprintf("%.2g", version)
}

// IsThemeVsHugoVersionMismatch returns whether the current Hugo version is < theme's min_version
func IsThemeVsHugoVersionMismatch() (mismatch bool, requiredMinVersion string) {
	if !ThemeSet() {
		return
	}

	themeDir, err := getThemeDirPath("")

	if err != nil {
		return
	}

	fs := hugofs.SourceFs
	path := filepath.Join(themeDir, "theme.toml")

	exists, err := Exists(path, fs)

	if err != nil || !exists {
		return
	}

	f, err := fs.Open(path)

	if err != nil {
		return
	}

	defer f.Close()

	b, err := ioutil.ReadAll(f)

	if err != nil {
		return
	}

	c, err := parser.HandleTOMLMetaData(b)

	if err != nil {
		return
	}

	config := c.(map[string]interface{})

	if minVersion, ok := config["min_version"]; ok {
		switch minVersion.(type) {
		case float32:
			return hugoVersionMain < minVersion.(float32), fmt.Sprint(minVersion)
		case float64:
			return hugoVersionMain < minVersion.(float64), fmt.Sprint(minVersion)
		default:
			return
		}

	}

	return
}
<commit_msg>bumping master version to v0.15-DEV<commit_after>// Copyright © 2015 Steve Francia <spf@spf13.com>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helpers

import (
	"fmt"
	"io/ioutil"
	"path/filepath"

	"github.com/spf13/hugo/hugofs"
	"github.com/spf13/hugo/parser"
)

// this should be the only one
const hugoVersionMain = 0.15
const hugoVersionSuffix = "DEV" // blank this when doing a release

// HugoVersion returns the current Hugo version. It will include
// a suffix, typically '-DEV', if it's development version.
func HugoVersion() string {
	return hugoVersion(hugoVersionMain, hugoVersionSuffix)
}

// HugoReleaseVersion is same as HugoVersion, but no suffix.
func HugoReleaseVersion() string {
	return hugoVersionNoSuffix(hugoVersionMain)
}

// NextHugoReleaseVersion returns the next Hugo release version.
func NextHugoReleaseVersion() string {
	return hugoVersionNoSuffix(hugoVersionMain + 0.01)
}

func hugoVersion(version float32, suffix string) string {
	return fmt.Sprintf("%.2g%s", version, suffix)
}

func hugoVersionNoSuffix(version float32) string {
	return fmt.Sprintf("%.2g", version)
}

// IsThemeVsHugoVersionMismatch returns whether the current Hugo version is < theme's min_version
func IsThemeVsHugoVersionMismatch() (mismatch bool, requiredMinVersion string) {
	if !ThemeSet() {
		return
	}

	themeDir, err := getThemeDirPath("")

	if err != nil {
		return
	}

	fs := hugofs.SourceFs
	path := filepath.Join(themeDir, "theme.toml")

	exists, err := Exists(path, fs)

	if err != nil || !exists {
		return
	}

	f, err := fs.Open(path)

	if err != nil {
		return
	}

	defer f.Close()

	b, err := ioutil.ReadAll(f)

	if err != nil {
		return
	}

	c, err := parser.HandleTOMLMetaData(b)

	if err != nil {
		return
	}

	config := c.(map[string]interface{})

	if minVersion, ok := config["min_version"]; ok {
		switch minVersion.(type) {
		case float32:
			return hugoVersionMain < minVersion.(float32), fmt.Sprint(minVersion)
		case float64:
			return hugoVersionMain < minVersion.(float64), fmt.Sprint(minVersion)
		default:
			return
		}

	}

	return
}
<|endoftext|>


<commit_before>package cli

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"strconv"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/flynn/go-docopt"
	"github.com/flynn/flynn/host/types"
	"github.com/flynn/flynn/pkg/cluster"
)

func init() {
	Register("log", runLog, `
usage: flynn-host log [--init] [-f|--follow] [--lines=<number>] ID

Get the logs of a job`)
}

func runLog(args *docopt.Args, client *cluster.Client) error {
	jobID := args.String["ID"]
	hostID, err := cluster.ExtractHostID(jobID)
	if err != nil {
		return err
	}

	lines := 0
	if args.String["--lines"] != "" {
		lines, err = strconv.Atoi(args.String["--lines"])
		if err != nil {
			return err
		}
	}

	if lines > 0 {
		stdoutR, stdoutW := io.Pipe()
		stderrR, stderrW := io.Pipe()

		go func() {
			getLog(hostID, jobID, client, false, args.Bool["--init"], stdoutW, stderrW)
			stdoutW.Close()
			stderrW.Close()
		}()
		tailLogs(stdoutR, stderrR, lines, os.Stdout, os.Stderr)
		return nil
	}
	return getLog(
		hostID,
		jobID,
		client,
		args.Bool["-f"] || args.Bool["--follow"],
		args.Bool["--init"],
		os.Stdout,
		os.Stderr,
	)
}

func getLog(hostID, jobID string, client *cluster.Client, follow, init bool, stdout, stderr io.Writer) error {
	hostClient, err := client.Host(hostID)
	if err != nil {
		return fmt.Errorf("could not connect to host %s: %s", hostID, err)
	}
	attachReq := &host.AttachReq{
		JobID: jobID,
		Flags: host.AttachFlagStdout | host.AttachFlagStderr | host.AttachFlagLogs,
	}
	if follow {
		attachReq.Flags |= host.AttachFlagStream
	}
	if init {
		attachReq.Flags |= host.AttachFlagInitLog
	}
	attachClient, err := hostClient.Attach(attachReq, false)
	if err != nil {
		switch err {
		case host.ErrJobNotRunning, host.ErrAttached:
			return nil
		case cluster.ErrWouldWait:
			return errors.New("no such job")
		}
		return err
	}
	defer attachClient.Close()
	_, err = attachClient.Receive(stdout, stderr)
	return err
}

type LogLine struct {
	Token   int
	Content []byte
}

type LogRing struct {
	logLines []*LogLine
	start    int
}

func NewLogRing(capacity int) *LogRing {
	return &LogRing{
		logLines: make([]*LogLine, 0, capacity),
	}
}

func (r *LogRing) Add(l *LogLine) {
	if len(r.logLines) < cap(r.logLines) {
		r.logLines = append(r.logLines, l)
	} else {
		r.logLines[r.start] = l
		r.start++

		if r.start == cap(r.logLines) {
			r.start = 0
		}
	}
}

func (r *LogRing) Read() []*LogLine {
	buf := make([]*LogLine, len(r.logLines))
	if n := copy(buf, r.logLines[r.start:len(r.logLines)]); n < len(r.logLines) {
		copy(buf[n:], r.logLines[:r.start])
	}
	return buf
}

func scanLogs(reader io.Reader, token int, output chan LogLine) {
	scanner := bufio.NewScanner(reader)
	for scanner.Scan() {
		buf := make([]byte, len(scanner.Bytes())+1)
		copy(buf, scanner.Bytes())
		buf[len(buf)-1] = '\n'
		output <- LogLine{Token: token, Content: buf}
	}
	close(output)
}

func tailLogs(stdoutR, stderrR io.Reader, lines int, stdoutW, stderrW io.Writer) {
	gather1 := make(chan LogLine)
	gather2 := make(chan LogLine)
	r := NewLogRing(lines)
	go scanLogs(stdoutR, 0, gather1)
	go scanLogs(stderrR, 1, gather2)
	for {
		select {
		case v, ok := <-gather1:
			if ok {
				r.Add(&v)
			} else {
				gather1 = nil
			}
		case v, ok := <-gather2:
			if ok {
				r.Add(&v)
			} else {
				gather2 = nil
			}
		}
		if gather1 == nil && gather2 == nil {
			break
		}
	}

	for _, ll := range r.Read() {
		switch ll.Token {
		case 0:
			stdoutW.Write(ll.Content)
		case 1:
			stderrW.Write(ll.Content)
		}
	}
}
<commit_msg>host/cli: Send stderr from log stream to stdout by default<commit_after>package cli

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"strconv"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/flynn/go-docopt"
	"github.com/flynn/flynn/host/types"
	"github.com/flynn/flynn/pkg/cluster"
)

func init() {
	Register("log", runLog, `
usage: flynn-host log [--init] [-f|--follow] [--lines=<number>] [--split-stderr] ID

Get the logs of a job`)
}

func runLog(args *docopt.Args, client *cluster.Client) error {
	jobID := args.String["ID"]
	hostID, err := cluster.ExtractHostID(jobID)
	if err != nil {
		return err
	}

	lines := 0
	if args.String["--lines"] != "" {
		lines, err = strconv.Atoi(args.String["--lines"])
		if err != nil {
			return err
		}
	}

	stderr := os.Stdout
	if args.Bool["--split-stderr"] {
		stderr = os.Stderr
	}

	if lines > 0 {
		stdoutR, stdoutW := io.Pipe()
		stderrR, stderrW := io.Pipe()

		go func() {
			getLog(hostID, jobID, client, false, args.Bool["--init"], stdoutW, stderrW)
			stdoutW.Close()
			stderrW.Close()
		}()
		tailLogs(stdoutR, stderrR, lines, os.Stdout, stderr)
		return nil
	}
	return getLog(
		hostID,
		jobID,
		client,
		args.Bool["-f"] || args.Bool["--follow"],
		args.Bool["--init"],
		os.Stdout,
		stderr,
	)
}

func getLog(hostID, jobID string, client *cluster.Client, follow, init bool, stdout, stderr io.Writer) error {
	hostClient, err := client.Host(hostID)
	if err != nil {
		return fmt.Errorf("could not connect to host %s: %s", hostID, err)
	}
	attachReq := &host.AttachReq{
		JobID: jobID,
		Flags: host.AttachFlagStdout | host.AttachFlagStderr | host.AttachFlagLogs,
	}
	if follow {
		attachReq.Flags |= host.AttachFlagStream
	}
	if init {
		attachReq.Flags |= host.AttachFlagInitLog
	}
	attachClient, err := hostClient.Attach(attachReq, false)
	if err != nil {
		switch err {
		case host.ErrJobNotRunning, host.ErrAttached:
			return nil
		case cluster.ErrWouldWait:
			return errors.New("no such job")
		}
		return err
	}
	defer attachClient.Close()
	_, err = attachClient.Receive(stdout, stderr)
	return err
}

type LogLine struct {
	Token   int
	Content []byte
}

type LogRing struct {
	logLines []*LogLine
	start    int
}

func NewLogRing(capacity int) *LogRing {
	return &LogRing{
		logLines: make([]*LogLine, 0, capacity),
	}
}

func (r *LogRing) Add(l *LogLine) {
	if len(r.logLines) < cap(r.logLines) {
		r.logLines = append(r.logLines, l)
	} else {
		r.logLines[r.start] = l
		r.start++

		if r.start == cap(r.logLines) {
			r.start = 0
		}
	}
}

func (r *LogRing) Read() []*LogLine {
	buf := make([]*LogLine, len(r.logLines))
	if n := copy(buf, r.logLines[r.start:len(r.logLines)]); n < len(r.logLines) {
		copy(buf[n:], r.logLines[:r.start])
	}
	return buf
}

func scanLogs(reader io.Reader, token int, output chan LogLine) {
	scanner := bufio.NewScanner(reader)
	for scanner.Scan() {
		buf := make([]byte, len(scanner.Bytes())+1)
		copy(buf, scanner.Bytes())
		buf[len(buf)-1] = '\n'
		output <- LogLine{Token: token, Content: buf}
	}
	close(output)
}

func tailLogs(stdoutR, stderrR io.Reader, lines int, stdoutW, stderrW io.Writer) {
	gather1 := make(chan LogLine)
	gather2 := make(chan LogLine)
	r := NewLogRing(lines)
	go scanLogs(stdoutR, 0, gather1)
	go scanLogs(stderrR, 1, gather2)
	for {
		select {
		case v, ok := <-gather1:
			if ok {
				r.Add(&v)
			} else {
				gather1 = nil
			}
		case v, ok := <-gather2:
			if ok {
				r.Add(&v)
			} else {
				gather2 = nil
			}
		}
		if gather1 == nil && gather2 == nil {
			break
		}
	}

	for _, ll := range r.Read() {
		switch ll.Token {
		case 0:
			stdoutW.Write(ll.Content)
		case 1:
			stderrW.Write(ll.Content)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Ankyra

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package core

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/ankyra/escape-core/parsers"
	"github.com/ankyra/escape-core/script"
	"github.com/ankyra/escape-core/templates"
	"github.com/ankyra/escape-core/util"
	"github.com/ankyra/escape-core/variables"
	"io/ioutil"
	"path/filepath"
	"strings"
)

type ExecStage struct {
	Script string `json:"script"`
}

type ConsumerConfig struct {
    Name string `json:"name"`
}

func NewConsumerConfig(name string) *ConsumerConfig {
    return &ConsumerConfig{name}
}

type ProviderConfig struct {
    Name string `json:"name"`
}

func NewProviderConfig(name string) *ProviderConfig {
    return &ProviderConfig{name}
}

type DependencyConfig struct {
    ReleaseId string `json:"release_id"`
}

func NewDependencyConfig(releaseId string) *DependencyConfig {
    return &DependencyConfig{releaseId}
}

type ExtensionConfig struct {
    ReleaseId string `json:"release_id"`
}

func NewExtensionConfig(releaseId string) *ExtensionConfig {
    return &ExtensionConfig{releaseId}
}

type ReleaseMetadata struct {
	ApiVersion  string                `json:"api_version"`
	Branch      string                `json:"branch"`
	Description string                `json:"description"`
	Files       map[string]string     `json:"files", {}`
	Logo        string                `json:"logo"`
	Name        string                `json:"name"`
	Revision    string                `json:"git_revision"`
	Metadata    map[string]string     `json:"metadata"`
	Version     string                `json:"version"`

	Consumes    []*ConsumerConfig     `json:"consumes"`
	Depends     []*DependencyConfig   `json:"depends"`
	Errands     map[string]*Errand    `json:"errands"`
	Extends     []*ExtensionConfig    `json:"extends"`
	Inputs      []*variables.Variable `json:"inputs"`
	Outputs     []*variables.Variable `json:"outputs"`
	Provides    []*ProviderConfig           `json:"provides"`
	Stages      map[string]*ExecStage `json:"stages"`
	Templates   []*templates.Template `json:"templates"`
	VariableCtx map[string]string     `json:"variable_context"`
}

func NewEmptyReleaseMetadata() *ReleaseMetadata {
	return &ReleaseMetadata{
		ApiVersion:  "3",
		Files:       map[string]string{},
		Metadata:    map[string]string{},

		Consumes:    []*ConsumerConfig{},
		Depends:     []*DependencyConfig{},
		Errands:     map[string]*Errand{},
		Extends:     []*ExtensionConfig{},
		Inputs:      []*variables.Variable{},
		Outputs:     []*variables.Variable{},
		Provides:    []*ProviderConfig{},
		Stages:      map[string]*ExecStage{},
		Templates:   []*templates.Template{},
		VariableCtx: map[string]string{},
	}
}

func NewReleaseMetadata(name, version string) *ReleaseMetadata {
	m := NewEmptyReleaseMetadata()
	m.Name = name
	m.Version = version
	return m
}

func NewReleaseMetadataFromJsonString(content string) (*ReleaseMetadata, error) {
	result := NewEmptyReleaseMetadata()
	if err := json.Unmarshal([]byte(content), &result); err != nil {
		return nil, fmt.Errorf("Couldn't unmarshal JSON release metadata: %s", err.Error())
	}
	return result, validate(result)
}

func NewReleaseMetadataFromFile(metadataFile string) (*ReleaseMetadata, error) {
	if !util.PathExists(metadataFile) {
		return nil, errors.New("Release metadata file " + metadataFile + " does not exist")
	}
	content, err := ioutil.ReadFile(metadataFile)
	if err != nil {
		return nil, err
	}
	return NewReleaseMetadataFromJsonString(string(content))
}

func validate(m *ReleaseMetadata) error {
	if m == nil {
		return fmt.Errorf("Missing release metadata")
	}
	if m.Name == "" {
		return fmt.Errorf("Missing name field in release metadata")
	}
	if m.Version == "" {
		return fmt.Errorf("Missing version field in release metadata")
	}
	if err := parsers.ValidateVersion(m.Version); err != nil {
		return err
	}
	for _, i := range m.Inputs {
		if err := i.Validate(); err != nil {
			return err
		}
	}
	for _, i := range m.Outputs {
		if err := i.Validate(); err != nil {
			return err
		}
	}
	return nil
}

func (m *ReleaseMetadata) AddExtension(releaseId string) {
	for _, e := range m.Extends {
		if e.ReleaseId == releaseId {
			return
		}
	}
	m.Extends = append(m.Extends, NewExtensionConfig(releaseId))
}

func (m *ReleaseMetadata) GetExtensions() []string {
    result := []string{}
    for _, ext := range m.Extends {
        result = append(result, ext.ReleaseId)
    }
    return result
}

func (m *ReleaseMetadata) GetStage(stage string) *ExecStage {
	result, ok := m.Stages[stage]
	if !ok {
		result = &ExecStage{}
		m.Stages[stage] = result
	}
	return result
}

func (m *ReleaseMetadata) SetStage(stage, script string) {
	if script == "" {
		return
	}
	st := m.GetStage(stage)
	st.Script = script
}

func (m *ReleaseMetadata) GetScript(stage string) string {
	return m.GetStage(stage).Script
}

func (m *ReleaseMetadata) AddConsumes( c string) {
    for _, consumer := range m.Consumes {
        if consumer.Name == c {
            return
        }
    }
    m.Consumes = append(m.Consumes, NewConsumerConfig(c))
}

func (m *ReleaseMetadata) SetConsumes(c []string) {
    for _, consumer := range c {
        m.AddConsumes(consumer)
    }
}

func (m *ReleaseMetadata) GetConsumes() []string {
    result := []string{}
    for _, c := range m.Consumes {
        result = append(result, c.Name)
    }
	return result
}

func (m *ReleaseMetadata) GetErrands() map[string]*Errand {
	result := map[string]*Errand{}
	for key, val := range m.Errands {
		result[key] = val
	}
	return result
}

func (m *ReleaseMetadata) GetInputs() []*variables.Variable {
	result := []*variables.Variable{}
	for _, i := range m.Inputs {
		result = append(result, i)
	}
	return result
}

func (m *ReleaseMetadata) GetOutputs() []*variables.Variable {
	result := []*variables.Variable{}
	for _, i := range m.Outputs {
		result = append(result, i)
	}
	return result
}

func (m *ReleaseMetadata) AddProvides(p string) {
    for _, provider := range m.Provides {
        if provider.Name == p {
            return
        }
    }
    m.Provides = append(m.Provides, NewProviderConfig(p))
}

func (m *ReleaseMetadata) GetProvides() []string {
    result := []string{}
    for _, c := range m.Consumes {
        result = append(result, c.Name)
    }
	return result
}

func (m *ReleaseMetadata) SetProvides(p []string) {
    for _, provider := range p {
        m.AddProvides(provider)
    }
}

func (m *ReleaseMetadata) GetDependencies() []string {
    result := []string{}
    for _, c := range m.Depends {
        result = append(result, c.ReleaseId)
    }
	return result
}

func (m *ReleaseMetadata) SetDependencies(deps []string) {
    result := []*DependencyConfig{}
    for _, d := range deps {
        result = append(result, NewDependencyConfig(d))
    }
	m.Depends = result
}

func (m *ReleaseMetadata) GetVariableContext() map[string]string {
	if m.VariableCtx == nil {
		return map[string]string{}
	}
	return m.VariableCtx
}

func (m *ReleaseMetadata) SetVariableInContext(v string, ref string) {
	ctx := m.GetVariableContext()
	ctx[v] = ref
	m.VariableCtx = ctx
}

func (m *ReleaseMetadata) GetReleaseId() string {
	return m.Name + "-v" + m.Version
}

func (m *ReleaseMetadata) GetVersionlessReleaseId() string {
	return m.Name
}

func (m *ReleaseMetadata) AddInputVariable(input *variables.Variable) {
	for _, i := range m.Inputs {
		if i.GetId() == input.GetId() {
			i.Default = input.Default
			return
		}
	}
	m.Inputs = append(m.Inputs, input)
}

func (m *ReleaseMetadata) AddOutputVariable(output *variables.Variable) {
	for _, i := range m.Outputs {
		if i.GetId() == output.GetId() {
			return
		}
	}
	m.Outputs = append(m.Outputs, output)
}

func (m *ReleaseMetadata) ToJson() string {
	str, err := json.MarshalIndent(m, "", "   ")
	if err != nil {
		panic(err)
	}
	return string(str)
}

func (m *ReleaseMetadata) ToDict() (map[string]interface{}, error) {
	asJson := []byte(m.ToJson())
	result := map[string]interface{}{}
	if err := json.Unmarshal(asJson, &result); err != nil {
		return nil, fmt.Errorf("Couldn't marshal release metadata: %s. This is a bug in Escape", err.Error())
	}
	return result, nil
}

func (m *ReleaseMetadata) WriteJsonFile(path string) error {
	contents := []byte(m.ToJson())
	return ioutil.WriteFile(path, contents, 0644)
}

func (m *ReleaseMetadata) AddFileWithDigest(path, hexDigest string) {
	m.Files[path] = hexDigest
}

func (m *ReleaseMetadata) ToDependency() *Dependency {
	return NewDependencyFromMetadata(m)
}

func (m *ReleaseMetadata) GetDirectories() []string {
	dirs := map[string]bool{}
	for file := range m.Files {
		dir, _ := filepath.Split(file)
		dirs[dir] = true
		root := ""
		for _, d := range strings.Split(dir, "/") {
			if d != "" {
				root += d + "/"
				dirs[root] = true
			}
		}
	}
	result := []string{}
	for d := range dirs {
		if d != "" {
			result = append(result, d)
		}
	}
	return result
}

func (m *ReleaseMetadata) ToScript() script.Script {
	return script.LiftDict(m.ToScriptMap())
}

func (m *ReleaseMetadata) ToScriptMap() map[string]script.Script {
	metadataDict := map[string]script.Script{}
	for key, val := range m.Metadata {
		metadataDict[key] = script.LiftString(val)
	}
	return map[string]script.Script{
		"metadata": script.LiftDict(metadataDict),

		"branch":      script.LiftString(m.Branch),
		"description": script.LiftString(m.Description),
		"logo":        script.LiftString(m.Logo),
		"name":        script.LiftString(m.Name),
		"revision":    script.LiftString(m.Revision),
		"version":     script.LiftString(m.Version),
		"id":          script.LiftString(m.GetReleaseId()),
	}
}
<commit_msg>Returning Provides instead of Consumes in GetProvides method. Could do with some testing...<commit_after>/*
Copyright 2017 Ankyra

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package core

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/ankyra/escape-core/parsers"
	"github.com/ankyra/escape-core/script"
	"github.com/ankyra/escape-core/templates"
	"github.com/ankyra/escape-core/util"
	"github.com/ankyra/escape-core/variables"
	"io/ioutil"
	"path/filepath"
	"strings"
)

type ExecStage struct {
	Script string `json:"script"`
}

type ConsumerConfig struct {
    Name string `json:"name"`
}

func NewConsumerConfig(name string) *ConsumerConfig {
    return &ConsumerConfig{name}
}

type ProviderConfig struct {
    Name string `json:"name"`
}

func NewProviderConfig(name string) *ProviderConfig {
    return &ProviderConfig{name}
}

type DependencyConfig struct {
    ReleaseId string `json:"release_id"`
}

func NewDependencyConfig(releaseId string) *DependencyConfig {
    return &DependencyConfig{releaseId}
}

type ExtensionConfig struct {
    ReleaseId string `json:"release_id"`
}

func NewExtensionConfig(releaseId string) *ExtensionConfig {
    return &ExtensionConfig{releaseId}
}

type ReleaseMetadata struct {
	ApiVersion  string                `json:"api_version"`
	Branch      string                `json:"branch"`
	Description string                `json:"description"`
	Files       map[string]string     `json:"files", {}`
	Logo        string                `json:"logo"`
	Name        string                `json:"name"`
	Revision    string                `json:"git_revision"`
	Metadata    map[string]string     `json:"metadata"`
	Version     string                `json:"version"`

	Consumes    []*ConsumerConfig     `json:"consumes"`
	Depends     []*DependencyConfig   `json:"depends"`
	Errands     map[string]*Errand    `json:"errands"`
	Extends     []*ExtensionConfig    `json:"extends"`
	Inputs      []*variables.Variable `json:"inputs"`
	Outputs     []*variables.Variable `json:"outputs"`
	Provides    []*ProviderConfig           `json:"provides"`
	Stages      map[string]*ExecStage `json:"stages"`
	Templates   []*templates.Template `json:"templates"`
	VariableCtx map[string]string     `json:"variable_context"`
}

func NewEmptyReleaseMetadata() *ReleaseMetadata {
	return &ReleaseMetadata{
		ApiVersion:  "3",
		Files:       map[string]string{},
		Metadata:    map[string]string{},

		Consumes:    []*ConsumerConfig{},
		Depends:     []*DependencyConfig{},
		Errands:     map[string]*Errand{},
		Extends:     []*ExtensionConfig{},
		Inputs:      []*variables.Variable{},
		Outputs:     []*variables.Variable{},
		Provides:    []*ProviderConfig{},
		Stages:      map[string]*ExecStage{},
		Templates:   []*templates.Template{},
		VariableCtx: map[string]string{},
	}
}

func NewReleaseMetadata(name, version string) *ReleaseMetadata {
	m := NewEmptyReleaseMetadata()
	m.Name = name
	m.Version = version
	return m
}

func NewReleaseMetadataFromJsonString(content string) (*ReleaseMetadata, error) {
	result := NewEmptyReleaseMetadata()
	if err := json.Unmarshal([]byte(content), &result); err != nil {
		return nil, fmt.Errorf("Couldn't unmarshal JSON release metadata: %s", err.Error())
	}
	return result, validate(result)
}

func NewReleaseMetadataFromFile(metadataFile string) (*ReleaseMetadata, error) {
	if !util.PathExists(metadataFile) {
		return nil, errors.New("Release metadata file " + metadataFile + " does not exist")
	}
	content, err := ioutil.ReadFile(metadataFile)
	if err != nil {
		return nil, err
	}
	return NewReleaseMetadataFromJsonString(string(content))
}

func validate(m *ReleaseMetadata) error {
	if m == nil {
		return fmt.Errorf("Missing release metadata")
	}
	if m.Name == "" {
		return fmt.Errorf("Missing name field in release metadata")
	}
	if m.Version == "" {
		return fmt.Errorf("Missing version field in release metadata")
	}
	if err := parsers.ValidateVersion(m.Version); err != nil {
		return err
	}
	for _, i := range m.Inputs {
		if err := i.Validate(); err != nil {
			return err
		}
	}
	for _, i := range m.Outputs {
		if err := i.Validate(); err != nil {
			return err
		}
	}
	return nil
}

func (m *ReleaseMetadata) AddExtension(releaseId string) {
	for _, e := range m.Extends {
		if e.ReleaseId == releaseId {
			return
		}
	}
	m.Extends = append(m.Extends, NewExtensionConfig(releaseId))
}

func (m *ReleaseMetadata) GetExtensions() []string {
    result := []string{}
    for _, ext := range m.Extends {
        result = append(result, ext.ReleaseId)
    }
    return result
}

func (m *ReleaseMetadata) GetStage(stage string) *ExecStage {
	result, ok := m.Stages[stage]
	if !ok {
		result = &ExecStage{}
		m.Stages[stage] = result
	}
	return result
}

func (m *ReleaseMetadata) SetStage(stage, script string) {
	if script == "" {
		return
	}
	st := m.GetStage(stage)
	st.Script = script
}

func (m *ReleaseMetadata) GetScript(stage string) string {
	return m.GetStage(stage).Script
}

func (m *ReleaseMetadata) AddConsumes( c string) {
    for _, consumer := range m.Consumes {
        if consumer.Name == c {
            return
        }
    }
    m.Consumes = append(m.Consumes, NewConsumerConfig(c))
}

func (m *ReleaseMetadata) SetConsumes(c []string) {
    for _, consumer := range c {
        m.AddConsumes(consumer)
    }
}

func (m *ReleaseMetadata) GetConsumes() []string {
    result := []string{}
    for _, c := range m.Consumes {
        result = append(result, c.Name)
    }
	return result
}

func (m *ReleaseMetadata) GetErrands() map[string]*Errand {
	result := map[string]*Errand{}
	for key, val := range m.Errands {
		result[key] = val
	}
	return result
}

func (m *ReleaseMetadata) GetInputs() []*variables.Variable {
	result := []*variables.Variable{}
	for _, i := range m.Inputs {
		result = append(result, i)
	}
	return result
}

func (m *ReleaseMetadata) GetOutputs() []*variables.Variable {
	result := []*variables.Variable{}
	for _, i := range m.Outputs {
		result = append(result, i)
	}
	return result
}

func (m *ReleaseMetadata) AddProvides(p string) {
    for _, provider := range m.Provides {
        if provider.Name == p {
            return
        }
    }
    m.Provides = append(m.Provides, NewProviderConfig(p))
}

func (m *ReleaseMetadata) GetProvides() []string {
    result := []string{}
    for _, c := range m.Provides {
        result = append(result, c.Name)
    }
	return result
}

func (m *ReleaseMetadata) SetProvides(p []string) {
    for _, provider := range p {
        m.AddProvides(provider)
    }
}

func (m *ReleaseMetadata) GetDependencies() []string {
    result := []string{}
    for _, c := range m.Depends {
        result = append(result, c.ReleaseId)
    }
	return result
}

func (m *ReleaseMetadata) SetDependencies(deps []string) {
    result := []*DependencyConfig{}
    for _, d := range deps {
        result = append(result, NewDependencyConfig(d))
    }
	m.Depends = result
}

func (m *ReleaseMetadata) GetVariableContext() map[string]string {
	if m.VariableCtx == nil {
		return map[string]string{}
	}
	return m.VariableCtx
}

func (m *ReleaseMetadata) SetVariableInContext(v string, ref string) {
	ctx := m.GetVariableContext()
	ctx[v] = ref
	m.VariableCtx = ctx
}

func (m *ReleaseMetadata) GetReleaseId() string {
	return m.Name + "-v" + m.Version
}

func (m *ReleaseMetadata) GetVersionlessReleaseId() string {
	return m.Name
}

func (m *ReleaseMetadata) AddInputVariable(input *variables.Variable) {
	for _, i := range m.Inputs {
		if i.GetId() == input.GetId() {
			i.Default = input.Default
			return
		}
	}
	m.Inputs = append(m.Inputs, input)
}

func (m *ReleaseMetadata) AddOutputVariable(output *variables.Variable) {
	for _, i := range m.Outputs {
		if i.GetId() == output.GetId() {
			return
		}
	}
	m.Outputs = append(m.Outputs, output)
}

func (m *ReleaseMetadata) ToJson() string {
	str, err := json.MarshalIndent(m, "", "   ")
	if err != nil {
		panic(err)
	}
	return string(str)
}

func (m *ReleaseMetadata) ToDict() (map[string]interface{}, error) {
	asJson := []byte(m.ToJson())
	result := map[string]interface{}{}
	if err := json.Unmarshal(asJson, &result); err != nil {
		return nil, fmt.Errorf("Couldn't marshal release metadata: %s. This is a bug in Escape", err.Error())
	}
	return result, nil
}

func (m *ReleaseMetadata) WriteJsonFile(path string) error {
	contents := []byte(m.ToJson())
	return ioutil.WriteFile(path, contents, 0644)
}

func (m *ReleaseMetadata) AddFileWithDigest(path, hexDigest string) {
	m.Files[path] = hexDigest
}

func (m *ReleaseMetadata) ToDependency() *Dependency {
	return NewDependencyFromMetadata(m)
}

func (m *ReleaseMetadata) GetDirectories() []string {
	dirs := map[string]bool{}
	for file := range m.Files {
		dir, _ := filepath.Split(file)
		dirs[dir] = true
		root := ""
		for _, d := range strings.Split(dir, "/") {
			if d != "" {
				root += d + "/"
				dirs[root] = true
			}
		}
	}
	result := []string{}
	for d := range dirs {
		if d != "" {
			result = append(result, d)
		}
	}
	return result
}

func (m *ReleaseMetadata) ToScript() script.Script {
	return script.LiftDict(m.ToScriptMap())
}

func (m *ReleaseMetadata) ToScriptMap() map[string]script.Script {
	metadataDict := map[string]script.Script{}
	for key, val := range m.Metadata {
		metadataDict[key] = script.LiftString(val)
	}
	return map[string]script.Script{
		"metadata": script.LiftDict(metadataDict),

		"branch":      script.LiftString(m.Branch),
		"description": script.LiftString(m.Description),
		"logo":        script.LiftString(m.Logo),
		"name":        script.LiftString(m.Name),
		"revision":    script.LiftString(m.Revision),
		"version":     script.LiftString(m.Version),
		"id":          script.LiftString(m.GetReleaseId()),
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package openstack

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/util/exec"
	"k8s.io/kubernetes/pkg/util/mount"
)

// metadataUrl is URL to OpenStack metadata server. It's hardcoded IPv4
// link-local address as documented in "OpenStack Cloud Administrator Guide",
// chapter Compute - Networking with nova-network.
// http://docs.openstack.org/admin-guide-cloud/compute-networking-nova.html#metadata-service
const metadataUrl = "http://169.254.169.254/openstack/2012-08-10/meta_data.json"

// Config drive is defined as an iso9660 or vfat (deprecated) drive
// with the "config-2" label.
// http://docs.openstack.org/user-guide/cli-config-drive.html
const configDriveLabel = "config-2"
const configDrivePath = "openstack/2012-08-10/meta_data.json"

var ErrBadMetadata = errors.New("Invalid OpenStack metadata, got empty uuid")

// Assumes the "2012-08-10" meta_data.json format.
// See http://docs.openstack.org/user-guide/cli_config_drive.html
type Metadata struct {
	Uuid             string `json:"uuid"`
	Name             string `json:"name"`
	AvailabilityZone string `json:"availability_zone"`
	// .. and other fields we don't care about.  Expand as necessary.
}

// parseMetadataUUID reads JSON from OpenStack metadata server and parses
// instance ID out of it.
func parseMetadata(r io.Reader) (*Metadata, error) {
	var metadata Metadata
	json := json.NewDecoder(r)
	if err := json.Decode(&metadata); err != nil {
		return nil, err
	}

	if metadata.Uuid == "" {
		return nil, ErrBadMetadata
	}

	return &metadata, nil
}

func getMetadataFromConfigDrive() (*Metadata, error) {
	// Try to read instance UUID from config drive.
	dev := "/dev/disk/by-label/" + configDriveLabel
	if _, err := os.Stat(dev); os.IsNotExist(err) {
		out, err := exec.New().Command(
			"blkid", "-l",
			"-t", "LABEL="+configDriveLabel,
			"-o", "device",
		).CombinedOutput()
		if err != nil {
			glog.V(2).Infof("Unable to run blkid: %v", err)
			return nil, err
		}
		dev = strings.TrimSpace(string(out))
	}

	mntdir, err := ioutil.TempDir("", "configdrive")
	if err != nil {
		return nil, err
	}
	defer os.Remove(mntdir)

	glog.V(4).Infof("Attempting to mount configdrive %s on %s", dev, mntdir)

	mounter := mount.New()
	err = mounter.Mount(dev, mntdir, "iso9660", []string{"ro"})
	if err != nil {
		err = mounter.Mount(dev, mntdir, "vfat", []string{"ro"})
	}
	if err != nil {
		glog.Errorf("Error mounting configdrive %s: %v", dev, err)
		return nil, err
	}
	defer mounter.Unmount(mntdir)

	glog.V(4).Infof("Configdrive mounted on %s", mntdir)

	f, err := os.Open(
		filepath.Join(mntdir, configDrivePath))
	if err != nil {
		glog.Errorf("Error reading %s on config drive: %v", configDrivePath, err)
		return nil, err
	}
	defer f.Close()

	return parseMetadata(f)
}

func getMetadataFromMetadataService() (*Metadata, error) {
	// Try to get JSON from metdata server.
	glog.V(4).Infof("Attempting to fetch metadata from %s", metadataUrl)
	resp, err := http.Get(metadataUrl)
	if err != nil {
		glog.V(3).Infof("Cannot read %s: %v", metadataUrl, err)
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		err = fmt.Errorf("Unexpected status code when reading metadata from %s: %s", metadataUrl, resp.Status)
		glog.V(3).Infof("%v", err)
		return nil, err
	}

	return parseMetadata(resp.Body)
}

// Metadata is fixed for the current host, so cache the value process-wide
var metadataCache *Metadata

func getMetadata() (*Metadata, error) {
	if metadataCache == nil {
		md, err := getMetadataFromConfigDrive()
		if err != nil {
			md, err = getMetadataFromMetadataService()
		}
		if err != nil {
			return nil, err
		}
		metadataCache = md
	}
	return metadataCache, nil
}
<commit_msg>Revert "Revert "Merge pull request #35821 from vishh/gci-mounter-scope""<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package openstack

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/util/exec"
	"k8s.io/kubernetes/pkg/util/mount"
)

// metadataUrl is URL to OpenStack metadata server. It's hardcoded IPv4
// link-local address as documented in "OpenStack Cloud Administrator Guide",
// chapter Compute - Networking with nova-network.
// http://docs.openstack.org/admin-guide-cloud/compute-networking-nova.html#metadata-service
const metadataUrl = "http://169.254.169.254/openstack/2012-08-10/meta_data.json"

// Config drive is defined as an iso9660 or vfat (deprecated) drive
// with the "config-2" label.
// http://docs.openstack.org/user-guide/cli-config-drive.html
const configDriveLabel = "config-2"
const configDrivePath = "openstack/2012-08-10/meta_data.json"

var ErrBadMetadata = errors.New("Invalid OpenStack metadata, got empty uuid")

// Assumes the "2012-08-10" meta_data.json format.
// See http://docs.openstack.org/user-guide/cli_config_drive.html
type Metadata struct {
	Uuid             string `json:"uuid"`
	Name             string `json:"name"`
	AvailabilityZone string `json:"availability_zone"`
	// .. and other fields we don't care about.  Expand as necessary.
}

// parseMetadataUUID reads JSON from OpenStack metadata server and parses
// instance ID out of it.
func parseMetadata(r io.Reader) (*Metadata, error) {
	var metadata Metadata
	json := json.NewDecoder(r)
	if err := json.Decode(&metadata); err != nil {
		return nil, err
	}

	if metadata.Uuid == "" {
		return nil, ErrBadMetadata
	}

	return &metadata, nil
}

func getMetadataFromConfigDrive() (*Metadata, error) {
	// Try to read instance UUID from config drive.
	dev := "/dev/disk/by-label/" + configDriveLabel
	if _, err := os.Stat(dev); os.IsNotExist(err) {
		out, err := exec.New().Command(
			"blkid", "-l",
			"-t", "LABEL="+configDriveLabel,
			"-o", "device",
		).CombinedOutput()
		if err != nil {
			glog.V(2).Infof("Unable to run blkid: %v", err)
			return nil, err
		}
		dev = strings.TrimSpace(string(out))
	}

	mntdir, err := ioutil.TempDir("", "configdrive")
	if err != nil {
		return nil, err
	}
	defer os.Remove(mntdir)

	glog.V(4).Infof("Attempting to mount configdrive %s on %s", dev, mntdir)

	mounter := mount.New("" /* default mount path */)
	err = mounter.Mount(dev, mntdir, "iso9660", []string{"ro"})
	if err != nil {
		err = mounter.Mount(dev, mntdir, "vfat", []string{"ro"})
	}
	if err != nil {
		glog.Errorf("Error mounting configdrive %s: %v", dev, err)
		return nil, err
	}
	defer mounter.Unmount(mntdir)

	glog.V(4).Infof("Configdrive mounted on %s", mntdir)

	f, err := os.Open(
		filepath.Join(mntdir, configDrivePath))
	if err != nil {
		glog.Errorf("Error reading %s on config drive: %v", configDrivePath, err)
		return nil, err
	}
	defer f.Close()

	return parseMetadata(f)
}

func getMetadataFromMetadataService() (*Metadata, error) {
	// Try to get JSON from metdata server.
	glog.V(4).Infof("Attempting to fetch metadata from %s", metadataUrl)
	resp, err := http.Get(metadataUrl)
	if err != nil {
		glog.V(3).Infof("Cannot read %s: %v", metadataUrl, err)
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		err = fmt.Errorf("Unexpected status code when reading metadata from %s: %s", metadataUrl, resp.Status)
		glog.V(3).Infof("%v", err)
		return nil, err
	}

	return parseMetadata(resp.Body)
}

// Metadata is fixed for the current host, so cache the value process-wide
var metadataCache *Metadata

func getMetadata() (*Metadata, error) {
	if metadataCache == nil {
		md, err := getMetadataFromConfigDrive()
		if err != nil {
			md, err = getMetadataFromMetadataService()
		}
		if err != nil {
			return nil, err
		}
		metadataCache = md
	}
	return metadataCache, nil
}
<|endoftext|>


<commit_before>package manta

import (
	"testing"

	"github.com/dotabuff/manta/dota"
	"github.com/stretchr/testify/assert"
)

func TestSep01Match1763193771(t *testing.T) {
	assert := assert.New(t)
	debugLevel = 0
	testLevel = 0

	buf := mustGetReplayData("1763193771", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1763193771.dem")
	parser, err := NewParser(buf)
	assert.Nil(err)

	err = parser.Start()
	assert.Nil(err)
}

func TestSep01Match1763177231(t *testing.T) {
	assert := assert.New(t)
	debugLevel = 0
	testLevel = 0

	buf := mustGetReplayData("1763177231", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1763177231.dem")
	parser, err := NewParser(buf)
	assert.Nil(err)

	err = parser.Start()
	assert.Nil(err)
}

func TestParseOneMatch(t *testing.T) {
	assert := assert.New(t)
	debugLevel = 0
	testLevel = 0

	buf := mustGetReplayData("1731962898", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1731962898.dem")
	parser, err := NewParser(buf)
	assert.Nil(err)

	parser.OnPacketEntity(func(pe *PacketEntity, e EntityEventType) error {
		return nil
	})

	err = parser.Start()
	assert.Nil(err)
}

func TestParseOneMatchWithoutPE(t *testing.T) {
	assert := assert.New(t)
	debugLevel = 0
	testLevel = 0

	buf := mustGetReplayData("1731962898", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1731962898.dem")
	parser, err := NewParser(buf)
	parser.ProcessPacketEntities = false
	assert.Nil(err)
	err = parser.Start()
	assert.Nil(err)
}

// Simply tests that we can read the outer structure of a real match
func TestParseRealMatches(t *testing.T) {
	assert := assert.New(t)

	// Important: make sure to include the most recent test last. These generate fixtures to easily
	// detect diffs in various data structures upon commit, and the latest replay should always be
	// last to provide a most up-to-date baseline.
	scenarios := []struct {
		matchId                string
		replayUrl              string
		expectCombatLogDamage  int32
		expectCombatLogHealing int32
		expectCombatLogDeaths  int32
		expectCombatLogEvents  int32
		expectUnitOrderEvents  int32
	}{
		{
			matchId:                "1763193771",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1763193771.dem",
			expectCombatLogDamage:  0,
			expectCombatLogHealing: 0,
			expectCombatLogDeaths:  0,
			expectCombatLogEvents:  0,
			expectUnitOrderEvents:  0,
		},
		{
			matchId:                "1763177231",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1763177231.dem",
			expectCombatLogDamage:  0,
			expectCombatLogHealing: 0,
			expectCombatLogDeaths:  0,
			expectCombatLogEvents:  0,
			expectUnitOrderEvents:  0,
		},
		{
			matchId:                "1734886116",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1734886116.dem",
			expectCombatLogDamage:  1048805,
			expectCombatLogHealing: 25089,
			expectCombatLogDeaths:  1447,
			expectCombatLogEvents:  42307,
			expectUnitOrderEvents:  59775,
		},
		{
			matchId:                "1731962898",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1731962898.dem",
			expectCombatLogDamage:  415560,
			expectCombatLogHealing: 20018,
			expectCombatLogDeaths:  690,
			expectCombatLogEvents:  24296,
			expectUnitOrderEvents:  27525,
		},
		{
			matchId:                "1605340040",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1605340040.dem",
			expectCombatLogDamage:  522367,
			expectCombatLogHealing: 31721,
			expectCombatLogDeaths:  795,
			expectCombatLogEvents:  21116,
			expectUnitOrderEvents:  40669,
		},
		{
			matchId:                "1560289528",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1560289528.dem",
			expectCombatLogDamage:  1180993,
			expectCombatLogHealing: 57511,
			expectCombatLogDeaths:  1449,
			expectCombatLogEvents:  49146,
			expectUnitOrderEvents:  63387,
		},
		{
			matchId:                "1560294294",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1560294294.dem",
			expectCombatLogDamage:  768154,
			expectCombatLogHealing: 11565,
			expectCombatLogDeaths:  954,
			expectCombatLogEvents:  24535,
			expectUnitOrderEvents:  30657,
		},
		{
			matchId:                "1560315800",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1560315800.dem",
			expectCombatLogDamage:  1332418,
			expectCombatLogHealing: 57874,
			expectCombatLogDeaths:  1645,
			expectCombatLogEvents:  51288,
			expectUnitOrderEvents:  63992,
		},
		{
			matchId:                "1582611189",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1582611189.dem",
			expectCombatLogDamage:  599388,
			expectCombatLogHealing: 28576,
			expectCombatLogDeaths:  930,
			expectCombatLogEvents:  23800,
			expectUnitOrderEvents:  40237,
		},
		{
			matchId:                "1648457986",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1648457986.dem",
			expectCombatLogDamage:  224773,
			expectCombatLogHealing: 5914,
			expectCombatLogDeaths:  466,
			expectCombatLogEvents:  10170,
			expectUnitOrderEvents:  17822,
		},
		{
			matchId:                "1712853372",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1712853372.dem",
			expectCombatLogDamage:  671297,
			expectCombatLogHealing: 23467,
			expectCombatLogDeaths:  1099,
			expectCombatLogEvents:  30381,
			expectUnitOrderEvents:  48107,
		},
		{
			matchId:                "1716444111",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1716444111.dem",
			expectCombatLogDamage:  1398735,
			expectCombatLogHealing: 49659,
			expectCombatLogDeaths:  2169,
			expectCombatLogEvents:  76921,
			expectUnitOrderEvents:  48822,
		},
	}

	for _, s := range scenarios {
		buf := mustGetReplayData(s.matchId, s.replayUrl)
		parser, err := NewParser(buf)
		if err != nil {
			t.Errorf("unable to instantiate parser: %s", err)
			continue
		}

		gotCombatLogDamage := int32(0)
		gotCombatLogHealing := int32(0)
		gotCombatLogDeaths := int32(0)
		gotCombatLogEvents := int32(0)
		gotUnitOrderEvents := int32(0)

		parser.Callbacks.OnCDOTAUserMsg_SpectatorPlayerUnitOrders(func(m *dota.CDOTAUserMsg_SpectatorPlayerUnitOrders) error {
			gotUnitOrderEvents += 1
			return nil
		})

		parser.OnGameEvent("dota_combatlog", func(m *GameEvent) error {
			gotCombatLogEvents += 1

			t, err := m.GetInt32("type")
			assert.Nil(err)

			switch dota.DOTA_COMBATLOG_TYPES(t) {
			case dota.DOTA_COMBATLOG_TYPES_DOTA_COMBATLOG_DAMAGE:
				v, err := m.GetInt32("value")
				assert.Nil(err)
				gotCombatLogDamage += v
			case dota.DOTA_COMBATLOG_TYPES_DOTA_COMBATLOG_DEATH:
				gotCombatLogDeaths += 1
			case dota.DOTA_COMBATLOG_TYPES_DOTA_COMBATLOG_HEAL:
				v, err := m.GetInt32("value")
				assert.Nil(err)
				gotCombatLogHealing += v
			}

			return nil
		})

		if fixturesMode {
			// Writes out the source_1_legacy_game_events_list.json fixture so that we can identify changes to schema.
			parser.Callbacks.OnCMsgSource1LegacyGameEventList(func(m *dota.CMsgSource1LegacyGameEventList) error {
				_dump_fixture("legacy_game_events/list_latest.json", _json_marshal(m))
				return nil
			})
		}

		err = parser.Start()
		assert.Nil(err, s.matchId)

		if fixturesMode {
			// Use this to write out instancebaseline fixtures
			t, _ := parser.StringTables.GetTableByName("instancebaseline")
			for _, i := range t.Items {
				classId, _ := atoi32(i.Key)
				className := parser.ClassInfo[classId]
				_dump_fixture(_sprintf("instancebaseline/%s_%s.rawbuf", className, s.matchId), i.Value)
			}
		}

		assert.Equal(s.expectCombatLogDamage, gotCombatLogDamage, s.matchId)
		assert.Equal(s.expectCombatLogHealing, gotCombatLogHealing, s.matchId)
		assert.Equal(s.expectCombatLogDeaths, gotCombatLogDeaths, s.matchId)
		assert.Equal(s.expectCombatLogEvents, gotCombatLogEvents, s.matchId)
		assert.Equal(s.expectUnitOrderEvents, gotUnitOrderEvents, s.matchId)
	}
}

func BenchmarkParseMatch(b *testing.B) {
	assert := assert.New(b)

	buf := mustGetReplayData("1560315800", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1560315800.dem")

	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		parser, err := NewParser(buf)
		assert.Nil(err)
		err = parser.Start()
		assert.Nil(err)
	}

	b.ReportAllocs()
}
<commit_msg>Enable debug on this test<commit_after>package manta

import (
	"testing"

	"github.com/dotabuff/manta/dota"
	"github.com/stretchr/testify/assert"
)

func TestSep01Match1763193771(t *testing.T) {
	assert := assert.New(t)
	debugLevel = 0
	testLevel = 0

	buf := mustGetReplayData("1763193771", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1763193771.dem")
	parser, err := NewParser(buf)
	assert.Nil(err)

	err = parser.Start()
	assert.Nil(err)
}

func TestSep01Match1763177231(t *testing.T) {
	assert := assert.New(t)
	debugLevel = 10
	testLevel = 0

	buf := mustGetReplayData("1763177231", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1763177231.dem")
	parser, err := NewParser(buf)
	assert.Nil(err)

	err = parser.Start()
	assert.Nil(err)
}

func TestParseOneMatch(t *testing.T) {
	assert := assert.New(t)
	debugLevel = 0
	testLevel = 0

	buf := mustGetReplayData("1731962898", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1731962898.dem")
	parser, err := NewParser(buf)
	assert.Nil(err)

	parser.OnPacketEntity(func(pe *PacketEntity, e EntityEventType) error {
		return nil
	})

	err = parser.Start()
	assert.Nil(err)
}

func TestParseOneMatchWithoutPE(t *testing.T) {
	assert := assert.New(t)
	debugLevel = 0
	testLevel = 0

	buf := mustGetReplayData("1731962898", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1731962898.dem")
	parser, err := NewParser(buf)
	parser.ProcessPacketEntities = false
	assert.Nil(err)
	err = parser.Start()
	assert.Nil(err)
}

// Simply tests that we can read the outer structure of a real match
func TestParseRealMatches(t *testing.T) {
	assert := assert.New(t)

	// Important: make sure to include the most recent test last. These generate fixtures to easily
	// detect diffs in various data structures upon commit, and the latest replay should always be
	// last to provide a most up-to-date baseline.
	scenarios := []struct {
		matchId                string
		replayUrl              string
		expectCombatLogDamage  int32
		expectCombatLogHealing int32
		expectCombatLogDeaths  int32
		expectCombatLogEvents  int32
		expectUnitOrderEvents  int32
	}{
		{
			matchId:                "1763193771",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1763193771.dem",
			expectCombatLogDamage:  0,
			expectCombatLogHealing: 0,
			expectCombatLogDeaths:  0,
			expectCombatLogEvents:  0,
			expectUnitOrderEvents:  0,
		},
		{
			matchId:                "1763177231",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1763177231.dem",
			expectCombatLogDamage:  0,
			expectCombatLogHealing: 0,
			expectCombatLogDeaths:  0,
			expectCombatLogEvents:  0,
			expectUnitOrderEvents:  0,
		},
		{
			matchId:                "1734886116",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1734886116.dem",
			expectCombatLogDamage:  1048805,
			expectCombatLogHealing: 25089,
			expectCombatLogDeaths:  1447,
			expectCombatLogEvents:  42307,
			expectUnitOrderEvents:  59775,
		},
		{
			matchId:                "1731962898",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1731962898.dem",
			expectCombatLogDamage:  415560,
			expectCombatLogHealing: 20018,
			expectCombatLogDeaths:  690,
			expectCombatLogEvents:  24296,
			expectUnitOrderEvents:  27525,
		},
		{
			matchId:                "1605340040",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1605340040.dem",
			expectCombatLogDamage:  522367,
			expectCombatLogHealing: 31721,
			expectCombatLogDeaths:  795,
			expectCombatLogEvents:  21116,
			expectUnitOrderEvents:  40669,
		},
		{
			matchId:                "1560289528",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1560289528.dem",
			expectCombatLogDamage:  1180993,
			expectCombatLogHealing: 57511,
			expectCombatLogDeaths:  1449,
			expectCombatLogEvents:  49146,
			expectUnitOrderEvents:  63387,
		},
		{
			matchId:                "1560294294",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1560294294.dem",
			expectCombatLogDamage:  768154,
			expectCombatLogHealing: 11565,
			expectCombatLogDeaths:  954,
			expectCombatLogEvents:  24535,
			expectUnitOrderEvents:  30657,
		},
		{
			matchId:                "1560315800",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1560315800.dem",
			expectCombatLogDamage:  1332418,
			expectCombatLogHealing: 57874,
			expectCombatLogDeaths:  1645,
			expectCombatLogEvents:  51288,
			expectUnitOrderEvents:  63992,
		},
		{
			matchId:                "1582611189",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1582611189.dem",
			expectCombatLogDamage:  599388,
			expectCombatLogHealing: 28576,
			expectCombatLogDeaths:  930,
			expectCombatLogEvents:  23800,
			expectUnitOrderEvents:  40237,
		},
		{
			matchId:                "1648457986",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1648457986.dem",
			expectCombatLogDamage:  224773,
			expectCombatLogHealing: 5914,
			expectCombatLogDeaths:  466,
			expectCombatLogEvents:  10170,
			expectUnitOrderEvents:  17822,
		},
		{
			matchId:                "1712853372",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1712853372.dem",
			expectCombatLogDamage:  671297,
			expectCombatLogHealing: 23467,
			expectCombatLogDeaths:  1099,
			expectCombatLogEvents:  30381,
			expectUnitOrderEvents:  48107,
		},
		{
			matchId:                "1716444111",
			replayUrl:              "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1716444111.dem",
			expectCombatLogDamage:  1398735,
			expectCombatLogHealing: 49659,
			expectCombatLogDeaths:  2169,
			expectCombatLogEvents:  76921,
			expectUnitOrderEvents:  48822,
		},
	}

	for _, s := range scenarios {
		buf := mustGetReplayData(s.matchId, s.replayUrl)
		parser, err := NewParser(buf)
		if err != nil {
			t.Errorf("unable to instantiate parser: %s", err)
			continue
		}

		gotCombatLogDamage := int32(0)
		gotCombatLogHealing := int32(0)
		gotCombatLogDeaths := int32(0)
		gotCombatLogEvents := int32(0)
		gotUnitOrderEvents := int32(0)

		parser.Callbacks.OnCDOTAUserMsg_SpectatorPlayerUnitOrders(func(m *dota.CDOTAUserMsg_SpectatorPlayerUnitOrders) error {
			gotUnitOrderEvents += 1
			return nil
		})

		parser.OnGameEvent("dota_combatlog", func(m *GameEvent) error {
			gotCombatLogEvents += 1

			t, err := m.GetInt32("type")
			assert.Nil(err)

			switch dota.DOTA_COMBATLOG_TYPES(t) {
			case dota.DOTA_COMBATLOG_TYPES_DOTA_COMBATLOG_DAMAGE:
				v, err := m.GetInt32("value")
				assert.Nil(err)
				gotCombatLogDamage += v
			case dota.DOTA_COMBATLOG_TYPES_DOTA_COMBATLOG_DEATH:
				gotCombatLogDeaths += 1
			case dota.DOTA_COMBATLOG_TYPES_DOTA_COMBATLOG_HEAL:
				v, err := m.GetInt32("value")
				assert.Nil(err)
				gotCombatLogHealing += v
			}

			return nil
		})

		if fixturesMode {
			// Writes out the source_1_legacy_game_events_list.json fixture so that we can identify changes to schema.
			parser.Callbacks.OnCMsgSource1LegacyGameEventList(func(m *dota.CMsgSource1LegacyGameEventList) error {
				_dump_fixture("legacy_game_events/list_latest.json", _json_marshal(m))
				return nil
			})
		}

		err = parser.Start()
		assert.Nil(err, s.matchId)

		if fixturesMode {
			// Use this to write out instancebaseline fixtures
			t, _ := parser.StringTables.GetTableByName("instancebaseline")
			for _, i := range t.Items {
				classId, _ := atoi32(i.Key)
				className := parser.ClassInfo[classId]
				_dump_fixture(_sprintf("instancebaseline/%s_%s.rawbuf", className, s.matchId), i.Value)
			}
		}

		assert.Equal(s.expectCombatLogDamage, gotCombatLogDamage, s.matchId)
		assert.Equal(s.expectCombatLogHealing, gotCombatLogHealing, s.matchId)
		assert.Equal(s.expectCombatLogDeaths, gotCombatLogDeaths, s.matchId)
		assert.Equal(s.expectCombatLogEvents, gotCombatLogEvents, s.matchId)
		assert.Equal(s.expectUnitOrderEvents, gotUnitOrderEvents, s.matchId)
	}
}

func BenchmarkParseMatch(b *testing.B) {
	assert := assert.New(b)

	buf := mustGetReplayData("1560315800", "https://s3-us-west-2.amazonaws.com/manta.dotabuff/1560315800.dem")

	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		parser, err := NewParser(buf)
		assert.Nil(err)
		err = parser.Start()
		assert.Nil(err)
	}

	b.ReportAllocs()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	Pagelimit = 40
)

/*
 * Httpclient object for easier interaction with API
 */
type Httpclient struct {
	Server      string
	Timeout     time.Duration
	Client      http.Client
	Useragent   string
	AccessToken string
}

func (hc *Httpclient) Init() {
	hc.Client = http.Client{
		Timeout: hc.Timeout,
	}
	hc.Useragent = "MastoPurge"
}

func (hc *Httpclient) ApiRequest(method string, endpoint string, params *url.Values) (body []byte, err error) {
	for {
		var req *http.Request
		var err error

		if method == http.MethodPost || method == http.MethodPut {
			req, err = http.NewRequest(method, hc.Server+endpoint, strings.NewReader(params.Encode()))
		} else {
			var paramsEncoded string
			if params != nil {
				paramsEncoded = "?" + params.Encode()
			}
			req, err = http.NewRequest(method, hc.Server+endpoint+paramsEncoded, nil)
		}
		if err != nil {
			log.Fatal(err)
		}
		req.Header.Set("User-Agent", hc.Useragent)
		if hc.AccessToken != "" {
			req.Header.Set("Authorization", "Bearer "+hc.AccessToken)
		}

		res, geterr := hc.Client.Do(req)
		if geterr != nil {
			log.Fatal(geterr)
		}

		body, err = ioutil.ReadAll(res.Body)
		if err != nil {
			log.Fatal(err)
		}

		// Only exit if request was not API rate limited
		if rateLimited(res) == false {
			break
		}
	}

	return body, nil
}

/*
 * Checks if API throttling is active. If yes, wait and repeat http request.
 */
func rateLimited(res *http.Response) (ratelimited bool) {
	if res.StatusCode == 429 {
		ratelimited = true

		var wait_duration time.Duration
		wait_until_time, e_wait_until_time := time.Parse(time.RFC3339, res.Header.Get("X-Ratelimit-Reset"))
		if e_wait_until_time != nil {
			fmt.Println("Cool down time was not defined by server. Waiting for 30 seconds.")
			wait_duration = time.Duration(30) * time.Second
		} else {
			wait_duration = time.Until(wait_until_time)
		}

		fmt.Printf(">>>>>> Server has run hot and is throttling. We have to wait for %d seconds until it has cooled down. Please be patient ...", int(wait_duration.Seconds()))
		time.Sleep(wait_duration)

		fmt.Println("Retrying ...")
	}
	return
}

/*
 * Helper function to easily read input from console
 */
func readFromConsole() (stringstore string) {
	reader := bufio.NewReader(os.Stdin)
	stringstore, _ = reader.ReadString('\n')
	stringstore = strings.Replace(stringstore, "\n", "", -1)
	return
}

/*
 * Various data structs
 */

type MastoPurgeSettings struct {
	Server       string `json:"server"`
	ClientID     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
	AccessToken  string `json:"access_token"`
}

type AccountInfo struct {
	ID       int    `json:"id,string"`
	Username string `json:"username"`
}

type RespAppRegister struct {
	ClientID     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
}

type RespAccessToken struct {
	AccessToken string `json:"access_token"`
}

type Status struct {
	ID uint64 `json:"id,string"`
	//Content     string  `json:"content"`
	CreatedAt time.Time `json:"created_at"`
}

func main() {
	var err error

	/*
	 * Add fancyness!
	 */
	fmt.Println("\nWelcome to ...")
	fmt.Println("                     _                                   ")
	fmt.Println(" _ __ ___   __ _ ___| |_ ___  _ __  _   _ _ __ __ _  ___ ")
	fmt.Println("| '_ ` _ \\ / _` / __| __/ _ \\| '_ \\| | | | '__/ _` |/ _ \\")
	fmt.Println("| | | | | | (_| \\__ \\ || (_) | |_) | |_| | | | (_| |  __/")
	fmt.Println("|_| |_| |_|\\__,_|___/\\__\\___/| .__/ \\__,_|_|  \\__, |\\___|")
	fmt.Println("                             |_|              |___/      ")
	fmt.Println("    ... add German Datenhygiene to your Mastodon-Account!")
	fmt.Print("\n\n")

	log.Println("Version 1.1.0")

	/*
	 * Set up settings and Httpclient
	 */
	settings := MastoPurgeSettings{}
	myhttpclient := Httpclient{}
	myhttpclient.Init()
	myhttpclient.Timeout = time.Second * 5

	/*
	 * Check if configuration file .mastopurgesettings exists
	 * - Create if it does not exist
	 * - Use if exists
	 */
	config_raw, readerr := ioutil.ReadFile(".mastopurgesettings")
	if readerr != nil {
		log.Println("MastoPurge configuration file .mastopurgesettings does not exist or is not accessible.")
		fmt.Println("\nFirst we need to connect MastoPurge to your Mastodon account.")
		fmt.Println("Enter the domain of your Mastodon home instance: (e.g. \"metalhead.club\")")
		fmt.Print("[Mastodon home instance]: ")
		settings.Server = readFromConsole()
		myhttpclient.Server = "https://" + settings.Server

		// Register application for user on server
		log.Println(">>>>>> Registering MastoPurge App on " + settings.Server)
		params := url.Values{}
		params.Add("client_name", "MastoPurge")
		params.Add("redirect_uris", "urn:ietf:wg:oauth:2.0:oob")
		params.Add("scopes", "read write")
		body, registerErr := myhttpclient.ApiRequest(http.MethodPost, "/api/v1/apps", &params)
		if registerErr != nil {
			log.Fatal(registerErr)
		}

		// Parse response JSON
		respAppRegister := RespAppRegister{}
		err = json.Unmarshal(body, &respAppRegister)
		if err != nil {
			log.Fatal(err)
		}
		settings.ClientID = respAppRegister.ClientID
		settings.ClientSecret = respAppRegister.ClientSecret

		// User must manually authenticate app
		authurl := "https://" + settings.Server + "/oauth/authorize?scope=read%20write&response_type=code&redirect_uri=urn:ietf:wg:oauth:2.0:oob&client_id=" + settings.ClientID
		fmt.Println("\n\nPlease visit this URL in your webbrowser:")
		fmt.Println(authurl)
		fmt.Println("\n\n... and enter the code here:")
		fmt.Print("[Auth code]: ")
		code := readFromConsole()

		// Request auth token via auth code ...
		params = url.Values{}
		params.Add("client_id", settings.ClientID)
		params.Add("client_secret", settings.ClientSecret)
		params.Add("grant_type", "authorization_code")
		params.Add("redirect_uri", "urn:ietf:wg:oauth:2.0:oob")
		params.Add("code", code)
		body, err = myhttpclient.ApiRequest(http.MethodPost, "/oauth/token", &params)
		if err != nil {
			log.Fatal(err)
		}

		respAccessToken := RespAccessToken{}
		err = json.Unmarshal(body, &respAccessToken)
		if err != nil {
			log.Fatal(err)
		}
		settings.AccessToken = respAccessToken.AccessToken
		myhttpclient.AccessToken = settings.AccessToken

		// Write settings to config file
		config_raw, _ = json.Marshal(settings)
		err = ioutil.WriteFile(".mastopurgesettings", config_raw, 0600)
		if err != nil {
			log.Fatal(err)
		}

	} else {
		log.Println("MastoPurge configuration found! Reading config.")

		/*
		 * Load settings
		 */
		err = json.Unmarshal(config_raw, &settings)
		if err != nil {
			log.Fatal("Config file is malformed :(\nPlease consider deleting .mastopurgesettings from your file system.")
		}

		myhttpclient.Server = "https://" + settings.Server
		myhttpclient.AccessToken = settings.AccessToken
	}

	/*
	 * Check if account access is okay
	 */

	log.Println("Requesting access to Mastodon account")
	body, accessErr := myhttpclient.ApiRequest(http.MethodGet, "/api/v1/accounts/verify_credentials", nil)
	if accessErr != nil {
		log.Fatal(accessErr)
	} else {
		accountinfo := AccountInfo{}
		err = json.Unmarshal(body, &accountinfo)

		if accountinfo.ID == 0 {
			log.Println("Access DENIED :-(")
			log.Fatal("Unfortunately API access was not granted. Consider deleting the .mastopurgesettings and starting MastoPurge again!")
		} else {
			log.Println("Access GRANTED :-)")
			fmt.Println(">>> Account ID:", accountinfo.ID)
			fmt.Println(">>> Username:", accountinfo.Username)

			// Do some date calculations ...
			var maxage time.Duration
			var maxtime time.Time

			for {
				fmt.Println("\nEnter the maximum age of the posts you want to KEEP, e.g. \"30 days\". Older posts will be deleted. Allowed units: hours, days, weeks, months.")
				fmt.Print("[Maximum post age]: ")
				maxagestring := readFromConsole()
				parts := strings.Split(maxagestring, " ")

				if len(parts) == 2 {
					maxagenum, converr := strconv.Atoi(parts[0])
					if converr == nil {
						var factor time.Duration

						switch parts[1] {
						case "hours":
							factor = time.Hour
						case "days":
							factor = time.Hour * time.Duration(24)
						case "weeks":
							factor = time.Hour * time.Duration(24*7)
						case "months":
							factor = time.Hour * time.Duration(24*30)
						default:
							factor = 0
						}

						if factor != 0 {
							maxage = factor * time.Duration(maxagenum)
							break
						}
					}
				}
				fmt.Println("Error: Invalid age format.")
			}

			maxtime = time.Now().Add(-maxage)
			fmt.Println("Okay, let's do it! ")
			fmt.Println("Posts older than", maxtime, "will be deleted!")
			fmt.Print("Loading gun ")
			for i := 0; i < 40; i++ {
				fmt.Print(".")
				time.Sleep(time.Duration(50) * time.Millisecond)
			}
			time.Sleep(time.Duration(2) * time.Second)
			fmt.Print("\n\n")

			var maxid uint64 = 0
			var deletedcount uint16

			// Fetch new pages until there are no more pages
			for {
				log.Printf("========== Fetching new statuses until status %d ==========\n", maxid)

				nodeletions := true

				// Fetch posts
				params := url.Values{}
				params.Add("limit", strconv.Itoa(Pagelimit))
				if maxid != 0 {
					params.Add("max_id", fmt.Sprint(maxid))
				}
				resp, fetchErr := myhttpclient.ApiRequest(http.MethodGet, "/api/v1/accounts/"+strconv.Itoa(accountinfo.ID)+"/statuses", &params)
				if fetchErr != nil {
					log.Fatal(fetchErr)
				}

				var statuses []Status
				err = json.Unmarshal(resp, &statuses)
				if err != nil {
					// Maybe server response is an error message?
					fmt.Println(string(resp))
					log.Fatal(err)
				}

				// Exit killer loop if there are no more statuses
				if len(statuses) == 0 {
					break
				}

				for _, status := range statuses {
					// Parse time
					if status.CreatedAt.Before(maxtime) {
						// Delete post
						nodeletions = false
						delResp, delErr := myhttpclient.ApiRequest(http.MethodDelete, "/api/v1/statuses/"+fmt.Sprint(status.ID), nil)
						if delErr != nil {
							log.Println("!!! Could not delete status " + fmt.Sprint(status.ID) + " !!!")
						}

						if string(delResp) == "{}" {
							//log.Println("Status " + fmt.Sprint(status.ID) + " successfully deleted!")
							deletedcount++
						} else {
							log.Println("Status " + fmt.Sprint(status.ID) + " could not be deleted :(")
						}
					}

					if status.ID < maxid || maxid == 0 {
						maxid = status.ID - 1
					}
				}

				if nodeletions {
					log.Println("No posts to be deleted on this page. Trying next page ...")
				} else {
					log.Println(deletedcount, "statuses deleted.")
					// Wait before fetching a new page. Give server time to re-assemble pages.
					time.Sleep(time.Duration(1) * time.Second)
				}
			}

			// No more pages, done! :-)
			fmt.Println(">>>>>> ", deletedcount, "statuses were successfully deleted.")
		}
	}
}
<commit_msg>Refactor rateLimited<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	Pagelimit = 40
)

/*
 * Httpclient object for easier interaction with API
 */
type Httpclient struct {
	Server      string
	Timeout     time.Duration
	Client      http.Client
	Useragent   string
	AccessToken string
}

func (hc *Httpclient) Init() {
	hc.Client = http.Client{
		Timeout: hc.Timeout,
	}
	hc.Useragent = "MastoPurge"
}

func (hc *Httpclient) ApiRequest(method string, endpoint string, params *url.Values) (body []byte, err error) {
	for {
		var req *http.Request
		var err error

		if method == http.MethodPost || method == http.MethodPut {
			req, err = http.NewRequest(method, hc.Server+endpoint, strings.NewReader(params.Encode()))
		} else {
			var paramsEncoded string
			if params != nil {
				paramsEncoded = "?" + params.Encode()
			}
			req, err = http.NewRequest(method, hc.Server+endpoint+paramsEncoded, nil)
		}
		if err != nil {
			log.Fatal(err)
		}
		req.Header.Set("User-Agent", hc.Useragent)
		if hc.AccessToken != "" {
			req.Header.Set("Authorization", "Bearer "+hc.AccessToken)
		}

		res, geterr := hc.Client.Do(req)
		if geterr != nil {
			log.Fatal(geterr)
		}

		body, err = ioutil.ReadAll(res.Body)
		if err != nil {
			log.Fatal(err)
		}

		// Only exit if request was not API rate limited
		if rateLimited(res) == false {
			break
		}
	}

	return body, nil
}

// rateLimited checks if API throttling is active. If yes, it waits the time
// defined by the server (or a default of 30 seconds) and repeats the http
// request. Returns whether the request was rate limited.
func rateLimited(res *http.Response) bool {
	// request was not rate limited - nothing to do.
	if res.StatusCode != 429 {
		return false
	}

	var waitDuration time.Duration
	waitUntil, err := time.Parse(time.RFC3339, res.Header.Get("X-Ratelimit-Reset"))
	if err != nil {
		fmt.Println("Cool down time was not defined by server. Waiting for 30 seconds.")
		waitDuration = 30 * time.Second
	} else {
		waitDuration = time.Until(waitUntil)
	}

	fmt.Printf(">>>>>> Server has run hot and is throttling. We have to wait for %d seconds until it has cooled down. Please be patient ...\n", int(waitDuration.Seconds()))
	time.Sleep(waitDuration)

	fmt.Println("Retrying ...")
	return true
}

/*
 * Helper function to easily read input from console
 */
func readFromConsole() (stringstore string) {
	reader := bufio.NewReader(os.Stdin)
	stringstore, _ = reader.ReadString('\n')
	stringstore = strings.Replace(stringstore, "\n", "", -1)
	return
}

/*
 * Various data structs
 */

type MastoPurgeSettings struct {
	Server       string `json:"server"`
	ClientID     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
	AccessToken  string `json:"access_token"`
}

type AccountInfo struct {
	ID       int    `json:"id,string"`
	Username string `json:"username"`
}

type RespAppRegister struct {
	ClientID     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
}

type RespAccessToken struct {
	AccessToken string `json:"access_token"`
}

type Status struct {
	ID uint64 `json:"id,string"`
	//Content     string  `json:"content"`
	CreatedAt time.Time `json:"created_at"`
}

func main() {
	var err error

	/*
	 * Add fancyness!
	 */
	fmt.Println("\nWelcome to ...")
	fmt.Println("                     _                                   ")
	fmt.Println(" _ __ ___   __ _ ___| |_ ___  _ __  _   _ _ __ __ _  ___ ")
	fmt.Println("| '_ ` _ \\ / _` / __| __/ _ \\| '_ \\| | | | '__/ _` |/ _ \\")
	fmt.Println("| | | | | | (_| \\__ \\ || (_) | |_) | |_| | | | (_| |  __/")
	fmt.Println("|_| |_| |_|\\__,_|___/\\__\\___/| .__/ \\__,_|_|  \\__, |\\___|")
	fmt.Println("                             |_|              |___/      ")
	fmt.Println("    ... add German Datenhygiene to your Mastodon-Account!")
	fmt.Print("\n\n")

	log.Println("Version 1.1.0")

	/*
	 * Set up settings and Httpclient
	 */
	settings := MastoPurgeSettings{}
	myhttpclient := Httpclient{}
	myhttpclient.Init()
	myhttpclient.Timeout = time.Second * 5

	/*
	 * Check if configuration file .mastopurgesettings exists
	 * - Create if it does not exist
	 * - Use if exists
	 */
	config_raw, readerr := ioutil.ReadFile(".mastopurgesettings")
	if readerr != nil {
		log.Println("MastoPurge configuration file .mastopurgesettings does not exist or is not accessible.")
		fmt.Println("\nFirst we need to connect MastoPurge to your Mastodon account.")
		fmt.Println("Enter the domain of your Mastodon home instance: (e.g. \"metalhead.club\")")
		fmt.Print("[Mastodon home instance]: ")
		settings.Server = readFromConsole()
		myhttpclient.Server = "https://" + settings.Server

		// Register application for user on server
		log.Println(">>>>>> Registering MastoPurge App on " + settings.Server)
		params := url.Values{}
		params.Add("client_name", "MastoPurge")
		params.Add("redirect_uris", "urn:ietf:wg:oauth:2.0:oob")
		params.Add("scopes", "read write")
		body, registerErr := myhttpclient.ApiRequest(http.MethodPost, "/api/v1/apps", &params)
		if registerErr != nil {
			log.Fatal(registerErr)
		}

		// Parse response JSON
		respAppRegister := RespAppRegister{}
		err = json.Unmarshal(body, &respAppRegister)
		if err != nil {
			log.Fatal(err)
		}
		settings.ClientID = respAppRegister.ClientID
		settings.ClientSecret = respAppRegister.ClientSecret

		// User must manually authenticate app
		authurl := "https://" + settings.Server + "/oauth/authorize?scope=read%20write&response_type=code&redirect_uri=urn:ietf:wg:oauth:2.0:oob&client_id=" + settings.ClientID
		fmt.Println("\n\nPlease visit this URL in your webbrowser:")
		fmt.Println(authurl)
		fmt.Println("\n\n... and enter the code here:")
		fmt.Print("[Auth code]: ")
		code := readFromConsole()

		// Request auth token via auth code ...
		params = url.Values{}
		params.Add("client_id", settings.ClientID)
		params.Add("client_secret", settings.ClientSecret)
		params.Add("grant_type", "authorization_code")
		params.Add("redirect_uri", "urn:ietf:wg:oauth:2.0:oob")
		params.Add("code", code)
		body, err = myhttpclient.ApiRequest(http.MethodPost, "/oauth/token", &params)
		if err != nil {
			log.Fatal(err)
		}

		respAccessToken := RespAccessToken{}
		err = json.Unmarshal(body, &respAccessToken)
		if err != nil {
			log.Fatal(err)
		}
		settings.AccessToken = respAccessToken.AccessToken
		myhttpclient.AccessToken = settings.AccessToken

		// Write settings to config file
		config_raw, _ = json.Marshal(settings)
		err = ioutil.WriteFile(".mastopurgesettings", config_raw, 0600)
		if err != nil {
			log.Fatal(err)
		}

	} else {
		log.Println("MastoPurge configuration found! Reading config.")

		/*
		 * Load settings
		 */
		err = json.Unmarshal(config_raw, &settings)
		if err != nil {
			log.Fatal("Config file is malformed :(\nPlease consider deleting .mastopurgesettings from your file system.")
		}

		myhttpclient.Server = "https://" + settings.Server
		myhttpclient.AccessToken = settings.AccessToken
	}

	/*
	 * Check if account access is okay
	 */

	log.Println("Requesting access to Mastodon account")
	body, accessErr := myhttpclient.ApiRequest(http.MethodGet, "/api/v1/accounts/verify_credentials", nil)
	if accessErr != nil {
		log.Fatal(accessErr)
	} else {
		accountinfo := AccountInfo{}
		err = json.Unmarshal(body, &accountinfo)

		if accountinfo.ID == 0 {
			log.Println("Access DENIED :-(")
			log.Fatal("Unfortunately API access was not granted. Consider deleting the .mastopurgesettings and starting MastoPurge again!")
		} else {
			log.Println("Access GRANTED :-)")
			fmt.Println(">>> Account ID:", accountinfo.ID)
			fmt.Println(">>> Username:", accountinfo.Username)

			// Do some date calculations ...
			var maxage time.Duration
			var maxtime time.Time

			for {
				fmt.Println("\nEnter the maximum age of the posts you want to KEEP, e.g. \"30 days\". Older posts will be deleted. Allowed units: hours, days, weeks, months.")
				fmt.Print("[Maximum post age]: ")
				maxagestring := readFromConsole()
				parts := strings.Split(maxagestring, " ")

				if len(parts) == 2 {
					maxagenum, converr := strconv.Atoi(parts[0])
					if converr == nil {
						var factor time.Duration

						switch parts[1] {
						case "hours":
							factor = time.Hour
						case "days":
							factor = time.Hour * time.Duration(24)
						case "weeks":
							factor = time.Hour * time.Duration(24*7)
						case "months":
							factor = time.Hour * time.Duration(24*30)
						default:
							factor = 0
						}

						if factor != 0 {
							maxage = factor * time.Duration(maxagenum)
							break
						}
					}
				}
				fmt.Println("Error: Invalid age format.")
			}

			maxtime = time.Now().Add(-maxage)
			fmt.Println("Okay, let's do it! ")
			fmt.Println("Posts older than", maxtime, "will be deleted!")
			fmt.Print("Loading gun ")
			for i := 0; i < 40; i++ {
				fmt.Print(".")
				time.Sleep(time.Duration(50) * time.Millisecond)
			}
			time.Sleep(time.Duration(2) * time.Second)
			fmt.Print("\n\n")

			var maxid uint64 = 0
			var deletedcount uint16

			// Fetch new pages until there are no more pages
			for {
				log.Printf("========== Fetching new statuses until status %d ==========\n", maxid)

				nodeletions := true

				// Fetch posts
				params := url.Values{}
				params.Add("limit", strconv.Itoa(Pagelimit))
				if maxid != 0 {
					params.Add("max_id", fmt.Sprint(maxid))
				}
				resp, fetchErr := myhttpclient.ApiRequest(http.MethodGet, "/api/v1/accounts/"+strconv.Itoa(accountinfo.ID)+"/statuses", &params)
				if fetchErr != nil {
					log.Fatal(fetchErr)
				}

				var statuses []Status
				err = json.Unmarshal(resp, &statuses)
				if err != nil {
					// Maybe server response is an error message?
					fmt.Println(string(resp))
					log.Fatal(err)
				}

				// Exit killer loop if there are no more statuses
				if len(statuses) == 0 {
					break
				}

				for _, status := range statuses {
					// Parse time
					if status.CreatedAt.Before(maxtime) {
						// Delete post
						nodeletions = false
						delResp, delErr := myhttpclient.ApiRequest(http.MethodDelete, "/api/v1/statuses/"+fmt.Sprint(status.ID), nil)
						if delErr != nil {
							log.Println("!!! Could not delete status " + fmt.Sprint(status.ID) + " !!!")
						}

						if string(delResp) == "{}" {
							//log.Println("Status " + fmt.Sprint(status.ID) + " successfully deleted!")
							deletedcount++
						} else {
							log.Println("Status " + fmt.Sprint(status.ID) + " could not be deleted :(")
						}
					}

					if status.ID < maxid || maxid == 0 {
						maxid = status.ID - 1
					}
				}

				if nodeletions {
					log.Println("No posts to be deleted on this page. Trying next page ...")
				} else {
					log.Println(deletedcount, "statuses deleted.")
					// Wait before fetching a new page. Give server time to re-assemble pages.
					time.Sleep(time.Duration(1) * time.Second)
				}
			}

			// No more pages, done! :-)
			fmt.Println(">>>>>> ", deletedcount, "statuses were successfully deleted.")
		}
	}
}
<|endoftext|>


<commit_before>package mock

// MockBot is a type that implements noye.Bot
// which can have parts of it overridden by changing the functors
type MockBot struct {
	SendFn    func(string, ...interface{})
	PrivmsgFn func(string, string)
	JoinFn    func(string)
	PartFn    func(string)
	QuitFn    func()
}

// NewMockBot returns a new MockBot with the fns set to no-ops
func NewMockBot() *MockBot {
	return &MockBot{
		func(string, ...interface{}) {},
		func(string, string) {},
		func(string) {},
		func(string) {},
		func() {},
	}
}

// Dial is just here for the interface
func (m *MockBot) Dial(addr, nick, user string) error { return nil }

// Close is just here for the interface
func (m *MockBot) Close() {}

// Send delegates to SendFn
func (m *MockBot) Send(f string, a ...interface{}) { m.SendFn(f, a) }

// Privmsg delegates to PrivmsgFn
func (m *MockBot) Privmsg(target, msg string) { m.PrivmsgFn(target, msg) }

// Join delegates to JoinFn
func (m *MockBot) Join(target string) { m.JoinFn(target) }

// Part delegates to PartFn
func (m *MockBot) Part(target string) { m.PartFn(target) }

// Quit delegates to QuitFn
func (m *MockBot) Part() { m.QuitFn() }

// Wait is just here for the interface
func (m *MockBot) Wait() <-chan struct{} { return nil }

// Ready is just here for the interface
func (m *MockBot) Ready() <-chan struct{} { return nil }
<commit_msg>fixed error in mock box, part->quit<commit_after>package mock

// MockBot is a type that implements noye.Bot
// which can have parts of it overridden by changing the functors
type MockBot struct {
	SendFn    func(string, ...interface{})
	PrivmsgFn func(string, string)
	JoinFn    func(string)
	PartFn    func(string)
	QuitFn    func()
}

// NewMockBot returns a new MockBot with the fns set to no-ops
func NewMockBot() *MockBot {
	return &MockBot{
		func(string, ...interface{}) {},
		func(string, string) {},
		func(string) {},
		func(string) {},
		func() {},
	}
}

// Dial is just here for the interface
func (m *MockBot) Dial(addr, nick, user string) error { return nil }

// Close is just here for the interface
func (m *MockBot) Close() {}

// Send delegates to SendFn
func (m *MockBot) Send(f string, a ...interface{}) { m.SendFn(f, a) }

// Privmsg delegates to PrivmsgFn
func (m *MockBot) Privmsg(target, msg string) { m.PrivmsgFn(target, msg) }

// Join delegates to JoinFn
func (m *MockBot) Join(target string) { m.JoinFn(target) }

// Part delegates to PartFn
func (m *MockBot) Part(target string) { m.PartFn(target) }

// Quit delegates to QuitFn
func (m *MockBot) Quit() { m.QuitFn() }

// Wait is just here for the interface
func (m *MockBot) Wait() <-chan struct{} { return nil }

// Ready is just here for the interface
func (m *MockBot) Ready() <-chan struct{} { return nil }
<|endoftext|>