<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fs_test

import (
	. "io/fs"
	"os"
	pathpkg "path"
	"path/filepath"
	"reflect"
	"testing"
	"testing/fstest"
)

type Node struct {
	name    string
	entries []*Node // nil if the entry is a file
	mark    int
}

var tree = &Node{
	"testdata",
	[]*Node{
		{"a", nil, 0},
		{"b", []*Node{}, 0},
		{"c", nil, 0},
		{
			"d",
			[]*Node{
				{"x", nil, 0},
				{"y", []*Node{}, 0},
				{
					"z",
					[]*Node{
						{"u", nil, 0},
						{"v", nil, 0},
					},
					0,
				},
			},
			0,
		},
	},
	0,
}

func walkTree(n *Node, path string, f func(path string, n *Node)) {
	f(path, n)
	for _, e := range n.entries {
		walkTree(e, pathpkg.Join(path, e.name), f)
	}
}

func makeTree(t *testing.T) FS {
	fsys := fstest.MapFS{}
	walkTree(tree, tree.name, func(path string, n *Node) {
		if n.entries == nil {
			fsys[path] = &fstest.MapFile{}
		} else {
			fsys[path] = &fstest.MapFile{Mode: ModeDir}
		}
	})
	return fsys
}

func markTree(n *Node) { walkTree(n, "", func(path string, n *Node) { n.mark++ }) }

func checkMarks(t *testing.T, report bool) {
	walkTree(tree, tree.name, func(path string, n *Node) {
		if n.mark != 1 && report {
			t.Errorf("node %s mark = %d; expected 1", path, n.mark)
		}
		n.mark = 0
	})
}

// Assumes that each node name is unique. Good enough for a test.
// If clear is true, any incoming error is cleared before return. The errors
// are always accumulated, though.
func mark(entry DirEntry, err error, errors *[]error, clear bool) error {
	name := entry.Name()
	walkTree(tree, tree.name, func(path string, n *Node) {
		if n.name == name {
			n.mark++
		}
	})
	if err != nil {
		*errors = append(*errors, err)
		if clear {
			return nil
		}
		return err
	}
	return nil
}

func TestWalkDir(t *testing.T) {
	tmpDir := t.TempDir()

	origDir, err := os.Getwd()
	if err != nil {
		t.Fatal("finding working dir:", err)
	}
	if err = os.Chdir(tmpDir); err != nil {
		t.Fatal("entering temp dir:", err)
	}
	defer os.Chdir(origDir)

	fsys := makeTree(t)
	errors := make([]error, 0, 10)
	clear := true
	markFn := func(path string, entry DirEntry, err error) error {
		return mark(entry, err, &errors, clear)
	}
	// Expect no errors.
	err = WalkDir(fsys, ".", markFn)
	if err != nil {
		t.Fatalf("no error expected, found: %s", err)
	}
	if len(errors) != 0 {
		t.Fatalf("unexpected errors: %s", errors)
	}
	checkMarks(t, true)
}

func TestIssue51617(t *testing.T) {
	dir := t.TempDir()
	for _, sub := range []string{"a", filepath.Join("a", "bad"), filepath.Join("a", "next")} {
		if err := os.Mkdir(filepath.Join(dir, sub), 0755); err != nil {
			t.Fatal(err)
		}
	}
	bad := filepath.Join(dir, "a", "bad")
	if err := os.Chmod(bad, 0); err != nil {
		t.Fatal(err)
	}
	defer os.Chmod(bad, 0700) // avoid errors on cleanup
	var saw []string
	err := WalkDir(os.DirFS(dir), ".", func(path string, d DirEntry, err error) error {
		if err != nil {
			return filepath.SkipDir
		}
		if d.IsDir() {
			saw = append(saw, path)
		}
		return nil
	})
	if err != nil {
		t.Fatal(err)
	}
	want := []string{".", "a", "a/bad", "a/next"}
	if !reflect.DeepEqual(saw, want) {
		t.Errorf("got directories %v, want %v", saw, want)
	}
}
<commit_msg>io/fs: clean up test helper functions<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fs_test

import (
	. "io/fs"
	"os"
	pathpkg "path"
	"path/filepath"
	"reflect"
	"testing"
	"testing/fstest"
)

type Node struct {
	name    string
	entries []*Node // nil if the entry is a file
	mark    int
}

var tree = &Node{
	"testdata",
	[]*Node{
		{"a", nil, 0},
		{"b", []*Node{}, 0},
		{"c", nil, 0},
		{
			"d",
			[]*Node{
				{"x", nil, 0},
				{"y", []*Node{}, 0},
				{
					"z",
					[]*Node{
						{"u", nil, 0},
						{"v", nil, 0},
					},
					0,
				},
			},
			0,
		},
	},
	0,
}

func walkTree(n *Node, path string, f func(path string, n *Node)) {
	f(path, n)
	for _, e := range n.entries {
		walkTree(e, pathpkg.Join(path, e.name), f)
	}
}

func makeTree() FS {
	fsys := fstest.MapFS{}
	walkTree(tree, tree.name, func(path string, n *Node) {
		if n.entries == nil {
			fsys[path] = &fstest.MapFile{}
		} else {
			fsys[path] = &fstest.MapFile{Mode: ModeDir}
		}
	})
	return fsys
}

// Assumes that each node name is unique. Good enough for a test.
// If clear is true, any incoming error is cleared before return. The errors
// are always accumulated, though.
func mark(entry DirEntry, err error, errors *[]error, clear bool) error {
	name := entry.Name()
	walkTree(tree, tree.name, func(path string, n *Node) {
		if n.name == name {
			n.mark++
		}
	})
	if err != nil {
		*errors = append(*errors, err)
		if clear {
			return nil
		}
		return err
	}
	return nil
}

func TestWalkDir(t *testing.T) {
	tmpDir := t.TempDir()

	origDir, err := os.Getwd()
	if err != nil {
		t.Fatal("finding working dir:", err)
	}
	if err = os.Chdir(tmpDir); err != nil {
		t.Fatal("entering temp dir:", err)
	}
	defer os.Chdir(origDir)

	fsys := makeTree()
	errors := make([]error, 0, 10)
	clear := true
	markFn := func(path string, entry DirEntry, err error) error {
		return mark(entry, err, &errors, clear)
	}
	// Expect no errors.
	err = WalkDir(fsys, ".", markFn)
	if err != nil {
		t.Fatalf("no error expected, found: %s", err)
	}
	if len(errors) != 0 {
		t.Fatalf("unexpected errors: %s", errors)
	}
	walkTree(tree, tree.name, func(path string, n *Node) {
		if n.mark != 1 {
			t.Errorf("node %s mark = %d; expected 1", path, n.mark)
		}
		n.mark = 0
	})
}

func TestIssue51617(t *testing.T) {
	dir := t.TempDir()
	for _, sub := range []string{"a", filepath.Join("a", "bad"), filepath.Join("a", "next")} {
		if err := os.Mkdir(filepath.Join(dir, sub), 0755); err != nil {
			t.Fatal(err)
		}
	}
	bad := filepath.Join(dir, "a", "bad")
	if err := os.Chmod(bad, 0); err != nil {
		t.Fatal(err)
	}
	defer os.Chmod(bad, 0700) // avoid errors on cleanup
	var saw []string
	err := WalkDir(os.DirFS(dir), ".", func(path string, d DirEntry, err error) error {
		if err != nil {
			return filepath.SkipDir
		}
		if d.IsDir() {
			saw = append(saw, path)
		}
		return nil
	})
	if err != nil {
		t.Fatal(err)
	}
	want := []string{".", "a", "a/bad", "a/next"}
	if !reflect.DeepEqual(saw, want) {
		t.Errorf("got directories %v, want %v", saw, want)
	}
}
<|endoftext|>


<commit_before>package main

import "bufio"
import "fmt"
import "math/rand"
import "net"
import "time"
import "sync"

type CacheItem struct {
    key string
    value string
}

// constants and configuration
// No constant arrays
var letters = []rune("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
const verbose = false
const numThreads = 10

// 4-character keys with 26 possibilities = 456,976 keys
// 5-character keys with 26 possibilities = 11,881,376 keys
const keyLength = 4
//const numKeys = 1000000
const numKeys = 100000

// concurrency bits
var wg sync.WaitGroup
var tasks chan *CacheItem

func main() {

    rand.Seed(time.Now().UTC().UnixNano())
    tasks = make(chan *CacheItem, 64)

    // spawn worker goroutines
    for i := 0; i < numThreads; i++ {
        wg.Add(1)
        go worker(connect("localhost"))
    }

    key := ""
    value := ""
    valLen := 0

    for i := 0; i < numKeys; i++ {
        key = randString(keyLength)

        // Random length between 1k and 10k
        valLen = rand.Intn(9 * 1024) + 1024
        value = randString(valLen)

        item := new(CacheItem)
        item.key = key
        item.value = value

        tasks <- item

        if i % 10000 == 0 {
            fmt.Printf("%v\r\n", i)
        }
    }

    close(tasks)
    wg.Wait()
}

func worker(conn net.Conn) {
    for item := range tasks {
        set(conn, item.key, item.value)
        //get(conn, item.key)
    }

    wg.Done()
}

func randString(n int) string {
    b := make([]rune, n)

    for i := range b {
        b[i] = letters[rand.Intn(len(letters))]
    }

    return string(b)
}

func connect(host string) net.Conn {
    conn, err := net.Dial("tcp", host + ":11212")

    if err != nil {
        panic(err)
    }

    println("Connected to memcached.")

    return conn
}

func set(conn net.Conn, key string, value string) {
    if verbose { println(fmt.Sprintf("Setting key %v to value of length %v", key, len(value))) }

    fmt.Fprintf(conn, "set %v 0 0 %v\r\n", key, len(value))
    fmt.Fprintf(conn, "%v\r\n", value)
    response, err := bufio.NewReader(conn).ReadString('\n')

    if err != nil { panic(err) }

    if verbose { print(response) }
}

func get(conn net.Conn, key string) {
    if verbose { println(fmt.Sprintf("Getting key %v", key)) }

    fmt.Fprintf(conn, "get %v\r\n", key)

    reader := bufio.NewReader(conn)

    // read the header line
    response, err := reader.ReadString('\n')

    if err != nil { panic(err) }

    if verbose { print(response) }

    // then read the value
    response, err = reader.ReadString('\n')

    if err != nil { panic(err) }

    //print(response)
    if verbose { println("(read the value)") }

    // then read the END
    response, err = reader.ReadString('\n')

    if err != nil { panic(err) }

    if verbose { print(response) }
}
<commit_msg>Re-adding gets in test program<commit_after>package main

import "bufio"
import "fmt"
import "math/rand"
import "net"
import "time"
import "sync"

type CacheItem struct {
    key string
    value string
}

// constants and configuration
// No constant arrays
var letters = []rune("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
const verbose = false
const numThreads = 10

// 4-character keys with 26 possibilities = 456,976 keys
// 5-character keys with 26 possibilities = 11,881,376 keys
const keyLength = 4
//const numKeys = 1000000
const numKeys = 100000

// concurrency bits
var wg sync.WaitGroup
var tasks chan *CacheItem

func main() {

    rand.Seed(time.Now().UTC().UnixNano())
    tasks = make(chan *CacheItem, 64)

    // spawn worker goroutines
    for i := 0; i < numThreads; i++ {
        wg.Add(1)
        go worker(connect("localhost"))
    }

    key := ""
    value := ""
    valLen := 0

    for i := 0; i < numKeys; i++ {
        key = randString(keyLength)

        // Random length between 1k and 10k
        valLen = rand.Intn(9 * 1024) + 1024
        value = randString(valLen)

        item := new(CacheItem)
        item.key = key
        item.value = value

        tasks <- item

        if i % 10000 == 0 {
            fmt.Printf("%v\r\n", i)
        }
    }

    close(tasks)
    wg.Wait()
}

func worker(conn net.Conn) {
    for item := range tasks {
        set(conn, item.key, item.value)
        get(conn, item.key)
    }

    wg.Done()
}

func randString(n int) string {
    b := make([]rune, n)

    for i := range b {
        b[i] = letters[rand.Intn(len(letters))]
    }

    return string(b)
}

func connect(host string) net.Conn {
    conn, err := net.Dial("tcp", host + ":11212")

    if err != nil {
        panic(err)
    }

    println("Connected to memcached.")

    return conn
}

func set(conn net.Conn, key string, value string) {
    if verbose { println(fmt.Sprintf("Setting key %v to value of length %v", key, len(value))) }

    fmt.Fprintf(conn, "set %v 0 0 %v\r\n", key, len(value))
    fmt.Fprintf(conn, "%v\r\n", value)
    response, err := bufio.NewReader(conn).ReadString('\n')

    if err != nil { panic(err) }

    if verbose { print(response) }
}

func get(conn net.Conn, key string) {
    if verbose { println(fmt.Sprintf("Getting key %v", key)) }

    fmt.Fprintf(conn, "get %v\r\n", key)

    reader := bufio.NewReader(conn)

    // read the header line
    response, err := reader.ReadString('\n')

    if err != nil { panic(err) }

    if verbose { print(response) }

    // then read the value
    response, err = reader.ReadString('\n')

    if err != nil { panic(err) }

    //print(response)
    if verbose { println("(read the value)") }

    // then read the END
    response, err = reader.ReadString('\n')

    if err != nil { panic(err) }

    if verbose { print(response) }
}
<|endoftext|>


<commit_before>package kafka

import (
	"time"
)

// Message is a data structure representing kafka messages.
type Message struct {
	// Topic indicates which topic this message was consumed from via Reader.
	//
	// When being used with Writer, this can be used to configured the topic if
	// not already specified on the writer itself.
	Topic string

	// Partition is read-only and MUST NOT be set when writing messages
	Partition     int
	Offset        int64
	HighWaterMark int64
	Key           []byte
	Value         []byte
	Headers       []Header

	// If not set at the creation, Time will be automatically set when
	// writing the message.
	Time time.Time
}

func (msg Message) message(cw *crc32Writer) message {
	m := message{
		MagicByte: 1,
		Key:       msg.Key,
		Value:     msg.Value,
		Timestamp: timestamp(msg.Time),
	}
	if cw != nil {
		m.CRC = m.crc32(cw)
	}
	return m
}

const timestampSize = 8

func (msg *Message) size() int32 {
	return 4 + 1 + 1 + sizeofBytes(msg.Key) + sizeofBytes(msg.Value) + timestampSize
}

type message struct {
	CRC        int32
	MagicByte  int8
	Attributes int8
	Timestamp  int64
	Key        []byte
	Value      []byte
}

func (m message) crc32(cw *crc32Writer) int32 {
	cw.crc32 = 0
	cw.writeInt8(m.MagicByte)
	cw.writeInt8(m.Attributes)
	if m.MagicByte != 0 {
		cw.writeInt64(m.Timestamp)
	}
	cw.writeBytes(m.Key)
	cw.writeBytes(m.Value)
	return int32(cw.crc32)
}

func (m message) size() int32 {
	size := 4 + 1 + 1 + sizeofBytes(m.Key) + sizeofBytes(m.Value)
	if m.MagicByte != 0 {
		size += timestampSize
	}
	return size
}

func (m message) writeTo(wb *writeBuffer) {
	wb.writeInt32(m.CRC)
	wb.writeInt8(m.MagicByte)
	wb.writeInt8(m.Attributes)
	if m.MagicByte != 0 {
		wb.writeInt64(m.Timestamp)
	}
	wb.writeBytes(m.Key)
	wb.writeBytes(m.Value)
}

type messageSetItem struct {
	Offset      int64
	MessageSize int32
	Message     message
}

func (m messageSetItem) size() int32 {
	return 8 + 4 + m.Message.size()
}

func (m messageSetItem) writeTo(wb *writeBuffer) {
	wb.writeInt64(m.Offset)
	wb.writeInt32(m.MessageSize)
	m.Message.writeTo(wb)
}

type messageSet []messageSetItem

func (s messageSet) size() (size int32) {
	for _, m := range s {
		size += m.size()
	}
	return
}

func (s messageSet) writeTo(wb *writeBuffer) {
	for _, m := range s {
		m.writeTo(wb)
	}
}

type timestampType int8

const (
	createTime    timestampType = 0
	logAppendTime timestampType = 1
)

type transactionType int8

const (
	nonTransactional transactionType = 0
	transactional    transactionType = 1
)

type controlType int8

const (
	nonControlMessage controlType = 0
	controlMessage    controlType = 1
)
<commit_msg>A very very small typo (#845)<commit_after>package kafka

import (
	"time"
)

// Message is a data structure representing kafka messages.
type Message struct {
	// Topic indicates which topic this message was consumed from via Reader.
	//
	// When being used with Writer, this can be used to configure the topic if
	// not already specified on the writer itself.
	Topic string

	// Partition is read-only and MUST NOT be set when writing messages
	Partition     int
	Offset        int64
	HighWaterMark int64
	Key           []byte
	Value         []byte
	Headers       []Header

	// If not set at the creation, Time will be automatically set when
	// writing the message.
	Time time.Time
}

func (msg Message) message(cw *crc32Writer) message {
	m := message{
		MagicByte: 1,
		Key:       msg.Key,
		Value:     msg.Value,
		Timestamp: timestamp(msg.Time),
	}
	if cw != nil {
		m.CRC = m.crc32(cw)
	}
	return m
}

const timestampSize = 8

func (msg *Message) size() int32 {
	return 4 + 1 + 1 + sizeofBytes(msg.Key) + sizeofBytes(msg.Value) + timestampSize
}

type message struct {
	CRC        int32
	MagicByte  int8
	Attributes int8
	Timestamp  int64
	Key        []byte
	Value      []byte
}

func (m message) crc32(cw *crc32Writer) int32 {
	cw.crc32 = 0
	cw.writeInt8(m.MagicByte)
	cw.writeInt8(m.Attributes)
	if m.MagicByte != 0 {
		cw.writeInt64(m.Timestamp)
	}
	cw.writeBytes(m.Key)
	cw.writeBytes(m.Value)
	return int32(cw.crc32)
}

func (m message) size() int32 {
	size := 4 + 1 + 1 + sizeofBytes(m.Key) + sizeofBytes(m.Value)
	if m.MagicByte != 0 {
		size += timestampSize
	}
	return size
}

func (m message) writeTo(wb *writeBuffer) {
	wb.writeInt32(m.CRC)
	wb.writeInt8(m.MagicByte)
	wb.writeInt8(m.Attributes)
	if m.MagicByte != 0 {
		wb.writeInt64(m.Timestamp)
	}
	wb.writeBytes(m.Key)
	wb.writeBytes(m.Value)
}

type messageSetItem struct {
	Offset      int64
	MessageSize int32
	Message     message
}

func (m messageSetItem) size() int32 {
	return 8 + 4 + m.Message.size()
}

func (m messageSetItem) writeTo(wb *writeBuffer) {
	wb.writeInt64(m.Offset)
	wb.writeInt32(m.MessageSize)
	m.Message.writeTo(wb)
}

type messageSet []messageSetItem

func (s messageSet) size() (size int32) {
	for _, m := range s {
		size += m.size()
	}
	return
}

func (s messageSet) writeTo(wb *writeBuffer) {
	for _, m := range s {
		m.writeTo(wb)
	}
}

type timestampType int8

const (
	createTime    timestampType = 0
	logAppendTime timestampType = 1
)

type transactionType int8

const (
	nonTransactional transactionType = 0
	transactional    transactionType = 1
)

type controlType int8

const (
	nonControlMessage controlType = 0
	controlMessage    controlType = 1
)
<|endoftext|>


<commit_before>package telebot

import (
	"time"
)

// Message object represents a message.
type Message struct {
	ID       int  `json:"message_id"`
	Sender   User `json:"from"`
	Unixtime int  `json:"date"`

	// For forwarded messages, sender of the original message.
	OriginalSender User `json:"forward_from"`

	// For forwarded messages, unixtime of the original message.
	OriginalUnixtime int `json:"forward_date"`

	// For replies, ReplyTo represents the original message.
	// Note that the Message object in this field will not
	// contain further ReplyTo fields even if it
	// itself is a reply.
	ReplyTo *Message `json:"reply_to_message"`

	// For a text message, the actual UTF-8 text of the message
	Text string `json:"text"`

	// For an audio recording, information about it.
	Audio Audio `json:"audio"`

	// For a general file, information about it.
	Document Document `json:"document"`

	// For a photo, available thumbnails.
	Photo []Thumbnail `json:"photo"`

	// For a sticker, information about it.
	Sticker Sticker `json:"sticker"`

	// For a video, information about it.
	Video Video `json:"video"`

	// For a contact, contact information itself.
	Contact Contact `json:"contact"`

	// For a location, its longitude and latitude.
	Location Location `json:"location"`

	// A group chat message belongs to, empty if personal.
	Chat User `json:"chat"`

	// For a service message, represents a user,
	// that just got added to chat, this message came from.
	//
	// Sender leads to User, capable of invite.
	//
	// UserJoined might be the Bot itself.
	UserJoined User `json:"new_chat_participant"`

	// For a service message, represents a user,
	// that just left chat, this message came from.
	//
	// If user was kicked, Sender leads to a User,
	// capable of this kick.
	//
	// UserLeft might be the Bot itself.
	UserLeft User `json:"left_chat_participant"`

	// For a service message, represents a new title
	// for chat this message came from.
	//
	// Sender would lead to a User, capable of change.
	NewChatTitle string `json:"new_chat_title"`

	// For a service message, represents all available
	// thumbnails of new chat photo.
	//
	// Sender would lead to a User, capable of change.
	NewChatPhoto []Thumbnail `json:"new_chat_photo"`

	// For a service message, true if chat photo just
	// got removed.
	//
	// Sender would lead to a User, capable of change.
	ChatPhotoDeleted bool `json:"delete_chat_photo"`

	// For a service message, true if group has been created.
	//
	// You would recieve such a message if you are one of
	// initial group chat members.
	//
	// Sender would lead to creator of the chat.
	ChatCreated bool `json:"group_chat_created"`
}

// Origin returns an origin of message: group chat / personal.
func (m Message) Origin() User {
	if (m.Chat != User{}) {
		return m.Chat
	}

	return m.Sender
}

// Time returns the moment of message creation in local time.
func (m Message) Time() time.Time {
	return time.Unix(int64(m.Unixtime), 0)
}

// IsForwarded says whether message is forwarded copy of another
// message or not.
func (m Message) IsForwarded() bool {
	if (m.OriginalSender != User{}) {
		return true
	}

	return false
}

// IsReply says whether message is reply to another message or not.
func (m Message) IsReply() bool {
	if m.ReplyTo != nil {
		return true
	}

	return false
}

// IsPersonal returns true, if message is a personal message,
// returns false if sent to group chat.
func (m Message) IsPersonal() bool {
	if (m.Chat != User{}) {
		return true
	}

	return false
}

// IsService returns true, if message is a service message,
// returns false otherwise.
//
// Service messages are automatically sent messages, which
// typically occur on some global action. For instance, when
// anyone leaves the chat or chat title changes.
func (m Message) IsService() bool {
	service := false

	if (m.UserJoined != User{}) {
		service = true
	}

	if (m.UserLeft != User{}) {
		service = true
	}

	if m.NewChatTitle != "" {
		service = true
	}

	if len(m.NewChatPhoto) > 0 {
		service = true
	}

	if m.ChatPhotoDeleted {
		service = true
	}

	if m.ChatCreated {
		service = true
	}

	return service
}
<commit_msg>A little code bloat fix<commit_after>package telebot

import (
	"time"
)

// Message object represents a message.
type Message struct {
	ID       int  `json:"message_id"`
	Sender   User `json:"from"`
	Unixtime int  `json:"date"`

	// For forwarded messages, sender of the original message.
	OriginalSender User `json:"forward_from"`

	// For forwarded messages, unixtime of the original message.
	OriginalUnixtime int `json:"forward_date"`

	// For replies, ReplyTo represents the original message.
	// Note that the Message object in this field will not
	// contain further ReplyTo fields even if it
	// itself is a reply.
	ReplyTo *Message `json:"reply_to_message"`

	// For a text message, the actual UTF-8 text of the message
	Text string `json:"text"`

	// For an audio recording, information about it.
	Audio Audio `json:"audio"`

	// For a general file, information about it.
	Document Document `json:"document"`

	// For a photo, available thumbnails.
	Photo []Thumbnail `json:"photo"`

	// For a sticker, information about it.
	Sticker Sticker `json:"sticker"`

	// For a video, information about it.
	Video Video `json:"video"`

	// For a contact, contact information itself.
	Contact Contact `json:"contact"`

	// For a location, its longitude and latitude.
	Location Location `json:"location"`

	// A group chat message belongs to, empty if personal.
	Chat User `json:"chat"`

	// For a service message, represents a user,
	// that just got added to chat, this message came from.
	//
	// Sender leads to User, capable of invite.
	//
	// UserJoined might be the Bot itself.
	UserJoined User `json:"new_chat_participant"`

	// For a service message, represents a user,
	// that just left chat, this message came from.
	//
	// If user was kicked, Sender leads to a User,
	// capable of this kick.
	//
	// UserLeft might be the Bot itself.
	UserLeft User `json:"left_chat_participant"`

	// For a service message, represents a new title
	// for chat this message came from.
	//
	// Sender would lead to a User, capable of change.
	NewChatTitle string `json:"new_chat_title"`

	// For a service message, represents all available
	// thumbnails of new chat photo.
	//
	// Sender would lead to a User, capable of change.
	NewChatPhoto []Thumbnail `json:"new_chat_photo"`

	// For a service message, true if chat photo just
	// got removed.
	//
	// Sender would lead to a User, capable of change.
	ChatPhotoDeleted bool `json:"delete_chat_photo"`

	// For a service message, true if group has been created.
	//
	// You would recieve such a message if you are one of
	// initial group chat members.
	//
	// Sender would lead to creator of the chat.
	ChatCreated bool `json:"group_chat_created"`
}

// Origin returns an origin of message: group chat / personal.
func (m Message) Origin() User {
	if (m.Chat != User{}) {
		return m.Chat
	}

	return m.Sender
}

// Time returns the moment of message creation in local time.
func (m Message) Time() time.Time {
	return time.Unix(int64(m.Unixtime), 0)
}

// IsForwarded says whether message is forwarded copy of another
// message or not.
func (m Message) IsForwarded() bool {
	return m.OriginalSender != User{}
}

// IsReply says whether message is reply to another message or not.
func (m Message) IsReply() bool {
	return m.ReplyTo != nil
}

// IsPersonal returns true, if message is a personal message,
// returns false if sent to group chat.
func (m Message) IsPersonal() bool {
	return m.Chat != User{}
}

// IsService returns true, if message is a service message,
// returns false otherwise.
//
// Service messages are automatically sent messages, which
// typically occur on some global action. For instance, when
// anyone leaves the chat or chat title changes.
func (m Message) IsService() bool {
	service := false

	if (m.UserJoined != User{}) {
		service = true
	}

	if (m.UserLeft != User{}) {
		service = true
	}

	if m.NewChatTitle != "" {
		service = true
	}

	if len(m.NewChatPhoto) > 0 {
		service = true
	}

	if m.ChatPhotoDeleted {
		service = true
	}

	if m.ChatCreated {
		service = true
	}

	return service
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcwire

import (
	"bytes"
	"fmt"
	"io"
	"unicode/utf8"
)

// commandSize is the fixed size of all commands in the common bitcoin message
// header.  Shorter commands must be zero padded.
const commandSize = 12

// maxMessagePayload is the maximum byes a message can be regardless of other
// individual limits imposed by messages themselves.
const maxMessagePayload = (1024 * 1024 * 32) // 32MB

// Commands used in bitcoin message headers which describe the type of message.
const (
	cmdVersion    = "version"
	cmdVerAck     = "verack"
	cmdGetAddr    = "getaddr"
	cmdAddr       = "addr"
	cmdGetBlocks  = "getblocks"
	cmdInv        = "inv"
	cmdGetData    = "getdata"
	cmdNotFound   = "notfound"
	cmdBlock      = "block"
	cmdTx         = "tx"
	cmdGetHeaders = "getheaders"
	cmdHeaders    = "headers"
	cmdPing       = "ping"
	cmdPong       = "pong"
	cmdAlert      = "alert"
	cmdMemPool    = "mempool"
)

// Message is an interface that describes a bitcoin message.  A type that
// implements Message has complete control over the representation of its data
// and may therefore contain additional or fewer fields than those which
// are used directly in the protocol encoded message.
type Message interface {
	BtcDecode(io.Reader, uint32) error
	BtcEncode(io.Writer, uint32) error
	Command() string
	MaxPayloadLength(uint32) uint32
}

// makeEmptyMessage creates a message of the appropriate concrete type based
// on the command.
func makeEmptyMessage(command string) (Message, error) {
	var msg Message
	switch command {
	case cmdVersion:
		msg = &MsgVersion{}

	case cmdVerAck:
		msg = &MsgVerAck{}

	case cmdGetAddr:
		msg = &MsgGetAddr{}

	case cmdAddr:
		msg = &MsgAddr{}

	case cmdGetBlocks:
		msg = &MsgGetBlocks{}

	case cmdBlock:
		msg = &MsgBlock{}

	case cmdInv:
		msg = &MsgInv{}

	case cmdGetData:
		msg = &MsgGetData{}

	case cmdNotFound:
		msg = &MsgNotFound{}

	case cmdTx:
		msg = &MsgTx{}

	case cmdPing:
		msg = &MsgPing{}

	case cmdPong:
		msg = &MsgPong{}

	case cmdGetHeaders:
		msg = &MsgGetHeaders{}

	case cmdHeaders:
		msg = &MsgHeaders{}

	case cmdAlert:
		msg = &MsgAlert{}

	case cmdMemPool:
		msg = &MsgMemPool{}

	default:
		return nil, fmt.Errorf("unhandled command [%s]", command)
	}
	return msg, nil
}

// messageHeader defines the header structure for all bitcoin protocol messages.
type messageHeader struct {
	magic    BitcoinNet // 4 bytes
	command  string     // 12 bytes
	length   uint32     // 4 bytes
	checksum [4]byte    // 4 bytes
}

// readMessageHeader reads a bitcoin messager header from r.
func readMessageHeader(r io.Reader) (*messageHeader, error) {
	var command [commandSize]byte

	hdr := messageHeader{}
	err := readElements(r, &hdr.magic, &command, &hdr.length, &hdr.checksum)
	if err != nil {
		return nil, err
	}

	// Strip trailing zeros from command string.
	hdr.command = string(bytes.TrimRight(command[:], string(0)))

	// Enforce maximum message payload.
	if hdr.length > maxMessagePayload {
		str := "readMessageHeader: message payload is too large - " +
			"Header indicates %d bytes, but max message payload is %d bytes."
		return nil, fmt.Errorf(str, hdr.length, maxMessagePayload)
	}

	return &hdr, nil
}

// discardInput reads n bytes from reader r in chunks and discards the read
// bytes.  This is used to skip payloads when various errors occur and helps
// prevent rogue nodes from causing massive memory allocation through forging
// header length.
func discardInput(r io.Reader, n uint32) {
	maxSize := uint32(10 * 1024) // 10k at a time
	numReads := n / maxSize
	bytesRemaining := n % maxSize
	if n > 0 {
		buf := make([]byte, maxSize)
		for i := uint32(0); i < numReads; i++ {
			io.ReadFull(r, buf)
		}
	}
	if bytesRemaining > 0 {
		buf := make([]byte, bytesRemaining)
		io.ReadFull(r, buf)
	}
}

// readMessage reads the next bitcoin message from r for the provided protocol
// version and message header.
func readMessage(r io.Reader, pver uint32, hdr *messageHeader) (Message, []byte, error) {
	if hdr == nil {
		return nil, nil, fmt.Errorf("readMessage: nil header")
	}

	command := hdr.command
	if !utf8.ValidString(command) {
		discardInput(r, hdr.length)
		str := "readMessage: invalid command %v"
		return nil, nil, fmt.Errorf(str, []byte(command))
	}

	// Create struct of appropriate message type based on the command.
	msg, err := makeEmptyMessage(command)
	if err != nil {
		discardInput(r, hdr.length)
		return nil, nil, fmt.Errorf("readMessage: %v", err)
	}

	// Check for maximum length based on the message type as a malicious client
	// could otherwise create a well-formed header and set the length to max
	// numbers in order to exhaust the machine's memory.
	mpl := msg.MaxPayloadLength(pver)
	if hdr.length > mpl {
		discardInput(r, hdr.length)
		str := "ReadMessage: payload exceeds max length - Header " +
			"indicates %v bytes, but max payload size for messages of type " +
			"[%v] is %v."
		return nil, nil, fmt.Errorf(str, hdr.length, command, mpl)
	}

	// Read payload.
	payload := make([]byte, hdr.length)
	n, err := io.ReadFull(r, payload)
	if err != nil {
		return nil, nil, err
	}
	if uint32(n) != hdr.length {
		str := "readMessage: failed to read payload - Read %v " +
			"bytes, but payload size is %v bytes."
		return nil, nil, fmt.Errorf(str, n, hdr.length)
	}

	// Test checksum.
	checksum := DoubleSha256(payload)[0:4]
	if !bytes.Equal(checksum[:], hdr.checksum[:]) {
		str := "readMessage: payload checksum failed - Header " +
			"indicates %v, but actual checksum is %v."
		return nil, nil, fmt.Errorf(str, hdr.checksum, checksum)
	}

	// Unmarshal message.
	pr := bytes.NewBuffer(payload)
	err = msg.BtcDecode(pr, pver)
	if err != nil {
		return nil, nil, err
	}

	return msg, payload, nil
}

// WriteMessage writes a bitcoin Message to w including the necessary header
// information.
func WriteMessage(w io.Writer, msg Message, pver uint32, btcnet BitcoinNet) error {
	var command [commandSize]byte

	cmd := msg.Command()
	if len(cmd) > commandSize {
		str := "WriteMessage: command is too long [%s]"
		return fmt.Errorf(str, command)
	}
	copy(command[:], []byte(cmd))

	var bw bytes.Buffer
	err := msg.BtcEncode(&bw, pver)
	if err != nil {
		return err
	}
	payload := bw.Bytes()
	lenp := len(payload)

	// Enforce maximum message payload.
	if lenp > maxMessagePayload {
		str := "WriteMessage: message payload is too large - " +
			"Encoded %d bytes, but maximum message payload is %d bytes."
		return fmt.Errorf(str, lenp, maxMessagePayload)
	}

	// Create header for the message.
	hdr := messageHeader{}
	hdr.magic = btcnet
	hdr.command = cmd
	hdr.length = uint32(lenp)
	copy(hdr.checksum[:], DoubleSha256(payload)[0:4])

	// Write header.
	err = writeElements(w, hdr.magic, command, hdr.length, hdr.checksum)
	if err != nil {
		return err
	}

	// Write payload.
	n, err := w.Write(payload)
	if err != nil {
		return err
	}
	if n != lenp {
		str := "WriteMessage: failed to write payload. " +
			"Wrote %v bytes, but payload size is %v bytes."
		return fmt.Errorf(str, n, lenp)
	}
	return nil
}

// ReadMessage reads, validates, and parses the next bitcoin Message from r.
func ReadMessage(r io.Reader, pver uint32, btcnet BitcoinNet) (Message, []byte, error) {
	hdr, err := readMessageHeader(r)
	if err != nil {
		return nil, nil, err
	}

	if hdr.magic != btcnet {
		str := "ReadMessage: message from other network [%v]"
		return nil, nil, fmt.Errorf(str, hdr.magic)
	}

	return readMessage(r, pver, hdr)
}
<commit_msg>Remove dead error check.<commit_after>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcwire

import (
	"bytes"
	"fmt"
	"io"
	"unicode/utf8"
)

// commandSize is the fixed size of all commands in the common bitcoin message
// header.  Shorter commands must be zero padded.
const commandSize = 12

// maxMessagePayload is the maximum byes a message can be regardless of other
// individual limits imposed by messages themselves.
const maxMessagePayload = (1024 * 1024 * 32) // 32MB

// Commands used in bitcoin message headers which describe the type of message.
const (
	cmdVersion    = "version"
	cmdVerAck     = "verack"
	cmdGetAddr    = "getaddr"
	cmdAddr       = "addr"
	cmdGetBlocks  = "getblocks"
	cmdInv        = "inv"
	cmdGetData    = "getdata"
	cmdNotFound   = "notfound"
	cmdBlock      = "block"
	cmdTx         = "tx"
	cmdGetHeaders = "getheaders"
	cmdHeaders    = "headers"
	cmdPing       = "ping"
	cmdPong       = "pong"
	cmdAlert      = "alert"
	cmdMemPool    = "mempool"
)

// Message is an interface that describes a bitcoin message.  A type that
// implements Message has complete control over the representation of its data
// and may therefore contain additional or fewer fields than those which
// are used directly in the protocol encoded message.
type Message interface {
	BtcDecode(io.Reader, uint32) error
	BtcEncode(io.Writer, uint32) error
	Command() string
	MaxPayloadLength(uint32) uint32
}

// makeEmptyMessage creates a message of the appropriate concrete type based
// on the command.
func makeEmptyMessage(command string) (Message, error) {
	var msg Message
	switch command {
	case cmdVersion:
		msg = &MsgVersion{}

	case cmdVerAck:
		msg = &MsgVerAck{}

	case cmdGetAddr:
		msg = &MsgGetAddr{}

	case cmdAddr:
		msg = &MsgAddr{}

	case cmdGetBlocks:
		msg = &MsgGetBlocks{}

	case cmdBlock:
		msg = &MsgBlock{}

	case cmdInv:
		msg = &MsgInv{}

	case cmdGetData:
		msg = &MsgGetData{}

	case cmdNotFound:
		msg = &MsgNotFound{}

	case cmdTx:
		msg = &MsgTx{}

	case cmdPing:
		msg = &MsgPing{}

	case cmdPong:
		msg = &MsgPong{}

	case cmdGetHeaders:
		msg = &MsgGetHeaders{}

	case cmdHeaders:
		msg = &MsgHeaders{}

	case cmdAlert:
		msg = &MsgAlert{}

	case cmdMemPool:
		msg = &MsgMemPool{}

	default:
		return nil, fmt.Errorf("unhandled command [%s]", command)
	}
	return msg, nil
}

// messageHeader defines the header structure for all bitcoin protocol messages.
type messageHeader struct {
	magic    BitcoinNet // 4 bytes
	command  string     // 12 bytes
	length   uint32     // 4 bytes
	checksum [4]byte    // 4 bytes
}

// readMessageHeader reads a bitcoin message header from r.
func readMessageHeader(r io.Reader) (*messageHeader, error) {
	var command [commandSize]byte

	hdr := messageHeader{}
	err := readElements(r, &hdr.magic, &command, &hdr.length, &hdr.checksum)
	if err != nil {
		return nil, err
	}

	// Strip trailing zeros from command string.
	hdr.command = string(bytes.TrimRight(command[:], string(0)))

	// Enforce maximum message payload.
	if hdr.length > maxMessagePayload {
		str := "readMessageHeader: message payload is too large - " +
			"Header indicates %d bytes, but max message payload is %d bytes."
		return nil, fmt.Errorf(str, hdr.length, maxMessagePayload)
	}

	return &hdr, nil
}

// discardInput reads n bytes from reader r in chunks and discards the read
// bytes.  This is used to skip payloads when various errors occur and helps
// prevent rogue nodes from causing massive memory allocation through forging
// header length.
func discardInput(r io.Reader, n uint32) {
	maxSize := uint32(10 * 1024) // 10k at a time
	numReads := n / maxSize
	bytesRemaining := n % maxSize
	if n > 0 {
		buf := make([]byte, maxSize)
		for i := uint32(0); i < numReads; i++ {
			io.ReadFull(r, buf)
		}
	}
	if bytesRemaining > 0 {
		buf := make([]byte, bytesRemaining)
		io.ReadFull(r, buf)
	}
}

// readMessage reads the next bitcoin message from r for the provided protocol
// version and message header.
func readMessage(r io.Reader, pver uint32, hdr *messageHeader) (Message, []byte, error) {
	if hdr == nil {
		return nil, nil, fmt.Errorf("readMessage: nil header")
	}

	command := hdr.command
	if !utf8.ValidString(command) {
		discardInput(r, hdr.length)
		str := "readMessage: invalid command %v"
		return nil, nil, fmt.Errorf(str, []byte(command))
	}

	// Create struct of appropriate message type based on the command.
	msg, err := makeEmptyMessage(command)
	if err != nil {
		discardInput(r, hdr.length)
		return nil, nil, fmt.Errorf("readMessage: %v", err)
	}

	// Check for maximum length based on the message type as a malicious client
	// could otherwise create a well-formed header and set the length to max
	// numbers in order to exhaust the machine's memory.
	mpl := msg.MaxPayloadLength(pver)
	if hdr.length > mpl {
		discardInput(r, hdr.length)
		str := "ReadMessage: payload exceeds max length - Header " +
			"indicates %v bytes, but max payload size for messages of type " +
			"[%v] is %v."
		return nil, nil, fmt.Errorf(str, hdr.length, command, mpl)
	}

	// Read payload.
	payload := make([]byte, hdr.length)
	_, err = io.ReadFull(r, payload)
	if err != nil {
		return nil, nil, err
	}

	// Test checksum.
	checksum := DoubleSha256(payload)[0:4]
	if !bytes.Equal(checksum[:], hdr.checksum[:]) {
		str := "readMessage: payload checksum failed - Header " +
			"indicates %v, but actual checksum is %v."
		return nil, nil, fmt.Errorf(str, hdr.checksum, checksum)
	}

	// Unmarshal message.
	pr := bytes.NewBuffer(payload)
	err = msg.BtcDecode(pr, pver)
	if err != nil {
		return nil, nil, err
	}

	return msg, payload, nil
}

// WriteMessage writes a bitcoin Message to w including the necessary header
// information.
func WriteMessage(w io.Writer, msg Message, pver uint32, btcnet BitcoinNet) error {
	var command [commandSize]byte

	cmd := msg.Command()
	if len(cmd) > commandSize {
		str := "WriteMessage: command is too long [%s]"
		return fmt.Errorf(str, command)
	}
	copy(command[:], []byte(cmd))

	var bw bytes.Buffer
	err := msg.BtcEncode(&bw, pver)
	if err != nil {
		return err
	}
	payload := bw.Bytes()
	lenp := len(payload)

	// Enforce maximum message payload.
	if lenp > maxMessagePayload {
		str := "WriteMessage: message payload is too large - " +
			"Encoded %d bytes, but maximum message payload is %d bytes."
		return fmt.Errorf(str, lenp, maxMessagePayload)
	}

	// Create header for the message.
	hdr := messageHeader{}
	hdr.magic = btcnet
	hdr.command = cmd
	hdr.length = uint32(lenp)
	copy(hdr.checksum[:], DoubleSha256(payload)[0:4])

	// Write header.
	err = writeElements(w, hdr.magic, command, hdr.length, hdr.checksum)
	if err != nil {
		return err
	}

	// Write payload.
	n, err := w.Write(payload)
	if err != nil {
		return err
	}
	if n != lenp {
		str := "WriteMessage: failed to write payload. " +
			"Wrote %v bytes, but payload size is %v bytes."
		return fmt.Errorf(str, n, lenp)
	}
	return nil
}

// ReadMessage reads, validates, and parses the next bitcoin Message from r.
func ReadMessage(r io.Reader, pver uint32, btcnet BitcoinNet) (Message, []byte, error) {
	hdr, err := readMessageHeader(r)
	if err != nil {
		return nil, nil, err
	}

	if hdr.magic != btcnet {
		str := "ReadMessage: message from other network [%v]"
		return nil, nil, fmt.Errorf(str, hdr.magic)
	}

	return readMessage(r, pver, hdr)
}
<|endoftext|>


<commit_before>package sej

import (
	"errors"
	"fmt"
	"io"
	"os"
	"time"
)

var (
	errMessageCorrupted = errors.New("last message of the journal file is courrupted")
)

// Message in a segmented journal file
type Message struct {
	Offset    uint64
	Timestamp time.Time
	Type      byte
	Key       []byte
	Value     []byte
}

type readSeekCloser interface {
	io.Reader
	io.Seeker
	io.Closer
}

// WriteTo writes the message to w
func (m *Message) WriteTo(w io.Writer) (int64, error) {
	cnt := int64(0) // total bytes written

	n, err := writeUint64(w, m.Offset)
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	ts := m.Timestamp
	if ts.IsZero() {
		ts = time.Now()
	}
	n, err = writeInt64(w, ts.UnixNano())
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	n, err = writeByte(w, m.Type)
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	n, err = writeInt8(w, int8(len(m.Key)))
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	n, err = w.Write(m.Key)
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	n, err = writeInt32(w, int32(len(m.Value)))
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	n, err = w.Write(m.Value)
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	n, err = writeInt32(w, int32(cnt)+4)
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	return cnt, nil
}

// ReadFrom reads a message from a io.ReadSeeker.
// When an error occurs, it will rollback the seeker and then returns the original error.
func (m *Message) ReadFrom(r io.ReadSeeker) (n int64, err error) {
	cnt := int64(0) // total bytes read

	nn, err := readUint64(r, &m.Offset)
	cnt += int64(nn)
	if err != nil {
		return cnt, err
	}

	var unixNano int64
	nn, err = readInt64(r, &unixNano)
	cnt += int64(nn)
	if err != nil {
		return cnt, err
	}
	m.Timestamp = time.Unix(0, unixNano)

	nn, err = readByte(r, &m.Type)
	cnt += int64(nn)
	if err != nil {
		return cnt, err
	}

	var keyLen int8
	nn, err = readInt8(r, &keyLen)
	cnt += int64(nn)
	if err != nil {
		return cnt, err
	}

	m.Key = make([]byte, int(keyLen))
	nn, err = io.ReadFull(r, m.Key)
	cnt += int64(nn)
	if err != nil {
		return cnt, err
	}
	if nn != int(keyLen) {
		return cnt, fmt.Errorf("message is truncated at %d", m.Offset)
	}

	var valueLen int32
	nn, err = readInt32(r, &valueLen)
	cnt += int64(nn)
	if err != nil {
		return cnt, err
	}

	m.Value = make([]byte, int(valueLen))
	nn, err = io.ReadFull(r, m.Value)
	cnt += int64(nn)
	if err != nil {
		return cnt, err
	}
	if nn != int(valueLen) {
		return cnt, fmt.Errorf("message is truncated at %d", m.Offset)
	}

	var size int32
	nn, err = readInt32(r, &size)
	cnt += int64(nn)
	if err != nil {
		return cnt, err
	}
	if int64(size) != cnt {
		return cnt, fmt.Errorf("data corruption detected by size2 at %d", m.Offset)
	}

	return cnt, nil
}

func readMessageBackward(r io.ReadSeeker) (*Message, error) {
	var size int32
	if _, err := r.Seek(-4, os.SEEK_CUR); err != nil {
		return nil, err
	}
	if _, err := readInt32(r, &size); err != nil {
		return nil, err
	}
	if _, err := r.Seek(-int64(size), os.SEEK_CUR); err != nil {
		return nil, err
	}
	var msg Message
	_, err := msg.ReadFrom(r)
	return &msg, err
}

func writeInt8(w io.Writer, i int8) (int, error) {
	return w.Write([]byte{byte(i)})
}

func readInt8(r io.ReadSeeker, i *int8) (int, error) {
	var b [1]byte
	n, err := io.ReadFull(r, b[:])
	if err != nil {
		return n, err
	}
	*i = int8(b[0])
	return n, nil
}

func writeByte(w io.Writer, i byte) (int, error) {
	return w.Write([]byte{i})
}

func readByte(r io.ReadSeeker, i *byte) (int, error) {
	var b [1]byte
	n, err := io.ReadFull(r, b[:])
	if err != nil {
		return n, err
	}
	*i = b[0]
	return n, nil
}

func writeInt64(w io.Writer, i int64) (int, error) {
	return w.Write([]byte{byte(i >> 56), byte(i >> 48), byte(i >> 40), byte(i >> 32), byte(i >> 24), byte(i >> 16), byte(i >> 8), byte(i)})
}

func readInt64(r io.ReadSeeker, i *int64) (int, error) {
	var b [8]byte
	n, err := io.ReadFull(r, b[:])
	if err != nil {
		return n, err
	}
	*i = int64(b[0])<<56 | int64(b[1])<<48 | int64(b[2])<<40 | int64(b[3])<<32 |
		int64(b[4])<<24 | int64(b[5])<<16 | int64(b[6])<<8 | int64(b[7])
	return n, nil
}

func writeUint64(w io.Writer, i uint64) (int, error) {
	return w.Write([]byte{byte(i >> 56), byte(i >> 48), byte(i >> 40), byte(i >> 32), byte(i >> 24), byte(i >> 16), byte(i >> 8), byte(i)})
}

func readUint64(r io.ReadSeeker, i *uint64) (int, error) {
	var b [8]byte
	n, err := io.ReadFull(r, b[:])
	if err != nil {
		return n, err
	}
	*i = uint64(b[0])<<56 | uint64(b[1])<<48 | uint64(b[2])<<40 | uint64(b[3])<<32 |
		uint64(b[4])<<24 | uint64(b[5])<<16 | uint64(b[6])<<8 | uint64(b[7])
	return n, nil
}

func writeInt32(w io.Writer, i int32) (int, error) {
	return w.Write([]byte{byte(i >> 24), byte(i >> 16), byte(i >> 8), byte(i)})
}

func readInt32(r io.ReadSeeker, i *int32) (int, error) {
	var b [4]byte
	n, err := io.ReadFull(r, b[:])
	if err != nil {
		return n, err
	}
	*i = int32(b[0])<<24 | int32(b[1])<<16 | int32(b[2])<<8 | int32(b[3])
	return n, nil
}

func writeUint32(w io.Writer, i uint32) (int, error) {
	return w.Write([]byte{byte(i >> 24), byte(i >> 16), byte(i >> 8), byte(i)})
}

func readUint32(r io.ReadSeeker, i *uint32) (int, error) {
	var b [4]byte
	n, err := io.ReadFull(r, b[:])
	if err != nil {
		return n, err
	}
	*i = uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3])
	return n, nil
}

// LatestOffset returns the offset after the last message in a journal file
func (journalFile *JournalFile) LastOffset() (uint64, error) {
	file, err := os.Open(journalFile.FileName)
	if err != nil {
		return 0, err
	}
	defer file.Close()
	fileSize, err := file.Seek(0, os.SEEK_END)
	if err != nil {
		return 0, err
	}
	if fileSize == 0 {
		return journalFile.FirstOffset, nil
	}
	msg, err := readMessageBackward(file)
	if err != nil {
		return 0, errMessageCorrupted
	}
	return msg.Offset + 1, nil
}
<commit_msg>check minus size<commit_after>package sej

import (
	"errors"
	"fmt"
	"io"
	"os"
	"time"
)

var (
	errMessageCorrupted = errors.New("last message of the journal file is courrupted")
)

// Message in a segmented journal file
type Message struct {
	Offset    uint64
	Timestamp time.Time
	Type      byte
	Key       []byte
	Value     []byte
}

type readSeekCloser interface {
	io.Reader
	io.Seeker
	io.Closer
}

// WriteTo writes the message to w
func (m *Message) WriteTo(w io.Writer) (int64, error) {
	cnt := int64(0) // total bytes written

	n, err := writeUint64(w, m.Offset)
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	ts := m.Timestamp
	if ts.IsZero() {
		ts = time.Now()
	}
	n, err = writeInt64(w, ts.UnixNano())
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	n, err = writeByte(w, m.Type)
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	n, err = writeInt8(w, int8(len(m.Key)))
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	n, err = w.Write(m.Key)
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	n, err = writeInt32(w, int32(len(m.Value)))
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	n, err = w.Write(m.Value)
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	n, err = writeInt32(w, int32(cnt)+4)
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	return cnt, nil
}

// ReadFrom reads a message from a io.ReadSeeker.
// When an error occurs, it will rollback the seeker and then returns the original error.
func (m *Message) ReadFrom(r io.ReadSeeker) (n int64, err error) {
	cnt := int64(0) // total bytes read

	nn, err := readUint64(r, &m.Offset)
	cnt += int64(nn)
	if err != nil {
		return cnt, err
	}

	var unixNano int64
	nn, err = readInt64(r, &unixNano)
	cnt += int64(nn)
	if err != nil {
		return cnt, err
	}
	m.Timestamp = time.Unix(0, unixNano)

	nn, err = readByte(r, &m.Type)
	cnt += int64(nn)
	if err != nil {
		return cnt, err
	}

	var keyLen int8
	nn, err = readInt8(r, &keyLen)
	cnt += int64(nn)
	if err != nil {
		return cnt, err
	}
	if keyLen < 0 {
		return cnt, errMessageCorrupted
	}

	m.Key = make([]byte, int(keyLen))
	nn, err = io.ReadFull(r, m.Key)
	cnt += int64(nn)
	if err != nil {
		return cnt, err
	}
	if nn != int(keyLen) {
		return cnt, fmt.Errorf("message is truncated at %d", m.Offset)
	}

	var valueLen int32
	nn, err = readInt32(r, &valueLen)
	cnt += int64(nn)
	if err != nil {
		return cnt, err
	}
	if valueLen < 0 {
		return cnt, errMessageCorrupted
	}

	m.Value = make([]byte, int(valueLen))
	nn, err = io.ReadFull(r, m.Value)
	cnt += int64(nn)
	if err != nil {
		return cnt, err
	}
	if nn != int(valueLen) {
		return cnt, fmt.Errorf("message is truncated at %d", m.Offset)
	}

	var size int32
	nn, err = readInt32(r, &size)
	cnt += int64(nn)
	if err != nil {
		return cnt, err
	}
	if int64(size) != cnt {
		return cnt, fmt.Errorf("data corruption detected by size2 at %d", m.Offset)
	}

	return cnt, nil
}

func readMessageBackward(r io.ReadSeeker) (*Message, error) {
	var size int32
	if _, err := r.Seek(-4, os.SEEK_CUR); err != nil {
		return nil, err
	}
	if _, err := readInt32(r, &size); err != nil {
		return nil, err
	}
	if _, err := r.Seek(-int64(size), os.SEEK_CUR); err != nil {
		return nil, err
	}
	var msg Message
	_, err := msg.ReadFrom(r)
	return &msg, err
}

func writeInt8(w io.Writer, i int8) (int, error) {
	return w.Write([]byte{byte(i)})
}

func readInt8(r io.ReadSeeker, i *int8) (int, error) {
	var b [1]byte
	n, err := io.ReadFull(r, b[:])
	if err != nil {
		return n, err
	}
	*i = int8(b[0])
	return n, nil
}

func writeByte(w io.Writer, i byte) (int, error) {
	return w.Write([]byte{i})
}

func readByte(r io.ReadSeeker, i *byte) (int, error) {
	var b [1]byte
	n, err := io.ReadFull(r, b[:])
	if err != nil {
		return n, err
	}
	*i = b[0]
	return n, nil
}

func writeInt64(w io.Writer, i int64) (int, error) {
	return w.Write([]byte{byte(i >> 56), byte(i >> 48), byte(i >> 40), byte(i >> 32), byte(i >> 24), byte(i >> 16), byte(i >> 8), byte(i)})
}

func readInt64(r io.ReadSeeker, i *int64) (int, error) {
	var b [8]byte
	n, err := io.ReadFull(r, b[:])
	if err != nil {
		return n, err
	}
	*i = int64(b[0])<<56 | int64(b[1])<<48 | int64(b[2])<<40 | int64(b[3])<<32 |
		int64(b[4])<<24 | int64(b[5])<<16 | int64(b[6])<<8 | int64(b[7])
	return n, nil
}

func writeUint64(w io.Writer, i uint64) (int, error) {
	return w.Write([]byte{byte(i >> 56), byte(i >> 48), byte(i >> 40), byte(i >> 32), byte(i >> 24), byte(i >> 16), byte(i >> 8), byte(i)})
}

func readUint64(r io.ReadSeeker, i *uint64) (int, error) {
	var b [8]byte
	n, err := io.ReadFull(r, b[:])
	if err != nil {
		return n, err
	}
	*i = uint64(b[0])<<56 | uint64(b[1])<<48 | uint64(b[2])<<40 | uint64(b[3])<<32 |
		uint64(b[4])<<24 | uint64(b[5])<<16 | uint64(b[6])<<8 | uint64(b[7])
	return n, nil
}

func writeInt32(w io.Writer, i int32) (int, error) {
	return w.Write([]byte{byte(i >> 24), byte(i >> 16), byte(i >> 8), byte(i)})
}

func readInt32(r io.ReadSeeker, i *int32) (int, error) {
	var b [4]byte
	n, err := io.ReadFull(r, b[:])
	if err != nil {
		return n, err
	}
	*i = int32(b[0])<<24 | int32(b[1])<<16 | int32(b[2])<<8 | int32(b[3])
	return n, nil
}

func writeUint32(w io.Writer, i uint32) (int, error) {
	return w.Write([]byte{byte(i >> 24), byte(i >> 16), byte(i >> 8), byte(i)})
}

func readUint32(r io.ReadSeeker, i *uint32) (int, error) {
	var b [4]byte
	n, err := io.ReadFull(r, b[:])
	if err != nil {
		return n, err
	}
	*i = uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3])
	return n, nil
}

// LatestOffset returns the offset after the last message in a journal file
func (journalFile *JournalFile) LastOffset() (uint64, error) {
	file, err := os.Open(journalFile.FileName)
	if err != nil {
		return 0, err
	}
	defer file.Close()
	fileSize, err := file.Seek(0, os.SEEK_END)
	if err != nil {
		return 0, err
	}
	if fileSize == 0 {
		return journalFile.FirstOffset, nil
	}
	msg, err := readMessageBackward(file)
	if err != nil {
		return 0, errMessageCorrupted
	}
	return msg.Offset + 1, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Dorival de Moraes Pedroso. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package goga

import (
	"math"

	"github.com/cpmech/gosl/chk"
	"github.com/cpmech/gosl/utl"
)

// Metrics holds metric data such as non-dominated Pareto fronts
type Metrics struct {
	prms   *Parameters   // parameters
	Omin   []float64     // current min ova
	Omax   []float64     // current max ova
	Fmin   []float64     // current min float
	Fmax   []float64     // current max float
	Imin   []int         // current min int
	Imax   []int         // current max int
	Fsizes []int         // front sizes
	Fronts [][]*Solution // non-dominated fronts
}

// Init initialises Metrics
func (o *Metrics) Init(nsol int, prms *Parameters) {
	o.prms = prms
	o.Omin = make([]float64, prms.Nova)
	o.Omax = make([]float64, prms.Nova)
	o.Fmin = make([]float64, prms.Nflt)
	o.Fmax = make([]float64, prms.Nflt)
	o.Imin = make([]int, prms.Nint)
	o.Imax = make([]int, prms.Nint)
	o.Fsizes = make([]int, nsol)
	o.Fronts = make([][]*Solution, nsol)
	for i := 0; i < nsol; i++ {
		o.Fronts[i] = make([]*Solution, nsol)
	}
}

// Compute computes all metric variables
func (o *Metrics) Compute(sols []*Solution) (nfronts int) {

	// reset counters and find limits
	fz := o.Fsizes
	nsol := len(sols)
	for i, sol := range sols {

		// reset values
		sol.Repeated = false
		sol.Nwins = 0
		sol.Nlosses = 0
		sol.FrontId = 0
		sol.DistCrowd = 0
		sol.DistNeigh = INF
		fz[i] = 0

		// ovas range
		for j := 0; j < o.prms.Nova; j++ {
			x := sol.Ova[j]
			if math.IsNaN(x) {
				chk.Panic("NaN found in objective value array\n\txFlt = %v\n\txInt = %v\n\tova = %v\n\toor = %v", sol.Flt, sol.Int, sol.Ova, sol.Oor)
			}
			if i == 0 {
				o.Omin[j] = x
				o.Omax[j] = x
			} else {
				o.Omin[j] = utl.Min(o.Omin[j], x)
				o.Omax[j] = utl.Max(o.Omax[j], x)
			}
		}

		// floats range
		for j := 0; j < o.prms.Nflt; j++ {
			x := sol.Flt[j]
			if i == 0 {
				o.Fmin[j] = x
				o.Fmax[j] = x
			} else {
				o.Fmin[j] = utl.Min(o.Fmin[j], x)
				o.Fmax[j] = utl.Max(o.Fmax[j], x)
			}
		}

		// ints range
		for j := 0; j < o.prms.Nint; j++ {
			x := sol.Int[j]
			if i == 0 {
				o.Imin[j] = x
				o.Imax[j] = x
			} else {
				o.Imin[j] = utl.Imin(o.Imin[j], x)
				o.Imax[j] = utl.Imax(o.Imax[j], x)
			}
		}
	}

	// compute neighbour distances
	dmax := 1e-15
	if o.prms.use_metrics_ovadistance {
		for i := 0; i < o.prms.Nova; i++ {
			dmax += math.Pow(o.Omax[i]-o.Omin[i], 2.0)
		}
	} else {
		for i := 0; i < o.prms.Nflt; i++ {
			dmax += math.Pow(o.Fmax[i]-o.Fmin[i], 2.0)
		}
	}
	dmax = math.Sqrt(dmax)
	for i := 0; i < nsol; i++ {
		A := sols[i]
		for j := i + 1; j < nsol; j++ {
			B := sols[j]
			var dist float64
			if o.prms.use_metrics_ovadistance {
				dist = A.OvaDistance(B, o.Omin, o.Omax)
			} else {
				dist = A.Distance(B, o.Fmin, o.Fmax, o.Imin, o.Imax)
			}
			o.closest(A, B, dist)
			if dist < o.prms.Mdmin*dmax {
				B.Repeated = o.prms.use_metrics_repeated_enabled
				if o.prms.use_metrics_repeated_enabled {
					B.FrontId = nsol
				}
			}
		}
	}

	// compute neighbour distances and dominance data
	for i := 0; i < nsol; i++ {
		A := sols[i]
		if A.Repeated {
			continue
		}
		for j := i + 1; j < nsol; j++ {
			B := sols[j]
			if B.Repeated {
				continue
			}
			A_dom, B_dom := A.Compare(B)
			if A_dom {
				A.WinOver[A.Nwins] = B // i dominates j
				A.Nwins++              // i has another dominated item
				B.Nlosses++            // j is being dominated by i
			}
			if B_dom {
				B.WinOver[B.Nwins] = A // j dominates i
				B.Nwins++              // j has another dominated item
				A.Nlosses++            // i is being dominated by j
			}
		}
	}

	// first front
	for _, sol := range sols {
		if sol.Repeated {
			continue
		}
		if sol.Nlosses == 0 {
			o.Fronts[0][fz[0]] = sol
			fz[0]++
		}
	}

	// next fronts
	for r, front := range o.Fronts {
		if fz[r] == 0 {
			break
		}
		nfronts++
		for s := 0; s < fz[r]; s++ {
			A := front[s]
			if A.Repeated {
				chk.Panic("repeated is wrong 1")
			}
			for k := 0; k < A.Nwins; k++ {
				B := A.WinOver[k]
				if B.Repeated {
					chk.Panic("repeatd is wrong 2")
				}
				B.Nlosses--
				if B.Nlosses == 0 { // B belongs to next front
					B.FrontId = r + 1
					o.Fronts[r+1][fz[r+1]] = B
					fz[r+1]++
				}
			}
		}
	}

	// crowd distances
	for r := 0; r < nfronts; r++ {
		l, m, n := fz[r], fz[r]-1, fz[r]-2
		if l == 1 {
			o.Fronts[r][0].DistCrowd = -1
			continue
		}
		F := o.Fronts[r][:l]
		for j := 0; j < o.prms.Nova; j++ {
			SortByOva(F, j)
			δ := o.Omax[j] - o.Omin[j] + 1e-15
			if o.prms.use_metrics_inf_crowd_dist {
				F[0].DistCrowd = INF
				F[m].DistCrowd = INF
			} else {
				F[0].DistCrowd += math.Pow((F[1].Ova[j]-F[0].Ova[j])/δ, 2.0)
				F[m].DistCrowd += math.Pow((F[m].Ova[j]-F[n].Ova[j])/δ, 2.0)
			}
			for i := 1; i < m; i++ {
				F[i].DistCrowd += ((F[i].Ova[j] - F[i-1].Ova[j]) / δ) * ((F[i+1].Ova[j] - F[i].Ova[j]) / δ)
			}
		}
	}
	return
}

func (o *Metrics) closest(A, B *Solution, dist float64) {
	if dist < A.DistNeigh {
		A.DistNeigh = dist
		A.Closest = B
	}
	if dist < B.DistNeigh {
		B.DistNeigh = dist
		B.Closest = A
	}
}
<commit_msg>metrics organised a little<commit_after>// Copyright 2015 Dorival de Moraes Pedroso. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package goga

import (
	"math"

	"github.com/cpmech/gosl/chk"
	"github.com/cpmech/gosl/utl"
)

// Metrics holds metric data such as non-dominated Pareto fronts
type Metrics struct {
	prms   *Parameters   // parameters
	Omin   []float64     // current min ova
	Omax   []float64     // current max ova
	Fmin   []float64     // current min float
	Fmax   []float64     // current max float
	Imin   []int         // current min int
	Imax   []int         // current max int
	Fsizes []int         // front sizes
	Fronts [][]*Solution // non-dominated fronts
}

// Init initialises Metrics
func (o *Metrics) Init(nsol int, prms *Parameters) {
	o.prms = prms
	o.Omin = make([]float64, prms.Nova)
	o.Omax = make([]float64, prms.Nova)
	o.Fmin = make([]float64, prms.Nflt)
	o.Fmax = make([]float64, prms.Nflt)
	o.Imin = make([]int, prms.Nint)
	o.Imax = make([]int, prms.Nint)
	o.Fsizes = make([]int, nsol)
	o.Fronts = make([][]*Solution, nsol)
	for i := 0; i < nsol; i++ {
		o.Fronts[i] = make([]*Solution, nsol)
	}
}

// Compute computes all metric variables
func (o *Metrics) Compute(sols []*Solution) (nfronts int) {

	// reset counters and find limits
	fz := o.Fsizes
	nsol := len(sols)
	for i, sol := range sols {

		// reset values
		sol.Repeated = false
		sol.Nwins = 0
		sol.Nlosses = 0
		sol.FrontId = 0
		sol.DistCrowd = 0
		sol.DistNeigh = INF
		fz[i] = 0

		// ovas range
		for j := 0; j < o.prms.Nova; j++ {
			x := sol.Ova[j]
			if math.IsNaN(x) {
				chk.Panic("NaN found in objective value array\n\txFlt = %v\n\txInt = %v\n\tova = %v\n\toor = %v", sol.Flt, sol.Int, sol.Ova, sol.Oor)
			}
			if i == 0 {
				o.Omin[j] = x
				o.Omax[j] = x
			} else {
				o.Omin[j] = utl.Min(o.Omin[j], x)
				o.Omax[j] = utl.Max(o.Omax[j], x)
			}
		}

		// floats range
		for j := 0; j < o.prms.Nflt; j++ {
			x := sol.Flt[j]
			if i == 0 {
				o.Fmin[j] = x
				o.Fmax[j] = x
			} else {
				o.Fmin[j] = utl.Min(o.Fmin[j], x)
				o.Fmax[j] = utl.Max(o.Fmax[j], x)
			}
		}

		// ints range
		for j := 0; j < o.prms.Nint; j++ {
			x := sol.Int[j]
			if i == 0 {
				o.Imin[j] = x
				o.Imax[j] = x
			} else {
				o.Imin[j] = utl.Imin(o.Imin[j], x)
				o.Imax[j] = utl.Imax(o.Imax[j], x)
			}
		}
	}

	// compute neighbour distances
	dmax := o.calcdmax()
	for i := 0; i < nsol; i++ {
		A := sols[i]
		for j := i + 1; j < nsol; j++ {
			B := sols[j]
			o.closest(A, B, dmax, nsol)
		}
	}

	// compute dominance data
	for i := 0; i < nsol; i++ {
		A := sols[i]
		if A.Repeated {
			continue
		}
		for j := i + 1; j < nsol; j++ {
			B := sols[j]
			if B.Repeated {
				continue
			}
			A_dom, B_dom := A.Compare(B)
			if A_dom {
				A.WinOver[A.Nwins] = B // i dominates j
				A.Nwins++              // i has another dominated item
				B.Nlosses++            // j is being dominated by i
			}
			if B_dom {
				B.WinOver[B.Nwins] = A // j dominates i
				B.Nwins++              // j has another dominated item
				A.Nlosses++            // i is being dominated by j
			}
		}
	}

	// first front
	for _, sol := range sols {
		if sol.Repeated {
			continue
		}
		if sol.Nlosses == 0 {
			o.Fronts[0][fz[0]] = sol
			fz[0]++
		}
	}

	// next fronts
	for r, front := range o.Fronts {
		if fz[r] == 0 {
			break
		}
		nfronts++
		for s := 0; s < fz[r]; s++ {
			A := front[s]
			if A.Repeated {
				chk.Panic("_internal_error_ repeated is wrong 1")
			}
			for k := 0; k < A.Nwins; k++ {
				B := A.WinOver[k]
				if B.Repeated {
					chk.Panic("_internal_error_ repeatd is wrong 2")
				}
				B.Nlosses--
				if B.Nlosses == 0 { // B belongs to next front
					B.FrontId = r + 1
					o.Fronts[r+1][fz[r+1]] = B
					fz[r+1]++
				}
			}
		}
	}

	// crowd distances
	for r := 0; r < nfronts; r++ {
		l, m, n := fz[r], fz[r]-1, fz[r]-2
		if l == 1 {
			o.Fronts[r][0].DistCrowd = -1
			continue
		}
		F := o.Fronts[r][:l]
		for j := 0; j < o.prms.Nova; j++ {
			SortByOva(F, j)
			δ := o.Omax[j] - o.Omin[j] + 1e-15
			if o.prms.use_metrics_inf_crowd_dist {
				F[0].DistCrowd = INF
				F[m].DistCrowd = INF
			} else {
				F[0].DistCrowd += math.Pow((F[1].Ova[j]-F[0].Ova[j])/δ, 2.0)
				F[m].DistCrowd += math.Pow((F[m].Ova[j]-F[n].Ova[j])/δ, 2.0)
			}
			for i := 1; i < m; i++ {
				F[i].DistCrowd += ((F[i].Ova[j] - F[i-1].Ova[j]) / δ) * ((F[i+1].Ova[j] - F[i].Ova[j]) / δ)
			}
		}
	}
	return
}

func (o *Metrics) closest(A, B *Solution, dmax float64, nsol int) {
	var dist float64
	if o.prms.use_metrics_ovadistance {
		dist = A.OvaDistance(B, o.Omin, o.Omax)
	} else {
		dist = A.Distance(B, o.Fmin, o.Fmax, o.Imin, o.Imax)
	}
	if dist < A.DistNeigh {
		A.DistNeigh = dist
		A.Closest = B
	}
	if dist < B.DistNeigh {
		B.DistNeigh = dist
		B.Closest = A
	}
	if dist < o.prms.Mdmin*dmax {
		B.Repeated = o.prms.use_metrics_repeated_enabled
		if o.prms.use_metrics_repeated_enabled {
			B.FrontId = nsol
		}
	}
}

func (o *Metrics) calcdmax() (dmax float64) {
	dmax = 1e-15
	if o.prms.use_metrics_ovadistance {
		for i := 0; i < o.prms.Nova; i++ {
			dmax += math.Pow(o.Omax[i]-o.Omin[i], 2.0)
		}
	} else {
		for i := 0; i < o.prms.Nflt; i++ {
			dmax += math.Pow(o.Fmax[i]-o.Fmin[i], 2.0)
		}
	}
	dmax = math.Sqrt(dmax)
	return
}
<|endoftext|>


<commit_before>// Copyright 2017-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helpers

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/kevinburke/ssh_config"
	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
)

// SSHCommand stores the data associated with executing a command.
// TODO: this is poorly named in that it's not related to a command only
// ran over SSH - rename this.
type SSHCommand struct {
	// TODO: path is not a clear name - rename to something more clear.
	Path   string
	Env    []string
	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer
}

// SSHClient stores the information needed to SSH into a remote location for
// running tests.
type SSHClient struct {
	Config *ssh.ClientConfig // ssh client configuration information.
	Host   string            // Ip/Host from the target virtualserver
	Port   int               // Port to connect to the target server
	client *ssh.Client       // Client implements a traditional SSH client that supports shells,
	// subprocesses, TCP port/streamlocal forwarding and tunneled dialing.
}

// GetHostPort returns the host port representation of the ssh client
func (cli *SSHClient) GetHostPort() string {
	return net.JoinHostPort(cli.Host, strconv.Itoa(cli.Port))
}

// SSHConfig contains metadata for an SSH session.
type SSHConfig struct {
	target       string
	host         string
	user         string
	port         int
	identityFile string
}

// SSHConfigs maps the name of a host (VM) to its corresponding SSHConfiguration
type SSHConfigs map[string]*SSHConfig

// GetSSHClient initializes an SSHClient based on the provided SSHConfig
func (cfg *SSHConfig) GetSSHClient() *SSHClient {
	sshConfig := &ssh.ClientConfig{
		User: cfg.user,
		Auth: []ssh.AuthMethod{
			cfg.GetSSHAgent(),
		},
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
		Timeout:         15 * time.Second,
	}

	return &SSHClient{
		Config: sshConfig,
		Host:   cfg.host,
		Port:   cfg.port,
	}
}

func (client *SSHClient) String() string {
	return fmt.Sprintf("host: %s, port: %d, user: %s", client.Host, client.Port, client.Config.User)
}

func (cfg *SSHConfig) String() string {
	return fmt.Sprintf("target: %s, host: %s, port %d, user, %s, identityFile: %s", cfg.target, cfg.host, cfg.port, cfg.user, cfg.identityFile)
}

// GetSSHAgent returns the ssh.AuthMethod corresponding to SSHConfig cfg.
func (cfg *SSHConfig) GetSSHAgent() ssh.AuthMethod {
	key, err := ioutil.ReadFile(cfg.identityFile)
	if err != nil {
		log.Fatalf("unable to retrieve ssh-key on target '%s': %s", cfg.target, err)
	}

	signer, err := ssh.ParsePrivateKey(key)
	if err != nil {
		log.Fatalf("unable to parse private key on target '%s': %s", cfg.target, err)
	}
	return ssh.PublicKeys(signer)
}

// ImportSSHconfig imports the SSH configuration stored at the provided path.
// Returns an error if the SSH configuration could not be instantiated.
func ImportSSHconfig(config []byte) (SSHConfigs, error) {
	result := make(SSHConfigs)
	cfg, err := ssh_config.Decode(bytes.NewBuffer(config))
	if err != nil {
		return nil, err
	}

	for _, host := range cfg.Hosts {
		key := host.Patterns[0].String()
		if key == "*" {
			continue
		}
		port, _ := cfg.Get(key, "Port")
		hostConfig := SSHConfig{target: key}
		hostConfig.host, _ = cfg.Get(key, "Hostname")
		hostConfig.identityFile, _ = cfg.Get(key, "identityFile")
		hostConfig.user, _ = cfg.Get(key, "User")
		hostConfig.port, _ = strconv.Atoi(port)
		result[key] = &hostConfig
	}
	return result, nil
}

// copyWait runs an instance of io.Copy() in a goroutine, and returns a channel
// to receive the error result.
func copyWait(dst io.Writer, src io.Reader) chan error {
	c := make(chan error)
	go func() {
		_, err := io.Copy(dst, src)
		c <- err
	}()
	return c
}

// runCommand runs the specified command on the provided SSH session, and
// gathers both of the sterr and stdout output into the writers provided by
// cmd. Returns nil when the command completes successfully and all stderr,
// stdout output has been written. Returns an error otherwise.
func runCommand(session *ssh.Session, cmd *SSHCommand) error {
	stderr, err := session.StderrPipe()
	if err != nil {
		return fmt.Errorf("Unable to setup stderr for session: %v", err)
	}
	errChan := copyWait(cmd.Stderr, stderr)

	stdout, err := session.StdoutPipe()
	if err != nil {
		return fmt.Errorf("Unable to setup stdout for session: %v", err)
	}
	outChan := copyWait(cmd.Stdout, stdout)

	if err = session.Run(cmd.Path); err != nil {
		return err
	}

	if err = <-errChan; err != nil {
		return err
	}
	if err = <-outChan; err != nil {
		return err
	}
	return nil
}

// RunCommand runs a SSHCommand using SSHClient client. The returned error is
// nil if the command runs, has no problems copying stdin, stdout, and stderr,
// and exits with a zero exit status.
func (client *SSHClient) RunCommand(cmd *SSHCommand) error {
	session, err := client.newSession()
	if err != nil {
		return err
	}
	defer session.Close()

	return runCommand(session, cmd)
}

// RunCommandInBackground runs an SSH command in a similar way to
// RunCommandContext, but with a context which allows the command to be
// cancelled at any time. When cancel is called the error of the command is
// returned instead the context error.
func (client *SSHClient) RunCommandInBackground(ctx context.Context, cmd *SSHCommand) error {
	if ctx == nil {
		panic("nil context provided to RunCommandInBackground()")
	}

	session, err := client.newSession()
	if err != nil {
		return err
	}
	defer session.Close()

	modes := ssh.TerminalModes{
		ssh.ECHO:          1,     // enable echoing
		ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	}
	session.RequestPty("xterm-256color", 80, 80, modes)

	stdin, err := session.StdinPipe()
	if err != nil {
		log.Errorf("Could not get stdin: %s", err)
	}

	go func() {
		select {
		case <-ctx.Done():
			_, err := stdin.Write([]byte{3})
			if err != nil {
				log.Errorf("write ^C error: %s", err)
			}
			err = session.Wait()
			if err != nil {
				log.Errorf("wait error: %s", err)
			}
			if err = session.Signal(ssh.SIGHUP); err != nil {
				log.Errorf("failed to kill command: %s", err)
			}
			if err = session.Close(); err != nil {
				log.Errorf("failed to close session: %s", err)
			}
		}
	}()
	return runCommand(session, cmd)
}

// RunCommandContext runs an SSH command in a similar way to RunCommand but with
// a context. If context is canceled it will return the error of that given
// context.
func (client *SSHClient) RunCommandContext(ctx context.Context, cmd *SSHCommand) error {
	if ctx == nil {
		panic("nil context provided to RunCommandContext()")
	}

	session, err := client.newSession()
	if err != nil {
		return err
	}
	defer session.Close()

	stdin, err := session.StdinPipe()
	if err != nil {
		log.Errorf("Could not get stdin %s", err)
	}

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		select {
		case <-ctx.Done():
			_, err := stdin.Write([]byte{3})
			if err != nil {
				log.Errorf("write ^C error: %s", err)
			}
			err = session.Wait()
			if err != nil {
				log.Errorf("wait error: %s", err)
			}
			if err = session.Signal(ssh.SIGHUP); err != nil {
				log.Errorf("failed to kill command: %s", err)
			}
			if err = session.Close(); err != nil {
				log.Errorf("failed to close session: %s", err)
			}
		}
		wg.Done()
	}()
	err = runCommand(session, cmd)
	select {
	case <-ctx.Done():
		// Wait until the ssh session is stopped
		wg.Wait()
		return ctx.Err()
	default:
		return err
	}
}

func (client *SSHClient) newSession() (*ssh.Session, error) {
	var connection *ssh.Client
	var err error

	if client.client != nil {
		connection = client.client
	} else {
		connection, err = ssh.Dial(
			"tcp",
			fmt.Sprintf("%s:%d", client.Host, client.Port),
			client.Config)

		if err != nil {
			return nil, fmt.Errorf("failed to dial: %s", err)
		}
		client.client = connection
	}

	session, err := connection.NewSession()
	if err != nil {
		return nil, fmt.Errorf("failed to create session: %s", err)
	}

	return session, nil
}

// SSHAgent returns the ssh.Authmethod using the Public keys. Returns nil if
// a connection to SSH_AUTH_SHOCK does not succeed.
func SSHAgent() ssh.AuthMethod {
	if sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")); err == nil {
		return ssh.PublicKeysCallback(agent.NewClient(sshAgent).Signers)
	}
	return nil
}

// GetSSHClient initializes an SSHClient for the specified host/port/user
// combination.
func GetSSHClient(host string, port int, user string) *SSHClient {

	sshConfig := &ssh.ClientConfig{
		User: user,
		Auth: []ssh.AuthMethod{
			SSHAgent(),
		},
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
		Timeout:         15 * time.Second,
	}

	return &SSHClient{
		Config: sshConfig,
		Host:   host,
		Port:   port,
	}

}
<commit_msg>test: Fix hang when endpoints never become ready<commit_after>// Copyright 2017-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helpers

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/kevinburke/ssh_config"
	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
)

// SSHCommand stores the data associated with executing a command.
// TODO: this is poorly named in that it's not related to a command only
// ran over SSH - rename this.
type SSHCommand struct {
	// TODO: path is not a clear name - rename to something more clear.
	Path   string
	Env    []string
	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer
}

// SSHClient stores the information needed to SSH into a remote location for
// running tests.
type SSHClient struct {
	Config *ssh.ClientConfig // ssh client configuration information.
	Host   string            // Ip/Host from the target virtualserver
	Port   int               // Port to connect to the target server
	client *ssh.Client       // Client implements a traditional SSH client that supports shells,
	// subprocesses, TCP port/streamlocal forwarding and tunneled dialing.
}

// GetHostPort returns the host port representation of the ssh client
func (cli *SSHClient) GetHostPort() string {
	return net.JoinHostPort(cli.Host, strconv.Itoa(cli.Port))
}

// SSHConfig contains metadata for an SSH session.
type SSHConfig struct {
	target       string
	host         string
	user         string
	port         int
	identityFile string
}

// SSHConfigs maps the name of a host (VM) to its corresponding SSHConfiguration
type SSHConfigs map[string]*SSHConfig

// GetSSHClient initializes an SSHClient based on the provided SSHConfig
func (cfg *SSHConfig) GetSSHClient() *SSHClient {
	sshConfig := &ssh.ClientConfig{
		User: cfg.user,
		Auth: []ssh.AuthMethod{
			cfg.GetSSHAgent(),
		},
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
		Timeout:         15 * time.Second,
	}

	return &SSHClient{
		Config: sshConfig,
		Host:   cfg.host,
		Port:   cfg.port,
	}
}

func (client *SSHClient) String() string {
	return fmt.Sprintf("host: %s, port: %d, user: %s", client.Host, client.Port, client.Config.User)
}

func (cfg *SSHConfig) String() string {
	return fmt.Sprintf("target: %s, host: %s, port %d, user, %s, identityFile: %s", cfg.target, cfg.host, cfg.port, cfg.user, cfg.identityFile)
}

// GetSSHAgent returns the ssh.AuthMethod corresponding to SSHConfig cfg.
func (cfg *SSHConfig) GetSSHAgent() ssh.AuthMethod {
	key, err := ioutil.ReadFile(cfg.identityFile)
	if err != nil {
		log.Fatalf("unable to retrieve ssh-key on target '%s': %s", cfg.target, err)
	}

	signer, err := ssh.ParsePrivateKey(key)
	if err != nil {
		log.Fatalf("unable to parse private key on target '%s': %s", cfg.target, err)
	}
	return ssh.PublicKeys(signer)
}

// ImportSSHconfig imports the SSH configuration stored at the provided path.
// Returns an error if the SSH configuration could not be instantiated.
func ImportSSHconfig(config []byte) (SSHConfigs, error) {
	result := make(SSHConfigs)
	cfg, err := ssh_config.Decode(bytes.NewBuffer(config))
	if err != nil {
		return nil, err
	}

	for _, host := range cfg.Hosts {
		key := host.Patterns[0].String()
		if key == "*" {
			continue
		}
		port, _ := cfg.Get(key, "Port")
		hostConfig := SSHConfig{target: key}
		hostConfig.host, _ = cfg.Get(key, "Hostname")
		hostConfig.identityFile, _ = cfg.Get(key, "identityFile")
		hostConfig.user, _ = cfg.Get(key, "User")
		hostConfig.port, _ = strconv.Atoi(port)
		result[key] = &hostConfig
	}
	return result, nil
}

// copyWait runs an instance of io.Copy() in a goroutine, and returns a channel
// to receive the error result.
func copyWait(dst io.Writer, src io.Reader) chan error {
	c := make(chan error)
	go func() {
		_, err := io.Copy(dst, src)
		c <- err
	}()
	return c
}

// runCommand runs the specified command on the provided SSH session, and
// gathers both of the sterr and stdout output into the writers provided by
// cmd. Returns whether the command was run and an optional error.
// Returns nil when the command completes successfully and all stderr,
// stdout output has been written. Returns an error otherwise.
func runCommand(session *ssh.Session, cmd *SSHCommand) (bool, error) {
	stderr, err := session.StderrPipe()
	if err != nil {
		return false, fmt.Errorf("Unable to setup stderr for session: %v", err)
	}
	errChan := copyWait(cmd.Stderr, stderr)

	stdout, err := session.StdoutPipe()
	if err != nil {
		return false, fmt.Errorf("Unable to setup stdout for session: %v", err)
	}
	outChan := copyWait(cmd.Stdout, stdout)

	if err = session.Run(cmd.Path); err != nil {
		return false, err
	}

	if err = <-errChan; err != nil {
		return true, err
	}
	if err = <-outChan; err != nil {
		return true, err
	}
	return true, nil
}

// RunCommand runs a SSHCommand using SSHClient client. The returned error is
// nil if the command runs, has no problems copying stdin, stdout, and stderr,
// and exits with a zero exit status.
func (client *SSHClient) RunCommand(cmd *SSHCommand) error {
	session, err := client.newSession()
	if err != nil {
		return err
	}
	defer session.Close()

	_, err = runCommand(session, cmd)
	return err
}

// RunCommandInBackground runs an SSH command in a similar way to
// RunCommandContext, but with a context which allows the command to be
// cancelled at any time. When cancel is called the error of the command is
// returned instead the context error.
func (client *SSHClient) RunCommandInBackground(ctx context.Context, cmd *SSHCommand) error {
	if ctx == nil {
		panic("nil context provided to RunCommandInBackground()")
	}

	session, err := client.newSession()
	if err != nil {
		return err
	}
	defer session.Close()

	modes := ssh.TerminalModes{
		ssh.ECHO:          1,     // enable echoing
		ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	}
	session.RequestPty("xterm-256color", 80, 80, modes)

	stdin, err := session.StdinPipe()
	if err != nil {
		log.Errorf("Could not get stdin: %s", err)
	}

	go func() {
		select {
		case <-ctx.Done():
			_, err := stdin.Write([]byte{3})
			if err != nil {
				log.Errorf("write ^C error: %s", err)
			}
			err = session.Wait()
			if err != nil {
				log.Errorf("wait error: %s", err)
			}
			if err = session.Signal(ssh.SIGHUP); err != nil {
				log.Errorf("failed to kill command: %s", err)
			}
			if err = session.Close(); err != nil {
				log.Errorf("failed to close session: %s", err)
			}
		}
	}()
	_, err = runCommand(session, cmd)
	return err
}

// RunCommandContext runs an SSH command in a similar way to RunCommand but with
// a context. If context is canceled it will return the error of that given
// context.
func (client *SSHClient) RunCommandContext(ctx context.Context, cmd *SSHCommand) error {
	if ctx == nil {
		panic("nil context provided to RunCommandContext()")
	}

	session, err := client.newSession()
	if err != nil {
		return err
	}
	defer session.Close()

	stdin, err := session.StdinPipe()
	if err != nil {
		log.Errorf("Could not get stdin %s", err)
	}

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		select {
		case <-ctx.Done():
			_, err := stdin.Write([]byte{3})
			if err != nil {
				log.Errorf("write ^C error: %s", err)
			}
			err = session.Wait()
			if err != nil {
				log.Errorf("wait error: %s", err)
			}
			if err = session.Signal(ssh.SIGHUP); err != nil {
				log.Errorf("failed to kill command: %s", err)
			}
			if err = session.Close(); err != nil {
				log.Errorf("failed to close session: %s", err)
			}
		}
		wg.Done()
	}()

	running, err := runCommand(session, cmd)
	if !running {
		return err
	}
	select {
	case <-ctx.Done():
		// Wait until the ssh session is stopped
		wg.Wait()
		return ctx.Err()
	default:
		return err
	}
}

func (client *SSHClient) newSession() (*ssh.Session, error) {
	var connection *ssh.Client
	var err error

	if client.client != nil {
		connection = client.client
	} else {
		connection, err = ssh.Dial(
			"tcp",
			fmt.Sprintf("%s:%d", client.Host, client.Port),
			client.Config)

		if err != nil {
			return nil, fmt.Errorf("failed to dial: %s", err)
		}
		client.client = connection
	}

	session, err := connection.NewSession()
	if err != nil {
		return nil, fmt.Errorf("failed to create session: %s", err)
	}

	return session, nil
}

// SSHAgent returns the ssh.Authmethod using the Public keys. Returns nil if
// a connection to SSH_AUTH_SHOCK does not succeed.
func SSHAgent() ssh.AuthMethod {
	if sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")); err == nil {
		return ssh.PublicKeysCallback(agent.NewClient(sshAgent).Signers)
	}
	return nil
}

// GetSSHClient initializes an SSHClient for the specified host/port/user
// combination.
func GetSSHClient(host string, port int, user string) *SSHClient {

	sshConfig := &ssh.ClientConfig{
		User: user,
		Auth: []ssh.AuthMethod{
			SSHAgent(),
		},
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
		Timeout:         15 * time.Second,
	}

	return &SSHClient{
		Config: sshConfig,
		Host:   host,
		Port:   port,
	}

}
<|endoftext|>