<commit_before>package protocol

import (
	"errors"
	"io"
	"time"
)

// RecordReader is an interface representing a sequence of records. Record sets
// are used in both produce and fetch requests to represent the sequence of
// records that are sent to or receive from kafka brokers.
//
// RecordSet values are not safe to use concurrently from multiple goroutines.
type RecordReader interface {
	// Returns the next record in the set, or io.EOF if the end of the sequence
	// has been reached.
	//
	// The returned Record is guaranteed to be valid until the next call to
	// ReadRecord. If the program needs to retain the Record value it must make
	// a copy.
	ReadRecord() (*Record, error)
}

// NewRecordReader constructs a reader exposing the records passed as arguments.
func NewRecordReader(records ...Record) RecordReader {
	switch len(records) {
	case 0:
		return emptyRecordReader{}
	default:
		r := &recordReader{records: make([]Record, len(records))}
		copy(r.records, records)
		return r
	}
}

// MultiRecordReader merges multiple record batches into one.
func MultiRecordReader(batches ...RecordReader) RecordReader {
	switch len(batches) {
	case 0:
		return emptyRecordReader{}
	case 1:
		return batches[0]
	default:
		m := &multiRecordReader{batches: make([]RecordReader, len(batches))}
		copy(m.batches, batches)
		return m
	}
}

// ResetRecordReader is a helper function to reset record batches that have a
// `Reset()` method.
func ResetRecordReader(rb RecordReader) error {
	if r, _ := rb.(resetter); r != nil {
		return r.Reset()
	}
	return ErrNoReset
}

type resetter interface {
	Reset() error
}

func forEachRecord(r RecordReader, f func(int, *Record) error) error {
	for i := 0; ; i++ {
		rec, err := r.ReadRecord()

		if err != nil {
			if errors.Is(err, io.EOF) {
				err = nil
			}
			return err
		}

		if err := handleRecord(i, rec, f); err != nil {
			return err
		}
	}
}

func handleRecord(i int, r *Record, f func(int, *Record) error) error {
	if r.Key != nil {
		defer r.Key.Close()
	}
	if r.Value != nil {
		defer r.Value.Close()
	}
	if r.Offset == 0 {
		r.Offset = int64(i)
	}
	return f(i, r)
}

type recordReader struct {
	records []Record
	index   int
}

func (r *recordReader) ReadRecord() (*Record, error) {
	if i := r.index; i >= 0 && i < len(r.records) {
		r.index++
		return &r.records[i], nil
	}
	return nil, io.EOF
}

func (r *recordReader) Reset() error {
	r.index = 0
	for i := range r.records {
		r := &r.records[i]
		resetBytes(r.Key)
		resetBytes(r.Value)
	}
	return nil
}

type multiRecordReader struct {
	batches []RecordReader
	index   int
}

func (m *multiRecordReader) ReadRecord() (*Record, error) {
	for {
		if m.index == len(m.batches) {
			return nil, io.EOF
		}
		r, err := m.batches[m.index].ReadRecord()
		if err == nil {
			return r, nil
		}
		if !errors.Is(err, io.EOF) {
			return nil, err
		}
		m.index++
	}
}

func (m *multiRecordReader) Reset() (err error) {
	m.index, err = resetRecordReaders(m.batches[:m.index])
	return
}

func concatRecordReader(head RecordReader, tail RecordReader) RecordReader {
	if head == nil {
		return tail
	}
	if m, _ := head.(*multiRecordReader); m != nil {
		m.batches = append(m.batches, tail)
		return m
	}
	return MultiRecordReader(head, tail)
}

// optimizedRecordReader is an implementation of a RecordReader which exposes a
// sequence
type optimizedRecordReader struct {
	records []optimizedRecord
	index   int
	buffer  Record
	headers [][]Header
}

func (r *optimizedRecordReader) Reset() error {
	for i := range r.records {
		resetBytes(r.records[i].key())
		resetBytes(r.records[i].value())
	}
	return nil
}

func (r *optimizedRecordReader) ReadRecord() (*Record, error) {
	if i := r.index; i >= 0 && i < len(r.records) {
		rec := &r.records[i]
		r.index++
		r.buffer = Record{
			Offset:  rec.offset,
			Time:    rec.time(),
			Key:     rec.key(),
			Value:   rec.value(),
			Headers: r.headers[i],
		}
		return &r.buffer, nil
	}
	return nil, io.EOF
}

type optimizedRecord struct {
	offset    int64
	timestamp int64
	keyRef    *pageRef
	valueRef  *pageRef
}

func (r *optimizedRecord) time() time.Time {
	return makeTime(r.timestamp)
}

func (r *optimizedRecord) key() Bytes {
	return makeBytes(r.keyRef)
}

func (r *optimizedRecord) value() Bytes {
	return makeBytes(r.valueRef)
}

func makeBytes(ref *pageRef) Bytes {
	if ref == nil {
		return nil
	}
	return ref
}

type emptyRecordReader struct{}

func (emptyRecordReader) ReadRecord() (*Record, error) { return nil, io.EOF }

// ControlRecord represents a record read from a control batch.
type ControlRecord struct {
	Offset  int64
	Time    time.Time
	Version int16
	Type    int16
	Data    []byte
	Headers []Header
}

func ReadControlRecord(r *Record) (*ControlRecord, error) {
	k, err := ReadFull(r.Key)
	if err != nil {
		return nil, err
	}
	if k == nil {
		return nil, Error("invalid control record with nil key")
	}
	if len(k) != 4 {
		return nil, Errorf("invalid control record with key of size %d", len(k))
	}

	v, err := ReadFull(r.Value)
	if err != nil {
		return nil, err
	}

	c := &ControlRecord{
		Offset:  r.Offset,
		Time:    r.Time,
		Version: readInt16(k[:2]),
		Type:    readInt16(k[2:]),
		Data:    v,
		Headers: r.Headers,
	}

	return c, nil
}

func (cr *ControlRecord) Key() Bytes {
	k := make([]byte, 4)
	writeInt16(k[:2], cr.Version)
	writeInt16(k[2:], cr.Type)
	return NewBytes(k)
}

func (cr *ControlRecord) Value() Bytes {
	return NewBytes(cr.Data)
}

func (cr *ControlRecord) Record() Record {
	return Record{
		Offset:  cr.Offset,
		Time:    cr.Time,
		Key:     cr.Key(),
		Value:   cr.Value(),
		Headers: cr.Headers,
	}
}

// ControlBatch is an implementation of the RecordReader interface representing
// control batches returned by kafka brokers.
type ControlBatch struct {
	Attributes           Attributes
	PartitionLeaderEpoch int32
	BaseOffset           int64
	ProducerID           int64
	ProducerEpoch        int16
	BaseSequence         int32
	Records              RecordReader
}

// NewControlBatch constructs a control batch from the list of records passed as
// arguments.
func NewControlBatch(records ...ControlRecord) *ControlBatch {
	rawRecords := make([]Record, len(records))
	for i, cr := range records {
		rawRecords[i] = cr.Record()
	}
	return &ControlBatch{
		Records: NewRecordReader(rawRecords...),
	}
}

func (c *ControlBatch) ReadRecord() (*Record, error) {
	return c.Records.ReadRecord()
}

func (c *ControlBatch) ReadControlRecord() (*ControlRecord, error) {
	r, err := c.ReadRecord()
	if err != nil {
		return nil, err
	}
	if r.Key != nil {
		defer r.Key.Close()
	}
	if r.Value != nil {
		defer r.Value.Close()
	}
	return ReadControlRecord(r)
}

func (c *ControlBatch) Reset() error {
	return ResetRecordReader(c.Records)
}

func (c *ControlBatch) Offset() int64 {
	return c.BaseOffset
}

func (c *ControlBatch) Version() int {
	return 2
}

// RecordBatch is an implementation of the RecordReader interface representing
// regular record batches (v2).
type RecordBatch struct {
	Attributes           Attributes
	PartitionLeaderEpoch int32
	BaseOffset           int64
	ProducerID           int64
	ProducerEpoch        int16
	BaseSequence         int32
	Records              RecordReader
}

func (r *RecordBatch) ReadRecord() (*Record, error) {
	return r.Records.ReadRecord()
}

func (r *RecordBatch) Reset() error {
	return ResetRecordReader(r.Records)
}

func (r *RecordBatch) Offset() int64 {
	return r.BaseOffset
}

func (r *RecordBatch) Version() int {
	return 2
}

// MessageSet is an implementation of the RecordReader interface representing
// regular message sets (v1).
type MessageSet struct {
	Attributes Attributes
	BaseOffset int64
	Records    RecordReader
}

func (m *MessageSet) ReadRecord() (*Record, error) {
	return m.Records.ReadRecord()
}

func (m *MessageSet) Reset() error {
	return ResetRecordReader(m.Records)
}

func (m *MessageSet) Offset() int64 {
	return m.BaseOffset
}

func (m *MessageSet) Version() int {
	return 1
}

// RecordStream is an implementation of the RecordReader interface which
// combines multiple underlying RecordReader and only expose records that
// are not from control batches.
type RecordStream struct {
	Records []RecordReader
	index   int
}

func (s *RecordStream) ReadRecord() (*Record, error) {
	for {
		if s.index < 0 || s.index >= len(s.Records) {
			return nil, io.EOF
		}

		if _, isControl := s.Records[s.index].(*ControlBatch); isControl {
			s.index++
			continue
		}

		r, err := s.Records[s.index].ReadRecord()
		if err != nil {
			if errors.Is(err, io.EOF) {
				s.index++
				continue
			}
		}

		return r, err
	}
}

func (s *RecordStream) Reset() (err error) {
	s.index, err = resetRecordReaders(s.Records[:s.index])
	return
}

type message struct {
	Record Record
	read   bool
}

func (m *message) ReadRecord() (*Record, error) {
	if m.read {
		return nil, io.EOF
	}
	m.read = true
	return &m.Record, nil
}

func (m *message) Reset() error {
	m.read = false
	return nil
}

func (m *message) Offset() int64 {
	return m.Record.Offset
}

func (m *message) Version() int {
	return 1
}

func resetRecordReaders(records []RecordReader) (int, error) {
	i := len(records) - 1

	for i >= 0 {
		if err := ResetRecordReader(records[i]); err != nil {
			return i + 1, err
		}
		i--
	}

	return i + 1, nil
}
<commit_msg>fix panic accessing optimized record reader with a nil headers slice<commit_after>package protocol

import (
	"errors"
	"io"
	"time"
)

// RecordReader is an interface representing a sequence of records. Record sets
// are used in both produce and fetch requests to represent the sequence of
// records that are sent to or receive from kafka brokers.
//
// RecordSet values are not safe to use concurrently from multiple goroutines.
type RecordReader interface {
	// Returns the next record in the set, or io.EOF if the end of the sequence
	// has been reached.
	//
	// The returned Record is guaranteed to be valid until the next call to
	// ReadRecord. If the program needs to retain the Record value it must make
	// a copy.
	ReadRecord() (*Record, error)
}

// NewRecordReader constructs a reader exposing the records passed as arguments.
func NewRecordReader(records ...Record) RecordReader {
	switch len(records) {
	case 0:
		return emptyRecordReader{}
	default:
		r := &recordReader{records: make([]Record, len(records))}
		copy(r.records, records)
		return r
	}
}

// MultiRecordReader merges multiple record batches into one.
func MultiRecordReader(batches ...RecordReader) RecordReader {
	switch len(batches) {
	case 0:
		return emptyRecordReader{}
	case 1:
		return batches[0]
	default:
		m := &multiRecordReader{batches: make([]RecordReader, len(batches))}
		copy(m.batches, batches)
		return m
	}
}

// ResetRecordReader is a helper function to reset record batches that have a
// `Reset()` method.
func ResetRecordReader(rb RecordReader) error {
	if r, _ := rb.(resetter); r != nil {
		return r.Reset()
	}
	return ErrNoReset
}

type resetter interface {
	Reset() error
}

func forEachRecord(r RecordReader, f func(int, *Record) error) error {
	for i := 0; ; i++ {
		rec, err := r.ReadRecord()

		if err != nil {
			if errors.Is(err, io.EOF) {
				err = nil
			}
			return err
		}

		if err := handleRecord(i, rec, f); err != nil {
			return err
		}
	}
}

func handleRecord(i int, r *Record, f func(int, *Record) error) error {
	if r.Key != nil {
		defer r.Key.Close()
	}
	if r.Value != nil {
		defer r.Value.Close()
	}
	if r.Offset == 0 {
		r.Offset = int64(i)
	}
	return f(i, r)
}

type recordReader struct {
	records []Record
	index   int
}

func (r *recordReader) ReadRecord() (*Record, error) {
	if i := r.index; i >= 0 && i < len(r.records) {
		r.index++
		return &r.records[i], nil
	}
	return nil, io.EOF
}

func (r *recordReader) Reset() error {
	r.index = 0
	for i := range r.records {
		r := &r.records[i]
		resetBytes(r.Key)
		resetBytes(r.Value)
	}
	return nil
}

type multiRecordReader struct {
	batches []RecordReader
	index   int
}

func (m *multiRecordReader) ReadRecord() (*Record, error) {
	for {
		if m.index == len(m.batches) {
			return nil, io.EOF
		}
		r, err := m.batches[m.index].ReadRecord()
		if err == nil {
			return r, nil
		}
		if !errors.Is(err, io.EOF) {
			return nil, err
		}
		m.index++
	}
}

func (m *multiRecordReader) Reset() (err error) {
	m.index, err = resetRecordReaders(m.batches[:m.index])
	return
}

func concatRecordReader(head RecordReader, tail RecordReader) RecordReader {
	if head == nil {
		return tail
	}
	if m, _ := head.(*multiRecordReader); m != nil {
		m.batches = append(m.batches, tail)
		return m
	}
	return MultiRecordReader(head, tail)
}

// optimizedRecordReader is an implementation of a RecordReader which exposes a
// sequence
type optimizedRecordReader struct {
	records []optimizedRecord
	index   int
	buffer  Record
	headers [][]Header
}

func (r *optimizedRecordReader) Reset() error {
	for i := range r.records {
		resetBytes(r.records[i].key())
		resetBytes(r.records[i].value())
	}
	return nil
}

func (r *optimizedRecordReader) ReadRecord() (*Record, error) {
	if i := r.index; i >= 0 && i < len(r.records) {
		rec := &r.records[i]
		r.index++
		r.buffer = Record{
			Offset: rec.offset,
			Time:   rec.time(),
			Key:    rec.key(),
			Value:  rec.value(),
		}
		if i < len(r.headers) {
			r.buffer.Headers = r.headers[i]
		}
		return &r.buffer, nil
	}
	return nil, io.EOF
}

type optimizedRecord struct {
	offset    int64
	timestamp int64
	keyRef    *pageRef
	valueRef  *pageRef
}

func (r *optimizedRecord) time() time.Time {
	return makeTime(r.timestamp)
}

func (r *optimizedRecord) key() Bytes {
	return makeBytes(r.keyRef)
}

func (r *optimizedRecord) value() Bytes {
	return makeBytes(r.valueRef)
}

func makeBytes(ref *pageRef) Bytes {
	if ref == nil {
		return nil
	}
	return ref
}

type emptyRecordReader struct{}

func (emptyRecordReader) ReadRecord() (*Record, error) { return nil, io.EOF }

// ControlRecord represents a record read from a control batch.
type ControlRecord struct {
	Offset  int64
	Time    time.Time
	Version int16
	Type    int16
	Data    []byte
	Headers []Header
}

func ReadControlRecord(r *Record) (*ControlRecord, error) {
	k, err := ReadFull(r.Key)
	if err != nil {
		return nil, err
	}
	if k == nil {
		return nil, Error("invalid control record with nil key")
	}
	if len(k) != 4 {
		return nil, Errorf("invalid control record with key of size %d", len(k))
	}

	v, err := ReadFull(r.Value)
	if err != nil {
		return nil, err
	}

	c := &ControlRecord{
		Offset:  r.Offset,
		Time:    r.Time,
		Version: readInt16(k[:2]),
		Type:    readInt16(k[2:]),
		Data:    v,
		Headers: r.Headers,
	}

	return c, nil
}

func (cr *ControlRecord) Key() Bytes {
	k := make([]byte, 4)
	writeInt16(k[:2], cr.Version)
	writeInt16(k[2:], cr.Type)
	return NewBytes(k)
}

func (cr *ControlRecord) Value() Bytes {
	return NewBytes(cr.Data)
}

func (cr *ControlRecord) Record() Record {
	return Record{
		Offset:  cr.Offset,
		Time:    cr.Time,
		Key:     cr.Key(),
		Value:   cr.Value(),
		Headers: cr.Headers,
	}
}

// ControlBatch is an implementation of the RecordReader interface representing
// control batches returned by kafka brokers.
type ControlBatch struct {
	Attributes           Attributes
	PartitionLeaderEpoch int32
	BaseOffset           int64
	ProducerID           int64
	ProducerEpoch        int16
	BaseSequence         int32
	Records              RecordReader
}

// NewControlBatch constructs a control batch from the list of records passed as
// arguments.
func NewControlBatch(records ...ControlRecord) *ControlBatch {
	rawRecords := make([]Record, len(records))
	for i, cr := range records {
		rawRecords[i] = cr.Record()
	}
	return &ControlBatch{
		Records: NewRecordReader(rawRecords...),
	}
}

func (c *ControlBatch) ReadRecord() (*Record, error) {
	return c.Records.ReadRecord()
}

func (c *ControlBatch) ReadControlRecord() (*ControlRecord, error) {
	r, err := c.ReadRecord()
	if err != nil {
		return nil, err
	}
	if r.Key != nil {
		defer r.Key.Close()
	}
	if r.Value != nil {
		defer r.Value.Close()
	}
	return ReadControlRecord(r)
}

func (c *ControlBatch) Reset() error {
	return ResetRecordReader(c.Records)
}

func (c *ControlBatch) Offset() int64 {
	return c.BaseOffset
}

func (c *ControlBatch) Version() int {
	return 2
}

// RecordBatch is an implementation of the RecordReader interface representing
// regular record batches (v2).
type RecordBatch struct {
	Attributes           Attributes
	PartitionLeaderEpoch int32
	BaseOffset           int64
	ProducerID           int64
	ProducerEpoch        int16
	BaseSequence         int32
	Records              RecordReader
}

func (r *RecordBatch) ReadRecord() (*Record, error) {
	return r.Records.ReadRecord()
}

func (r *RecordBatch) Reset() error {
	return ResetRecordReader(r.Records)
}

func (r *RecordBatch) Offset() int64 {
	return r.BaseOffset
}

func (r *RecordBatch) Version() int {
	return 2
}

// MessageSet is an implementation of the RecordReader interface representing
// regular message sets (v1).
type MessageSet struct {
	Attributes Attributes
	BaseOffset int64
	Records    RecordReader
}

func (m *MessageSet) ReadRecord() (*Record, error) {
	return m.Records.ReadRecord()
}

func (m *MessageSet) Reset() error {
	return ResetRecordReader(m.Records)
}

func (m *MessageSet) Offset() int64 {
	return m.BaseOffset
}

func (m *MessageSet) Version() int {
	return 1
}

// RecordStream is an implementation of the RecordReader interface which
// combines multiple underlying RecordReader and only expose records that
// are not from control batches.
type RecordStream struct {
	Records []RecordReader
	index   int
}

func (s *RecordStream) ReadRecord() (*Record, error) {
	for {
		if s.index < 0 || s.index >= len(s.Records) {
			return nil, io.EOF
		}

		if _, isControl := s.Records[s.index].(*ControlBatch); isControl {
			s.index++
			continue
		}

		r, err := s.Records[s.index].ReadRecord()
		if err != nil {
			if errors.Is(err, io.EOF) {
				s.index++
				continue
			}
		}

		return r, err
	}
}

func (s *RecordStream) Reset() (err error) {
	s.index, err = resetRecordReaders(s.Records[:s.index])
	return
}

type message struct {
	Record Record
	read   bool
}

func (m *message) ReadRecord() (*Record, error) {
	if m.read {
		return nil, io.EOF
	}
	m.read = true
	return &m.Record, nil
}

func (m *message) Reset() error {
	m.read = false
	return nil
}

func (m *message) Offset() int64 {
	return m.Record.Offset
}

func (m *message) Version() int {
	return 1
}

func resetRecordReaders(records []RecordReader) (int, error) {
	i := len(records) - 1

	for i >= 0 {
		if err := ResetRecordReader(records[i]); err != nil {
			return i + 1, err
		}
		i--
	}

	return i + 1, nil
}
<|endoftext|>


<commit_before>// +build linux,cgo darwin,cgo !windows
// +build 386 amd64

/*
 * magic.go
 *
 * Copyright 2013 Krzysztof Wilczynski
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package magic

/*
#cgo LDFLAGS: -lmagic
#include "functions.h"
*/
import "C"

import (
	"fmt"
	"os"
	"reflect"
	"runtime"
	"strings"
	"sync"
	"syscall"
	"unsafe"
)

type magic struct {
	sync.Mutex
	flags  int
	path   []string
	cookie C.magic_t
}

func (m *magic) close() {
	if m != nil && m.cookie != nil {
		C.magic_close(m.cookie)
		m.cookie = nil
	}
	runtime.SetFinalizer(m, nil)
}

type Magic struct {
	*magic
}

func New(files ...string) (*Magic, error) {
	mgc, err := open()
	if err != nil {
		return nil, err
	}

	if err := mgc.Load(files...); err != nil {
		return nil, err
	}

	return mgc, nil
}

func (mgc *Magic) Close() {
	mgc.Lock()
	defer mgc.Unlock()
	mgc.magic.close()
}

func (mgc *Magic) String() string {
	return fmt.Sprintf("Magic{flags:%d path:%s cookie:%p}",
		mgc.flags, mgc.path, mgc.cookie)
}

func (mgc *Magic) Path() ([]string, error) {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return []string{}, mgc.error()
	}

	if len(mgc.path) != 0 && os.Getenv("MAGIC") == "" {
		return mgc.path, nil
	}
	rv := C.GoString(C.magic_getpath_wrapper())
	mgc.path = strings.Split(rv, ":")
	return mgc.path, nil
}

func (mgc *Magic) Flags() (int, error) {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return -1, mgc.error()
	}
	return mgc.flags, nil
}

func (mgc *Magic) SetFlags(flags int) error {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return mgc.error()
	}

	rv, err := C.magic_setflags_wrapper(mgc.cookie, C.int(flags))
	if rv != 0 && err != nil {
		errno := err.(syscall.Errno)
		return &MagicError{int(errno), errno.Error()}
	}

	mgc.flags = flags
	return nil
}

func (mgc *Magic) Load(files ...string) error {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return mgc.error()
	}

	var cfiles *C.char
	defer C.free(unsafe.Pointer(cfiles))

	if len(files) != 0 {
		cfiles = C.CString(strings.Join(files, ":"))
	} else {
		cfiles = C.magic_getpath_wrapper()
	}

	if rv := C.magic_load_wrapper(mgc.cookie, cfiles); rv != 0 {
		return mgc.error()
	}
	mgc.path = strings.Split(C.GoString(cfiles), ":")
	return nil
}

func (mgc *Magic) Compile(files ...string) (bool, error) {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return false, mgc.error()
	}

	var cfiles *C.char
	if len(files) != 0 {
		cfiles = C.CString(strings.Join(files, ":"))
		defer C.free(unsafe.Pointer(cfiles))
	}

	if rv := C.magic_compile_wrapper(mgc.cookie, cfiles); rv != 0 {
		return false, mgc.error()
	}
	return true, nil
}

func (mgc *Magic) Check(files ...string) (bool, error) {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return false, mgc.error()
	}

	var cfiles *C.char
	if len(files) != 0 {
		cfiles = C.CString(strings.Join(files, ":"))
		defer C.free(unsafe.Pointer(cfiles))
	}

	if rv := C.magic_check_wrapper(mgc.cookie, cfiles); rv != 0 {
		return false, mgc.error()
	}
	return true, nil
}

func (mgc *Magic) File(filename string) (string, error) {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return "", mgc.error()
	}

	cfilename := C.CString(filename)
	defer C.free(unsafe.Pointer(cfilename))

	cstring := C.magic_file(mgc.cookie, cfilename)
	if cstring == nil {
		return "", mgc.error()
	}
	return C.GoString(cstring), nil
}

func (mgc *Magic) Buffer(buffer []byte) (string, error) {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return "", mgc.error()
	}

	p, length := unsafe.Pointer(&buffer[0]), C.size_t(len(buffer))

	cstring := C.magic_buffer(mgc.cookie, p, length)
	if cstring == nil {
		return "", mgc.error()
	}
	return C.GoString(cstring), nil
}

func (mgc *Magic) Descriptor(fd uintptr) (string, error) {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return "", mgc.error()
	}

	cstring := C.magic_descriptor(mgc.cookie, C.int(fd))
	if cstring == nil {
		return "", mgc.error()
	}
	return C.GoString(cstring), nil
}

func (mgc *Magic) error() *MagicError {
	if mgc.cookie == nil {
		errno := syscall.EINVAL
		return &MagicError{int(syscall.EINVAL), errno.Error()}
	}

	errno := int(C.magic_errno(mgc.cookie))
	cstring := C.magic_error(mgc.cookie)
	if cstring != nil {
		return &MagicError{errno, C.GoString(cstring)}
	}
	return nil
}

func (mgc *Magic) destroy() {
	mgc.Close()
}

func open() (*Magic, error) {
	rv := C.magic_open(C.int(NONE))
	if rv == nil {
		errno := syscall.ENOMEM
		return nil, &MagicError{int(errno), errno.Error()}
	}

	mgc := &Magic{&magic{flags: NONE, cookie: rv}}
	runtime.SetFinalizer(mgc.magic, (*magic).close)
	return mgc, nil
}

func Open(f func(magic *Magic) error, files ...string) (err error) {
	var ok bool
	errno := syscall.EINVAL

	if f == nil || reflect.TypeOf(f).Kind() != reflect.Func {
		return &MagicError{int(errno), errno.Error()}
	}

	mgc, err := New(files...)
	if err != nil {
		return err
	}
	defer mgc.Close()

	defer func() {
		if r := recover(); r != nil {
			err, ok = r.(error)
			if !ok {
				err = &MagicError{int(errno), fmt.Sprintf("%v", r)}
			}
		}
	}()

	return f(mgc)
}

func Compile(files ...string) (bool, error) {
	mgc, err := open()
	if err != nil {
		return false, err
	}
	defer mgc.Close()

	rv, err := mgc.Compile(files...)
	if err != nil {
		return rv, nil
	}
	return rv, nil
}

func Check(files ...string) (bool, error) {
	mgc, err := open()
	if err != nil {
		return false, err
	}
	defer mgc.Close()

	rv, err := mgc.Check(files...)
	if err != nil {
		return rv, err
	}
	return rv, nil
}

func Version() (int, error) {
	rv, err := C.magic_version_wrapper()
	if rv != 0 && err != nil {
		errno := err.(syscall.Errno)
		return -1, &MagicError{int(errno), errno.Error()}
	}
	return int(rv), nil
}

func FileMime(filename string, files ...string) (string, error) {
	mgc, err := New(files...)
	if err != nil {
		return "", err
	}
	defer mgc.Close()

	mgc.SetFlags(MIME)
	return mgc.File(filename)
}

func FileEncoding(filename string, files ...string) (string, error) {
	mgc, err := New(files...)
	if err != nil {
		return "", err
	}
	defer mgc.Close()

	mgc.SetFlags(MIME_ENCODING)
	return mgc.File(filename)
}

func FileType(filename string, files ...string) (string, error) {
	mgc, err := New(files...)
	if err != nil {
		return "", err
	}
	defer mgc.Close()

	mgc.SetFlags(MIME_TYPE)
	return mgc.File(filename)
}

func BufferMime(buffer []byte, files ...string) (string, error) {
	mgc, err := New(files...)
	if err != nil {
		return "", err
	}
	defer mgc.Close()

	mgc.SetFlags(MIME)
	return mgc.Buffer(buffer)
}

func BufferEncoding(buffer []byte, files ...string) (string, error) {
	mgc, err := New(files...)
	if err != nil {
		return "", err
	}
	defer mgc.Close()

	mgc.SetFlags(MIME_ENCODING)
	return mgc.Buffer(buffer)
}

func BufferType(buffer []byte, files ...string) (string, error) {
	mgc, err := New(files...)
	if err != nil {
		return "", err
	}
	defer mgc.Close()

	mgc.SetFlags(MIME_TYPE)
	return mgc.Buffer(buffer)
}
<commit_msg>Make Magic.Load() return boolean value depending on failure/success.<commit_after>// +build linux,cgo darwin,cgo !windows
// +build 386 amd64

/*
 * magic.go
 *
 * Copyright 2013 Krzysztof Wilczynski
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package magic

/*
#cgo LDFLAGS: -lmagic
#include "functions.h"
*/
import "C"

import (
	"fmt"
	"os"
	"reflect"
	"runtime"
	"strings"
	"sync"
	"syscall"
	"unsafe"
)

type magic struct {
	sync.Mutex
	flags  int
	path   []string
	cookie C.magic_t
}

func (m *magic) close() {
	if m != nil && m.cookie != nil {
		C.magic_close(m.cookie)
		m.cookie = nil
	}
	runtime.SetFinalizer(m, nil)
}

type Magic struct {
	*magic
}

func New(files ...string) (*Magic, error) {
	mgc, err := open()
	if err != nil {
		return nil, err
	}

	if _, err := mgc.Load(files...); err != nil {
		return nil, err
	}

	return mgc, nil
}

func (mgc *Magic) Close() {
	mgc.Lock()
	defer mgc.Unlock()
	mgc.magic.close()
}

func (mgc *Magic) String() string {
	return fmt.Sprintf("Magic{flags:%d path:%s cookie:%p}",
		mgc.flags, mgc.path, mgc.cookie)
}

func (mgc *Magic) Path() ([]string, error) {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return []string{}, mgc.error()
	}

	if len(mgc.path) != 0 && os.Getenv("MAGIC") == "" {
		return mgc.path, nil
	}
	rv := C.GoString(C.magic_getpath_wrapper())
	mgc.path = strings.Split(rv, ":")
	return mgc.path, nil
}

func (mgc *Magic) Flags() (int, error) {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return -1, mgc.error()
	}
	return mgc.flags, nil
}

func (mgc *Magic) SetFlags(flags int) error {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return mgc.error()
	}

	rv, err := C.magic_setflags_wrapper(mgc.cookie, C.int(flags))
	if rv != 0 && err != nil {
		errno := err.(syscall.Errno)
		return &MagicError{int(errno), errno.Error()}
	}

	mgc.flags = flags
	return nil
}

func (mgc *Magic) Load(files ...string) (bool, error) {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return false, mgc.error()
	}

	var cfiles *C.char
	defer C.free(unsafe.Pointer(cfiles))

	if len(files) != 0 {
		cfiles = C.CString(strings.Join(files, ":"))
	} else {
		cfiles = C.magic_getpath_wrapper()
	}

	if rv := C.magic_load_wrapper(mgc.cookie, cfiles); rv != 0 {
		return false, mgc.error()
	}
	mgc.path = strings.Split(C.GoString(cfiles), ":")
	return true, nil
}

func (mgc *Magic) Compile(files ...string) (bool, error) {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return false, mgc.error()
	}

	var cfiles *C.char
	if len(files) != 0 {
		cfiles = C.CString(strings.Join(files, ":"))
		defer C.free(unsafe.Pointer(cfiles))
	}

	if rv := C.magic_compile_wrapper(mgc.cookie, cfiles); rv != 0 {
		return false, mgc.error()
	}
	return true, nil
}

func (mgc *Magic) Check(files ...string) (bool, error) {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return false, mgc.error()
	}

	var cfiles *C.char
	if len(files) != 0 {
		cfiles = C.CString(strings.Join(files, ":"))
		defer C.free(unsafe.Pointer(cfiles))
	}

	if rv := C.magic_check_wrapper(mgc.cookie, cfiles); rv != 0 {
		return false, mgc.error()
	}
	return true, nil
}

func (mgc *Magic) File(filename string) (string, error) {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return "", mgc.error()
	}

	cfilename := C.CString(filename)
	defer C.free(unsafe.Pointer(cfilename))

	cstring := C.magic_file(mgc.cookie, cfilename)
	if cstring == nil {
		return "", mgc.error()
	}
	return C.GoString(cstring), nil
}

func (mgc *Magic) Buffer(buffer []byte) (string, error) {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return "", mgc.error()
	}

	p, length := unsafe.Pointer(&buffer[0]), C.size_t(len(buffer))

	cstring := C.magic_buffer(mgc.cookie, p, length)
	if cstring == nil {
		return "", mgc.error()
	}
	return C.GoString(cstring), nil
}

func (mgc *Magic) Descriptor(fd uintptr) (string, error) {
	mgc.Lock()
	defer mgc.Unlock()

	if mgc.cookie == nil {
		return "", mgc.error()
	}

	cstring := C.magic_descriptor(mgc.cookie, C.int(fd))
	if cstring == nil {
		return "", mgc.error()
	}
	return C.GoString(cstring), nil
}

func (mgc *Magic) error() *MagicError {
	if mgc.cookie == nil {
		errno := syscall.EINVAL
		return &MagicError{int(syscall.EINVAL), errno.Error()}
	}

	errno := int(C.magic_errno(mgc.cookie))
	cstring := C.magic_error(mgc.cookie)
	if cstring != nil {
		return &MagicError{errno, C.GoString(cstring)}
	}
	return nil
}

func (mgc *Magic) destroy() {
	mgc.Close()
}

func open() (*Magic, error) {
	rv := C.magic_open(C.int(NONE))
	if rv == nil {
		errno := syscall.ENOMEM
		return nil, &MagicError{int(errno), errno.Error()}
	}

	mgc := &Magic{&magic{flags: NONE, cookie: rv}}
	runtime.SetFinalizer(mgc.magic, (*magic).close)
	return mgc, nil
}

func Open(f func(magic *Magic) error, files ...string) (err error) {
	var ok bool
	errno := syscall.EINVAL

	if f == nil || reflect.TypeOf(f).Kind() != reflect.Func {
		return &MagicError{int(errno), errno.Error()}
	}

	mgc, err := New(files...)
	if err != nil {
		return err
	}
	defer mgc.Close()

	defer func() {
		if r := recover(); r != nil {
			err, ok = r.(error)
			if !ok {
				err = &MagicError{int(errno), fmt.Sprintf("%v", r)}
			}
		}
	}()

	return f(mgc)
}

func Compile(files ...string) (bool, error) {
	mgc, err := open()
	if err != nil {
		return false, err
	}
	defer mgc.Close()

	rv, err := mgc.Compile(files...)
	if err != nil {
		return rv, nil
	}
	return rv, nil
}

func Check(files ...string) (bool, error) {
	mgc, err := open()
	if err != nil {
		return false, err
	}
	defer mgc.Close()

	rv, err := mgc.Check(files...)
	if err != nil {
		return rv, err
	}
	return rv, nil
}

func Version() (int, error) {
	rv, err := C.magic_version_wrapper()
	if rv != 0 && err != nil {
		errno := err.(syscall.Errno)
		return -1, &MagicError{int(errno), errno.Error()}
	}
	return int(rv), nil
}

func FileMime(filename string, files ...string) (string, error) {
	mgc, err := New(files...)
	if err != nil {
		return "", err
	}
	defer mgc.Close()

	mgc.SetFlags(MIME)
	return mgc.File(filename)
}

func FileEncoding(filename string, files ...string) (string, error) {
	mgc, err := New(files...)
	if err != nil {
		return "", err
	}
	defer mgc.Close()

	mgc.SetFlags(MIME_ENCODING)
	return mgc.File(filename)
}

func FileType(filename string, files ...string) (string, error) {
	mgc, err := New(files...)
	if err != nil {
		return "", err
	}
	defer mgc.Close()

	mgc.SetFlags(MIME_TYPE)
	return mgc.File(filename)
}

func BufferMime(buffer []byte, files ...string) (string, error) {
	mgc, err := New(files...)
	if err != nil {
		return "", err
	}
	defer mgc.Close()

	mgc.SetFlags(MIME)
	return mgc.Buffer(buffer)
}

func BufferEncoding(buffer []byte, files ...string) (string, error) {
	mgc, err := New(files...)
	if err != nil {
		return "", err
	}
	defer mgc.Close()

	mgc.SetFlags(MIME_ENCODING)
	return mgc.Buffer(buffer)
}

func BufferType(buffer []byte, files ...string) (string, error) {
	mgc, err := New(files...)
	if err != nil {
		return "", err
	}
	defer mgc.Close()

	mgc.SetFlags(MIME_TYPE)
	return mgc.Buffer(buffer)
}
<|endoftext|>


<commit_before>package shadowsocks

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"io"

	"github.com/v2ray/v2ray-core/common/alloc"
	"github.com/v2ray/v2ray-core/common/log"
	"github.com/v2ray/v2ray-core/common/serial"
	"github.com/v2ray/v2ray-core/transport"
)

const (
	AuthSize = 10
)

type KeyGenerator func() []byte

type Authenticator struct {
	key KeyGenerator
}

func NewAuthenticator(keygen KeyGenerator) *Authenticator {
	return &Authenticator{
		key: keygen,
	}
}

func (this *Authenticator) Authenticate(auth []byte, data []byte) []byte {
	hasher := hmac.New(sha1.New, this.key())
	hasher.Write(data)
	res := hasher.Sum(nil)
	return append(auth, res[:AuthSize]...)
}

func HeaderKeyGenerator(key []byte, iv []byte) func() []byte {
	return func() []byte {
		newKey := make([]byte, 0, len(key)+len(iv))
		newKey = append(newKey, iv...)
		newKey = append(newKey, key...)
		return newKey
	}
}

func ChunkKeyGenerator(iv []byte) func() []byte {
	chunkId := 0
	return func() []byte {
		newKey := make([]byte, 0, len(iv)+4)
		newKey = append(newKey, iv...)
		serial.IntToBytes(chunkId, newKey)
		chunkId++
		return newKey
	}
}

type ChunkReader struct {
	reader io.Reader
	auth   *Authenticator
}

func NewChunkReader(reader io.Reader, auth *Authenticator) *ChunkReader {
	return &ChunkReader{
		reader: reader,
		auth:   auth,
	}
}

func (this *ChunkReader) Release() {
	this.reader = nil
	this.auth = nil
}

func (this *ChunkReader) Read() (*alloc.Buffer, error) {
	buffer := alloc.NewLargeBuffer()
	if _, err := io.ReadFull(this.reader, buffer.Value[:2]); err != nil {
		buffer.Release()
		return nil, err
	}
	// There is a potential buffer overflow here. Large buffer is 64K bytes,
	// while uin16 + 10 will be more than that
	length := serial.BytesToUint16(buffer.Value[:2]) + AuthSize
	if _, err := io.ReadFull(this.reader, buffer.Value[:length]); err != nil {
		buffer.Release()
		return nil, err
	}
	buffer.Slice(0, int(length))

	authBytes := buffer.Value[:AuthSize]
	payload := buffer.Value[AuthSize:]

	actualAuthBytes := this.auth.Authenticate(nil, payload)
	if !bytes.Equal(authBytes, actualAuthBytes) {
		buffer.Release()
		log.Debug("AuthenticationReader: Unexpected auth: ", authBytes)
		return nil, transport.ErrCorruptedPacket
	}
	buffer.Value = payload

	return buffer, nil
}
<commit_msg>Fix Shadowsocks OTA<commit_after>package shadowsocks

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"io"

	"github.com/v2ray/v2ray-core/common/alloc"
	"github.com/v2ray/v2ray-core/common/log"
	"github.com/v2ray/v2ray-core/common/serial"
	"github.com/v2ray/v2ray-core/transport"
)

const (
	AuthSize = 10
)

type KeyGenerator func() []byte

type Authenticator struct {
	key KeyGenerator
}

func NewAuthenticator(keygen KeyGenerator) *Authenticator {
	return &Authenticator{
		key: keygen,
	}
}

func (this *Authenticator) Authenticate(auth []byte, data []byte) []byte {
	hasher := hmac.New(sha1.New, this.key())
	hasher.Write(data)
	res := hasher.Sum(nil)
	return append(auth, res[:AuthSize]...)
}

func HeaderKeyGenerator(key []byte, iv []byte) func() []byte {
	return func() []byte {
		newKey := make([]byte, 0, len(key)+len(iv))
		newKey = append(newKey, iv...)
		newKey = append(newKey, key...)
		return newKey
	}
}

func ChunkKeyGenerator(iv []byte) func() []byte {
	chunkId := 0
	return func() []byte {
		newKey := make([]byte, 0, len(iv)+4)
		newKey = append(newKey, iv...)
		newKey = serial.IntToBytes(chunkId, newKey)
		chunkId++
		return newKey
	}
}

type ChunkReader struct {
	reader io.Reader
	auth   *Authenticator
}

func NewChunkReader(reader io.Reader, auth *Authenticator) *ChunkReader {
	return &ChunkReader{
		reader: reader,
		auth:   auth,
	}
}

func (this *ChunkReader) Release() {
	this.reader = nil
	this.auth = nil
}

func (this *ChunkReader) Read() (*alloc.Buffer, error) {
	buffer := alloc.NewLargeBuffer()
	if _, err := io.ReadFull(this.reader, buffer.Value[:2]); err != nil {
		buffer.Release()
		return nil, err
	}
	// There is a potential buffer overflow here. Large buffer is 64K bytes,
	// while uin16 + 10 will be more than that
	length := serial.BytesToUint16(buffer.Value[:2]) + AuthSize
	if _, err := io.ReadFull(this.reader, buffer.Value[:length]); err != nil {
		buffer.Release()
		return nil, err
	}
	buffer.Slice(0, int(length))

	authBytes := buffer.Value[:AuthSize]
	payload := buffer.Value[AuthSize:]

	actualAuthBytes := this.auth.Authenticate(nil, payload)
	if !bytes.Equal(authBytes, actualAuthBytes) {
		buffer.Release()
		log.Debug("AuthenticationReader: Unexpected auth: ", authBytes)
		return nil, transport.ErrCorruptedPacket
	}
	buffer.Value = payload

	return buffer, nil
}
<|endoftext|>


<commit_before>package som

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestExpRadius(t *testing.T) {
	assert := assert.New(t)

	radius0 := 100.0
	totalIterations := 100
	strategy := "exp"

	assert.Equal(radius0, Radius(0, totalIterations, strategy, radius0))
	assert.InDelta(1.0, Radius(totalIterations, totalIterations, strategy, radius0), 0.01)
}

func TestLinRadius(t *testing.T) {
	assert := assert.New(t)

	radius0 := 100.0
	totalIterations := 100
	strategy := "lin"

	assert.Equal(radius0, Radius(0, totalIterations, strategy, radius0))
	assert.InDelta(1.0, Radius(totalIterations, totalIterations, strategy, radius0), 0.01)
}
<commit_msg>improved test coverage<commit_after>package som

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestExpRadius(t *testing.T) {
	testRadius(t, "exp")
}

func TestLinRadius(t *testing.T) {
	testRadius(t, "lin")
}

func TestDefaultRadius(t *testing.T) {
	testRadius(t, "some other")
}

func testRadius(t *testing.T, strategy string) {
	assert := assert.New(t)

	radius0 := 100.0
	totalIterations := 100

	assert.Equal(radius0, Radius(0, totalIterations, strategy, radius0))
	assert.InDelta(1.0, Radius(totalIterations, totalIterations, strategy, radius0), 0.01)

}
<|endoftext|>


<commit_before>package apps

import (
	"compress/gzip"
	"io"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/cozy/afero"
	"github.com/cozy/cozy-stack/pkg/magic"
	"github.com/cozy/cozy-stack/pkg/utils"
	"github.com/cozy/swift"
)

// Copier is an interface defining a common set of functions for the installer
// to copy the application into an unknown storage.
type Copier interface {
	Start(slug, version string) (exists bool, err error)
	Copy(stat os.FileInfo, src io.Reader) error
	Abort() error
	Commit() error
}

type swiftCopier struct {
	c         *swift.Connection
	appObj    string
	tmpObj    string
	container string
	started   bool
}

type aferoCopier struct {
	fs      afero.Fs
	appDir  string
	tmpDir  string
	started bool
}

// NewSwiftCopier defines a Copier storing data into a swift container.
func NewSwiftCopier(conn *swift.Connection, appsType AppType) Copier {
	return &swiftCopier{
		c:         conn,
		container: containerName(appsType),
	}
}

func (f *swiftCopier) Start(slug, version string) (bool, error) {
	f.appObj = path.Join(slug, version)
	_, _, err := f.c.Object(f.container, f.appObj)
	if err == nil {
		return true, nil
	}
	if err != swift.ObjectNotFound {
		return false, err
	}
	if _, _, err = f.c.Container(f.container); err == swift.ContainerNotFound {
		if err = f.c.ContainerCreate(f.container, nil); err != nil {
			return false, err
		}
	}
	f.tmpObj = "tmp-" + utils.RandomString(20) + "/"
	f.started = true
	return false, err
}

func (f *swiftCopier) Copy(stat os.FileInfo, src io.Reader) (err error) {
	if !f.started {
		panic("copier should call Start() before Copy()")
	}

	objName := path.Join(f.tmpObj, stat.Name())
	objMeta := swift.Metadata{
		"content-encoding":        "gzip",
		"original-content-length": strconv.FormatInt(stat.Size(), 10),
	}

	contentType := magic.MIMETypeByExtension(path.Ext(stat.Name()))
	if contentType == "" {
		contentType, src = magic.MIMETypeFromReader(src)
	}
	if contentType == "" {
		contentType = "application/octet-stream"
	}

	file, err := f.c.ObjectCreate(f.container, objName, true, "",
		contentType, objMeta.ObjectHeaders())
	if err != nil {
		return err
	}
	defer func() {
		if errc := file.Close(); errc != nil {
			err = errc
		}
	}()

	gw, err := gzip.NewWriterLevel(file, gzip.BestCompression)
	if err != nil {
		return err
	}
	defer func() {
		if errc := gw.Close(); errc != nil && err == nil {
			err = errc
		}
	}()

	_, err = io.Copy(gw, src)
	return err
}

func (f *swiftCopier) Abort() error {
	objectNames, err := f.c.ObjectNamesAll(f.container, &swift.ObjectsOpts{
		Prefix: f.tmpObj,
	})
	if err != nil {
		return err
	}
	_, err = f.c.BulkDelete(f.container, objectNames)
	return err
}

func (f *swiftCopier) Commit() (err error) {
	objectNames, err := f.c.ObjectNamesAll(f.container, &swift.ObjectsOpts{
		Prefix: f.tmpObj,
	})
	if err != nil {
		return err
	}
	// We check if the appObj has not been created concurrently by another
	// copier. If it exists
	_, _, err = f.c.Object(f.container, f.appObj)
	if err == nil {
		_, err = f.c.BulkDelete(f.container, objectNames)
		return err
	}
	if err = f.c.ObjectPutString(f.container, f.appObj, "", "text/plain"); err != nil {
		return err
	}
	defer func() {
		if err != nil {
			f.c.BulkDelete(f.container, append([]string{f.appObj}, objectNames...))
		}
	}()
	for _, srcObjectName := range objectNames {
		dstObjectName := path.Join(f.appObj, strings.TrimPrefix(srcObjectName, f.tmpObj))
		err = f.c.ObjectMove(f.container, srcObjectName, f.container, dstObjectName)
		if err != nil {
			return err
		}
	}
	return nil
}

// NewAferoCopier defines a copier using an afero.Fs filesystem to store the
// application data.
func NewAferoCopier(fs afero.Fs) Copier {
	return &aferoCopier{fs: fs}
}

func (f *aferoCopier) Start(slug, version string) (bool, error) {
	f.appDir = path.Join("/", slug, version)
	exists, err := afero.DirExists(f.fs, f.appDir)
	if err != nil || exists {
		return exists, err
	}
	dir := path.Dir(f.appDir)
	if err = f.fs.MkdirAll(dir, 0755); err != nil {
		return false, err
	}
	f.tmpDir, err = afero.TempDir(f.fs, dir, "tmp")
	if err != nil {
		return false, err
	}
	f.started = true
	return false, nil
}

func (f *aferoCopier) Copy(stat os.FileInfo, src io.Reader) (err error) {
	if !f.started {
		panic("copier should call Start() before Copy()")
	}

	fullpath := path.Join(f.tmpDir, stat.Name()) + ".gz"
	dir := path.Dir(fullpath)
	if err = f.fs.MkdirAll(dir, 0755); err != nil {
		return err
	}

	dst, err := f.fs.Create(fullpath)
	if err != nil {
		return err
	}
	defer func() {
		if errc := dst.Close(); errc != nil {
			err = errc
		}
	}()

	gw, err := gzip.NewWriterLevel(dst, gzip.BestCompression)
	if err != nil {
		return err
	}
	defer func() {
		if errc := gw.Close(); errc != nil && err == nil {
			err = errc
		}
	}()

	_, err = io.Copy(gw, src)
	return err
}

func (f *aferoCopier) Commit() error {
	return f.fs.Rename(f.tmpDir, f.appDir)
}

func (f *aferoCopier) Abort() error {
	return f.fs.RemoveAll(f.tmpDir)
}

type fileInfo struct {
	name string
	size int64
	mode os.FileMode
	time time.Time
}

func (f *fileInfo) Name() string       { return f.name }
func (f *fileInfo) Size() int64        { return f.size }
func (f *fileInfo) Mode() os.FileMode  { return f.mode }
func (f *fileInfo) ModTime() time.Time { return f.time }
func (f *fileInfo) IsDir() bool        { return false }
func (f *fileInfo) Sys() interface{}   { return nil }
<commit_msg>Fix concurrent application update<commit_after>package apps

import (
	"compress/gzip"
	"io"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/cozy/afero"
	"github.com/cozy/cozy-stack/pkg/magic"
	"github.com/cozy/cozy-stack/pkg/utils"
	"github.com/cozy/swift"
)

// Copier is an interface defining a common set of functions for the installer
// to copy the application into an unknown storage.
type Copier interface {
	Start(slug, version string) (exists bool, err error)
	Copy(stat os.FileInfo, src io.Reader) error
	Abort() error
	Commit() error
}

type swiftCopier struct {
	c         *swift.Connection
	appObj    string
	tmpObj    string
	container string
	started   bool
}

type aferoCopier struct {
	fs      afero.Fs
	appDir  string
	tmpDir  string
	started bool
}

// NewSwiftCopier defines a Copier storing data into a swift container.
func NewSwiftCopier(conn *swift.Connection, appsType AppType) Copier {
	return &swiftCopier{
		c:         conn,
		container: containerName(appsType),
	}
}

func (f *swiftCopier) Start(slug, version string) (bool, error) {
	f.appObj = path.Join(slug, version)
	_, _, err := f.c.Object(f.container, f.appObj)
	if err == nil {
		return true, nil
	}
	if err != swift.ObjectNotFound {
		return false, err
	}
	if _, _, err = f.c.Container(f.container); err == swift.ContainerNotFound {
		if err = f.c.ContainerCreate(f.container, nil); err != nil {
			return false, err
		}
	}
	f.tmpObj = "tmp-" + utils.RandomString(20) + "/"
	f.started = true
	return false, err
}

func (f *swiftCopier) Copy(stat os.FileInfo, src io.Reader) (err error) {
	if !f.started {
		panic("copier should call Start() before Copy()")
	}

	objName := path.Join(f.tmpObj, stat.Name())
	objMeta := swift.Metadata{
		"content-encoding":        "gzip",
		"original-content-length": strconv.FormatInt(stat.Size(), 10),
	}

	contentType := magic.MIMETypeByExtension(path.Ext(stat.Name()))
	if contentType == "" {
		contentType, src = magic.MIMETypeFromReader(src)
	}
	if contentType == "" {
		contentType = "application/octet-stream"
	}

	file, err := f.c.ObjectCreate(f.container, objName, true, "",
		contentType, objMeta.ObjectHeaders())
	if err != nil {
		return err
	}
	defer func() {
		if errc := file.Close(); errc != nil {
			err = errc
		}
	}()

	gw, err := gzip.NewWriterLevel(file, gzip.BestCompression)
	if err != nil {
		return err
	}
	defer func() {
		if errc := gw.Close(); errc != nil && err == nil {
			err = errc
		}
	}()

	_, err = io.Copy(gw, src)
	return err
}

func (f *swiftCopier) Abort() error {
	objectNames, err := f.c.ObjectNamesAll(f.container, &swift.ObjectsOpts{
		Prefix: f.tmpObj,
	})
	if err != nil {
		return err
	}
	_, err = f.c.BulkDelete(f.container, objectNames)
	return err
}

func (f *swiftCopier) Commit() (err error) {
	objectNames, err := f.c.ObjectNamesAll(f.container, &swift.ObjectsOpts{
		Prefix: f.tmpObj,
	})
	if err != nil {
		return err
	}
	defer f.c.BulkDelete(f.container, objectNames)
	// We check if the appObj has not been created concurrently by another
	// copier.
	_, _, err = f.c.Object(f.container, f.appObj)
	if err == nil {
		return nil
	}
	for _, srcObjectName := range objectNames {
		dstObjectName := path.Join(f.appObj, strings.TrimPrefix(srcObjectName, f.tmpObj))
		_, err = f.c.ObjectCopy(f.container, srcObjectName, f.container, dstObjectName, nil)
		if err != nil {
			return err
		}
	}
	return f.c.ObjectPutString(f.container, f.appObj, "", "text/plain")
}

// NewAferoCopier defines a copier using an afero.Fs filesystem to store the
// application data.
func NewAferoCopier(fs afero.Fs) Copier {
	return &aferoCopier{fs: fs}
}

func (f *aferoCopier) Start(slug, version string) (bool, error) {
	f.appDir = path.Join("/", slug, version)
	exists, err := afero.DirExists(f.fs, f.appDir)
	if err != nil || exists {
		return exists, err
	}
	dir := path.Dir(f.appDir)
	if err = f.fs.MkdirAll(dir, 0755); err != nil {
		return false, err
	}
	f.tmpDir, err = afero.TempDir(f.fs, dir, "tmp")
	if err != nil {
		return false, err
	}
	f.started = true
	return false, nil
}

func (f *aferoCopier) Copy(stat os.FileInfo, src io.Reader) (err error) {
	if !f.started {
		panic("copier should call Start() before Copy()")
	}

	fullpath := path.Join(f.tmpDir, stat.Name()) + ".gz"
	dir := path.Dir(fullpath)
	if err = f.fs.MkdirAll(dir, 0755); err != nil {
		return err
	}

	dst, err := f.fs.Create(fullpath)
	if err != nil {
		return err
	}
	defer func() {
		if errc := dst.Close(); errc != nil {
			err = errc
		}
	}()

	gw, err := gzip.NewWriterLevel(dst, gzip.BestCompression)
	if err != nil {
		return err
	}
	defer func() {
		if errc := gw.Close(); errc != nil && err == nil {
			err = errc
		}
	}()

	_, err = io.Copy(gw, src)
	return err
}

func (f *aferoCopier) Commit() error {
	return f.fs.Rename(f.tmpDir, f.appDir)
}

func (f *aferoCopier) Abort() error {
	return f.fs.RemoveAll(f.tmpDir)
}

type fileInfo struct {
	name string
	size int64
	mode os.FileMode
	time time.Time
}

func (f *fileInfo) Name() string       { return f.name }
func (f *fileInfo) Size() int64        { return f.size }
func (f *fileInfo) Mode() os.FileMode  { return f.mode }
func (f *fileInfo) ModTime() time.Time { return f.time }
func (f *fileInfo) IsDir() bool        { return false }
func (f *fileInfo) Sys() interface{}   { return nil }
<|endoftext|>


<commit_before>// Copyright (c) 2017 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cache

import (
	"reflect"
	"sync"
	"time"

	"github.com/patrickmn/go-cache"
	log "github.com/sirupsen/logrus"
	"k8s.io/client-go/util/workqueue"
)

// ResourceCache stores resources and queues updates when those resources
// are created, modified, or deleted. It de-duplicates updates by ensuring
// updates are only queued when an object has changed.
type ResourceCache interface {
	// Set sets the key to the provided value, and generates an update
	// on the queue the value has changed.
	Set(key string, value interface{})

	// Get gets the value associated with the given key.  Returns nil
	// if the key is not present.
	Get(key string) (interface{}, bool)

	// Prime sets the key to the provided value, but does not generate
	// and update on the queue ever.
	Prime(key string, value interface{})

	// Delete deletes the value identified by the given key from the cache, and
	// generates an update on the queue if a value was deleted.
	Delete(key string)

	// Clean removes the object identified by the given key from the cache.
	// It does not generate an update on the queue.
	Clean(key string)

	// ListKeys lists the keys currently in the cache.
	ListKeys() []string

	// Run enables the generation of events on the output queue starts
	// cache reconciliation.
	Run(reconcilerPeriod string)

	// GetQueue returns the cache's output queue, which emits a stream
	// of any keys which have been created, modified, or deleted.
	GetQueue() workqueue.RateLimitingInterface
}

// ResourceCacheArgs struct passed to constructor of ResourceCache.
// Groups togather all the arguments to pass in single struct.
type ResourceCacheArgs struct {
	// ListFunc returns a mapping of keys to objects from the Calico datastore.
	ListFunc func() (map[string]interface{}, error)

	// ObjectType is the type of object which is to be stored in this cache.
	ObjectType reflect.Type

	ReconcilerConfig ReconcilerConfig
}

// ReconcilerConfig contains configuration for the periodic reconciler.
type ReconcilerConfig struct {
	// DisableUpdateOnChange disables the queuing of updates when the reconciler
	// detects that a value has changed in the datastore.
	DisableUpdateOnChange bool

	// DisableMissingInDatastore disables queueing of updates when the reconciler
	// detects that a value is no longer in the datastore but still exists in the cache.
	DisableMissingInDatastore bool

	// DisableMissingInCache disables queueing of updates when reconciler detects
	// that a value that is still in the datastore no longer is in the cache.
	DisableMissingInCache bool
}

// calicoCache implements the ResourceCache interface
type calicoCache struct {
	threadSafeCache  *cache.Cache
	workqueue        workqueue.RateLimitingInterface
	ListFunc         func() (map[string]interface{}, error)
	ObjectType       reflect.Type
	log              *log.Entry
	running          bool
	mut              *sync.Mutex
	reconcilerConfig ReconcilerConfig
}

// NewResourceCache builds and returns a resource cache using the provided arguments.
func NewResourceCache(args ResourceCacheArgs) ResourceCache {
	// Make sure logging is context aware.
	return &calicoCache{
		threadSafeCache:  cache.New(cache.NoExpiration, cache.DefaultExpiration),
		workqueue:        workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
		ListFunc:         args.ListFunc,
		ObjectType:       args.ObjectType,
		log:              log.WithFields(log.Fields{"type": args.ObjectType}),
		mut:              &sync.Mutex{},
		reconcilerConfig: args.ReconcilerConfig,
	}
}

func (c *calicoCache) Set(key string, newObj interface{}) {
	if reflect.TypeOf(newObj) != c.ObjectType {
		c.log.Fatalf("Wrong object type recieved to store in cache. Expected: %s, Found: %s", c.ObjectType, reflect.TypeOf(newObj))
	}

	// Check if the object exists in the cache already.  If it does and hasn't changed,
	// then we don't need to send an update on the queue.
	if existingObj, found := c.threadSafeCache.Get(key); found {
		c.log.Debugf("%#v already exists in cache - comparing.", existingObj)
		if !reflect.DeepEqual(existingObj, newObj) {
			// The objects do not match - send an update over the queue.
			c.threadSafeCache.Set(key, newObj, cache.NoExpiration)
			if c.isRunning() {
				c.log.Debugf("Queueing update - %#v and %#v do not match.", newObj, existingObj)
				c.workqueue.Add(key)
			}
		}
	} else {
		c.threadSafeCache.Set(key, newObj, cache.NoExpiration)
		if c.isRunning() {
			c.log.Debugf("%#v not found in cache, adding it + queuing update.", newObj)
			c.workqueue.Add(key)
		}
	}
}

func (c *calicoCache) Delete(key string) {
	c.log.Debugf("Deleting %s from cache", key)
	c.threadSafeCache.Delete(key)
	c.workqueue.Add(key)
}

func (c *calicoCache) Clean(key string) {
	c.log.Debugf("Cleaning %s from cache, no update required", key)
	c.threadSafeCache.Delete(key)
}

func (c *calicoCache) Get(key string) (interface{}, bool) {
	obj, found := c.threadSafeCache.Get(key)
	if found {
		return obj, true
	}
	return nil, false
}

// Prime adds the key and value to the cache but will never generate
// an update on the queue.
func (c *calicoCache) Prime(key string, value interface{}) {
	c.threadSafeCache.Set(key, value, cache.NoExpiration)
}

// ListKeys returns a list of all the keys in the cache.
func (c *calicoCache) ListKeys() []string {
	cacheItems := c.threadSafeCache.Items()
	keys := make([]string, 0, len(cacheItems))
	for k := range cacheItems {
		keys = append(keys, k)
	}

	return keys
}

// GetQueue returns the output queue from the cache.  Whenever a key/value pair
// is modified, an event will appear on this queue.
func (c *calicoCache) GetQueue() workqueue.RateLimitingInterface {
	return c.workqueue
}

// Run starts the cache.  Any Set() calls prior to calling Run() will
// prime the cache, but not trigger any updates on the output queue.
func (c *calicoCache) Run(reconcilerPeriod string) {
	go c.reconcile(reconcilerPeriod)

	// Indicate that the cache is running, and so updates
	// can be queued.
	c.mut.Lock()
	c.running = true
	c.mut.Unlock()
}

func (c *calicoCache) isRunning() bool {
	c.mut.Lock()
	defer c.mut.Unlock()
	return c.running
}

// reconcile ensures a reconciliation is run every `reconcilerPeriod` in order to bring the datastore
// in sync with the cache. This is to correct any manual changes made in the datastore
// without the cache being aware.
func (c *calicoCache) reconcile(reconcilerPeriod string) {
	duration, err := time.ParseDuration(reconcilerPeriod)
	if err != nil {
		c.log.Fatalf("Invalid time duration format for reconciler: %s. Some valid examples: 5m, 30s, 2m30s etc.", reconcilerPeriod)
	}

	// If user has set duration to 0 then disable the reconciler job.
	if duration.Nanoseconds() == 0 {
		c.log.Infof("Reconciler period set to %d. Disabling reconciler.", duration.Nanoseconds())
		return
	}

	// Loop forever, performing a datastore reconciliation periodically.
	for {
		c.log.Debugf("Performing reconciliation")
		err := c.performDatastoreSync()
		if err != nil {
			c.log.WithError(err).Error("Reconciliation failed")
			continue
		}

		// Reconciliation was successful, sleep the configured duration.
		c.log.Debugf("Reconciliation complete, %+v until next one.", duration)
		time.Sleep(duration)
	}
}

func (c *calicoCache) performDatastoreSync() error {
	// Get all the objects we care about from the datastore using ListFunc.
	objMap, err := c.ListFunc()
	if err != nil {
		c.log.WithError(err).Errorf("unable to list objects from datastore while reconciling.")
		return err
	}

	// Build a map of existing keys in the datastore.
	allKeys := map[string]bool{}
	for key := range objMap {
		allKeys[key] = true
	}

	// Also add all existing keys in the cache.
	for _, key := range c.ListKeys() {
		allKeys[key] = true
	}

	c.log.Debugf("Reconciling %d keys in total", len(allKeys))
	for key := range allKeys {
		cachedObj, existsInCache := c.Get(key)
		if !existsInCache {
			// Key does not exist in the cache, queue an update to
			// remove it from the datastore if configured to do so.
			if !c.reconcilerConfig.DisableMissingInCache {
				c.log.WithField("key", key).Warn("Value for key should not exist, queueing update to remove")
				c.workqueue.Add(key)
			}
			continue
		}

		obj, existsInDatastore := objMap[key]
		if !existsInDatastore {
			// Key exists in the cache but not in the datastore - queue an update
			// to re-add it if configured to do so.
			if !c.reconcilerConfig.DisableMissingInDatastore {
				c.log.WithField("key", key).Warn("Value for key is missing in datastore, queueing update to reprogram")
				c.workqueue.Add(key)
			}
			continue
		}

		if !reflect.DeepEqual(obj, cachedObj) {
			// Objects differ - queue an update to re-program if configured to do so.
			if !c.reconcilerConfig.DisableUpdateOnChange {
				c.log.WithField("key", key).Warn("Value for key has changed, queueing update to reprogram")
				c.workqueue.Add(key)
			}
			continue
		}
	}
	return nil
}
<commit_msg>Add logs which show changed object<commit_after>// Copyright (c) 2017 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cache

import (
	"reflect"
	"sync"
	"time"

	"github.com/patrickmn/go-cache"
	log "github.com/sirupsen/logrus"
	"k8s.io/client-go/util/workqueue"
)

// ResourceCache stores resources and queues updates when those resources
// are created, modified, or deleted. It de-duplicates updates by ensuring
// updates are only queued when an object has changed.
type ResourceCache interface {
	// Set sets the key to the provided value, and generates an update
	// on the queue the value has changed.
	Set(key string, value interface{})

	// Get gets the value associated with the given key.  Returns nil
	// if the key is not present.
	Get(key string) (interface{}, bool)

	// Prime sets the key to the provided value, but does not generate
	// and update on the queue ever.
	Prime(key string, value interface{})

	// Delete deletes the value identified by the given key from the cache, and
	// generates an update on the queue if a value was deleted.
	Delete(key string)

	// Clean removes the object identified by the given key from the cache.
	// It does not generate an update on the queue.
	Clean(key string)

	// ListKeys lists the keys currently in the cache.
	ListKeys() []string

	// Run enables the generation of events on the output queue starts
	// cache reconciliation.
	Run(reconcilerPeriod string)

	// GetQueue returns the cache's output queue, which emits a stream
	// of any keys which have been created, modified, or deleted.
	GetQueue() workqueue.RateLimitingInterface
}

// ResourceCacheArgs struct passed to constructor of ResourceCache.
// Groups togather all the arguments to pass in single struct.
type ResourceCacheArgs struct {
	// ListFunc returns a mapping of keys to objects from the Calico datastore.
	ListFunc func() (map[string]interface{}, error)

	// ObjectType is the type of object which is to be stored in this cache.
	ObjectType reflect.Type

	ReconcilerConfig ReconcilerConfig
}

// ReconcilerConfig contains configuration for the periodic reconciler.
type ReconcilerConfig struct {
	// DisableUpdateOnChange disables the queuing of updates when the reconciler
	// detects that a value has changed in the datastore.
	DisableUpdateOnChange bool

	// DisableMissingInDatastore disables queueing of updates when the reconciler
	// detects that a value is no longer in the datastore but still exists in the cache.
	DisableMissingInDatastore bool

	// DisableMissingInCache disables queueing of updates when reconciler detects
	// that a value that is still in the datastore no longer is in the cache.
	DisableMissingInCache bool
}

// calicoCache implements the ResourceCache interface
type calicoCache struct {
	threadSafeCache  *cache.Cache
	workqueue        workqueue.RateLimitingInterface
	ListFunc         func() (map[string]interface{}, error)
	ObjectType       reflect.Type
	log              *log.Entry
	running          bool
	mut              *sync.Mutex
	reconcilerConfig ReconcilerConfig
}

// NewResourceCache builds and returns a resource cache using the provided arguments.
func NewResourceCache(args ResourceCacheArgs) ResourceCache {
	// Make sure logging is context aware.
	return &calicoCache{
		threadSafeCache:  cache.New(cache.NoExpiration, cache.DefaultExpiration),
		workqueue:        workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
		ListFunc:         args.ListFunc,
		ObjectType:       args.ObjectType,
		log:              log.WithFields(log.Fields{"type": args.ObjectType}),
		mut:              &sync.Mutex{},
		reconcilerConfig: args.ReconcilerConfig,
	}
}

func (c *calicoCache) Set(key string, newObj interface{}) {
	if reflect.TypeOf(newObj) != c.ObjectType {
		c.log.Fatalf("Wrong object type recieved to store in cache. Expected: %s, Found: %s", c.ObjectType, reflect.TypeOf(newObj))
	}

	// Check if the object exists in the cache already.  If it does and hasn't changed,
	// then we don't need to send an update on the queue.
	if existingObj, found := c.threadSafeCache.Get(key); found {
		c.log.Debugf("%#v already exists in cache - comparing.", existingObj)
		if !reflect.DeepEqual(existingObj, newObj) {
			// The objects do not match - send an update over the queue.
			c.threadSafeCache.Set(key, newObj, cache.NoExpiration)
			if c.isRunning() {
				c.log.Debugf("Queueing update - %#v and %#v do not match.", newObj, existingObj)
				c.workqueue.Add(key)
			}
		}
	} else {
		c.threadSafeCache.Set(key, newObj, cache.NoExpiration)
		if c.isRunning() {
			c.log.Debugf("%#v not found in cache, adding it + queuing update.", newObj)
			c.workqueue.Add(key)
		}
	}
}

func (c *calicoCache) Delete(key string) {
	c.log.Debugf("Deleting %s from cache", key)
	c.threadSafeCache.Delete(key)
	c.workqueue.Add(key)
}

func (c *calicoCache) Clean(key string) {
	c.log.Debugf("Cleaning %s from cache, no update required", key)
	c.threadSafeCache.Delete(key)
}

func (c *calicoCache) Get(key string) (interface{}, bool) {
	obj, found := c.threadSafeCache.Get(key)
	if found {
		return obj, true
	}
	return nil, false
}

// Prime adds the key and value to the cache but will never generate
// an update on the queue.
func (c *calicoCache) Prime(key string, value interface{}) {
	c.threadSafeCache.Set(key, value, cache.NoExpiration)
}

// ListKeys returns a list of all the keys in the cache.
func (c *calicoCache) ListKeys() []string {
	cacheItems := c.threadSafeCache.Items()
	keys := make([]string, 0, len(cacheItems))
	for k := range cacheItems {
		keys = append(keys, k)
	}

	return keys
}

// GetQueue returns the output queue from the cache.  Whenever a key/value pair
// is modified, an event will appear on this queue.
func (c *calicoCache) GetQueue() workqueue.RateLimitingInterface {
	return c.workqueue
}

// Run starts the cache.  Any Set() calls prior to calling Run() will
// prime the cache, but not trigger any updates on the output queue.
func (c *calicoCache) Run(reconcilerPeriod string) {
	go c.reconcile(reconcilerPeriod)

	// Indicate that the cache is running, and so updates
	// can be queued.
	c.mut.Lock()
	c.running = true
	c.mut.Unlock()
}

func (c *calicoCache) isRunning() bool {
	c.mut.Lock()
	defer c.mut.Unlock()
	return c.running
}

// reconcile ensures a reconciliation is run every `reconcilerPeriod` in order to bring the datastore
// in sync with the cache. This is to correct any manual changes made in the datastore
// without the cache being aware.
func (c *calicoCache) reconcile(reconcilerPeriod string) {
	duration, err := time.ParseDuration(reconcilerPeriod)
	if err != nil {
		c.log.Fatalf("Invalid time duration format for reconciler: %s. Some valid examples: 5m, 30s, 2m30s etc.", reconcilerPeriod)
	}

	// If user has set duration to 0 then disable the reconciler job.
	if duration.Nanoseconds() == 0 {
		c.log.Infof("Reconciler period set to %d. Disabling reconciler.", duration.Nanoseconds())
		return
	}

	// Loop forever, performing a datastore reconciliation periodically.
	for {
		c.log.Debugf("Performing reconciliation")
		err := c.performDatastoreSync()
		if err != nil {
			c.log.WithError(err).Error("Reconciliation failed")
			continue
		}

		// Reconciliation was successful, sleep the configured duration.
		c.log.Debugf("Reconciliation complete, %+v until next one.", duration)
		time.Sleep(duration)
	}
}

func (c *calicoCache) performDatastoreSync() error {
	// Get all the objects we care about from the datastore using ListFunc.
	objMap, err := c.ListFunc()
	if err != nil {
		c.log.WithError(err).Errorf("unable to list objects from datastore while reconciling.")
		return err
	}

	// Build a map of existing keys in the datastore.
	allKeys := map[string]bool{}
	for key := range objMap {
		allKeys[key] = true
	}

	// Also add all existing keys in the cache.
	for _, key := range c.ListKeys() {
		allKeys[key] = true
	}

	c.log.Debugf("Reconciling %d keys in total", len(allKeys))
	for key := range allKeys {
		cachedObj, existsInCache := c.Get(key)
		if !existsInCache {
			// Key does not exist in the cache, queue an update to
			// remove it from the datastore if configured to do so.
			if !c.reconcilerConfig.DisableMissingInCache {
				c.log.WithField("key", key).Warn("Value for key should not exist, queueing update to remove")
				c.workqueue.Add(key)
			}
			continue
		}

		obj, existsInDatastore := objMap[key]
		if !existsInDatastore {
			// Key exists in the cache but not in the datastore - queue an update
			// to re-add it if configured to do so.
			if !c.reconcilerConfig.DisableMissingInDatastore {
				c.log.WithField("key", key).Warn("Value for key is missing in datastore, queueing update to reprogram")
				c.workqueue.Add(key)
			}
			continue
		}

		if !reflect.DeepEqual(obj, cachedObj) {
			// Objects differ - queue an update to re-program if configured to do so.
			if !c.reconcilerConfig.DisableUpdateOnChange {
				c.log.WithField("key", key).Warn("Value for key has changed, queueing update to reprogram")
				c.log.Debugf("Cached:  %+v", cachedObj)
				c.log.Debugf("Updated: %+v", obj)
				c.workqueue.Add(key)
			}
			continue
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package docker

import (
	"fmt"
	"os"

	"golang.org/x/crypto/ssh/terminal"
	"golang.org/x/net/context"

	"github.com/flant/logboek"

	"github.com/docker/cli/cli/command"
	cliconfig "github.com/docker/cli/cli/config"
	"github.com/docker/cli/cli/flags"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/client"
)

var (
	cli       *command.DockerCli
	apiClient *client.Client
)

func Init(dockerConfigDir string) error {
	if dockerConfigDir != "" {
		cliconfig.SetDir(dockerConfigDir)
	}

	if err := os.Setenv("DOCKER_CONFIG", dockerConfigDir); err != nil {
		return fmt.Errorf("cannot set DOCKER_CONFIG to %s: %s", dockerConfigDir, err)
	}

	if err := setDockerClient(); err != nil {
		return err
	}

	if err := setDockerApiClient(); err != nil {
		return err
	}

	return nil
}

func ServerVersion() (*types.Version, error) {
	ctx := context.Background()
	version, err := apiClient.ServerVersion(ctx)
	if err != nil {
		return nil, err
	}

	return &version, nil
}

type customReadCloser struct {
	*os.File
}

// ignore closing stdin after interactive mode, e.g. docker run -ti
func (r customReadCloser) Close() error {
	return nil
}

func setDockerClient() error {
	cliOpts := []command.DockerCliOption{
		command.WithInputStream(customReadCloser{os.Stdin}),
		command.WithOutputStream(logboek.GetOutStream()),
		command.WithErrorStream(logboek.GetErrStream()),
		command.WithContentTrust(false),
	}

	newCli, err := command.NewDockerCli(cliOpts...)
	if err != nil {
		return err
	}

	newCli.Out().SetIsTerminal(terminal.IsTerminal(int(os.Stdout.Fd())))
	newCli.In().SetIsTerminal(terminal.IsTerminal(int(os.Stdin.Fd())))

	opts := flags.NewClientOptions()
	if err := newCli.Initialize(opts); err != nil {
		return err
	}

	cli = newCli

	return nil
}

func setDockerApiClient() error {
	ctx := context.Background()
	serverVersion, err := cli.Client().ServerVersion(ctx)
	if err != nil {
		return err
	}

	apiClient, err = client.NewClientWithOpts(client.WithVersion(serverVersion.APIVersion))
	if err != nil {
		return err
	}

	return nil
}

func Debug() bool {
	return os.Getenv("WERF_DEBUG_DOCKER") == "1"
}
<commit_msg>[windows] Fix introspection does not work<commit_after>package docker

import (
	"fmt"
	"os"

	"golang.org/x/net/context"

	"github.com/flant/logboek"

	"github.com/docker/cli/cli/command"
	cliconfig "github.com/docker/cli/cli/config"
	"github.com/docker/cli/cli/flags"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/client"
)

var (
	cli       *command.DockerCli
	apiClient *client.Client
)

func Init(dockerConfigDir string) error {
	if dockerConfigDir != "" {
		cliconfig.SetDir(dockerConfigDir)
	}

	if err := os.Setenv("DOCKER_CONFIG", dockerConfigDir); err != nil {
		return fmt.Errorf("cannot set DOCKER_CONFIG to %s: %s", dockerConfigDir, err)
	}

	if err := setDockerClient(); err != nil {
		return err
	}

	if err := setDockerApiClient(); err != nil {
		return err
	}

	return nil
}

func ServerVersion() (*types.Version, error) {
	ctx := context.Background()
	version, err := apiClient.ServerVersion(ctx)
	if err != nil {
		return nil, err
	}

	return &version, nil
}

func setDockerClient() error {
	cliOpts := []command.DockerCliOption{
		command.WithOutputStream(logboek.GetOutStream()),
		command.WithErrorStream(logboek.GetErrStream()),
		command.WithContentTrust(false),
	}

	newCli, err := command.NewDockerCli(cliOpts...)
	if err != nil {
		return err
	}

	opts := flags.NewClientOptions()
	if err := newCli.Initialize(opts); err != nil {
		return err
	}

	cli = newCli

	return nil
}

func setDockerApiClient() error {
	ctx := context.Background()
	serverVersion, err := cli.Client().ServerVersion(ctx)
	if err != nil {
		return err
	}

	apiClient, err = client.NewClientWithOpts(client.WithVersion(serverVersion.APIVersion))
	if err != nil {
		return err
	}

	return nil
}

func Debug() bool {
	return os.Getenv("WERF_DEBUG_DOCKER") == "1"
}
<|endoftext|>


<commit_before>package logger

import (
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"github.com/fatih/color"
	"golang.org/x/crypto/ssh/terminal"
)

const (
	defaultTerminalWidth = 120

	logProcessDefaultProcessMsg      = "[RUNNING]"
	logProcessSuccessStatus          = "[OK]"
	logProcessFailedStatus           = "[FAILED]"
	logProcessTimeFormat             = "%5.2f sec "
	logProcessInlineProcessMsgFormat = "%s ..."

	logStateRightPartsSeparator = " "
)

var (
	indent = 0
)

func LogProcessInline(msg string, processFunc func() error) {
	logProcessInlineBase(msg, processFunc, colorizeStep, colorizeSuccess)
}

func LogServiceProcessInline(msg string, processFunc func() error) {
	logProcessInlineBase(msg, processFunc, colorizeService, colorizeService)
}

func LogProcess(msg, processMsg string, processFunc func() error) {
	logProcessBase(msg, processMsg, processFunc, colorizeStep, colorizeSuccess)
}

func LogServiceProcess(msg, processMsg string, processFunc func() error) {
	logProcessBase(msg, processMsg, processFunc, colorizeService, colorizeService)
}

func LogState(msg, state string) {
	logStateBase(msg, state, "", colorizeStep, colorizeService)
}

func LogServiceState(msg, state string) {
	logStateBase(msg, state, "", colorizeService, colorizeService)
}

func LogStep(msg string) {
	LogStepF("%s\n", msg)
}

func LogStepF(format string, args ...interface{}) {
	colorizeLogBaseF(os.Stdout, colorizeStep, format, args...)
}

func LogService(msg string) {
	LogServiceF("%s\n", msg)
}

func LogServiceF(format string, args ...interface{}) {
	colorizeLogBaseF(os.Stdout, colorizeService, format, args...)
}

func LogInfo(msg string) {
	LogInfoF("%s\n", msg)
}

func LogInfoF(format string, args ...interface{}) {
	colorizeLogBaseF(os.Stdout, colorizeInfo, format, args...)
}

func LogWarning(msg string) {
	LogWarningF("%s\n", msg)
}

func LogWarningF(format string, args ...interface{}) {
	colorizeLogBaseF(os.Stderr, colorizeWarning, format, args...)
}

func colorizeLogBaseF(w io.Writer, colorizeFunc func(string) string, format string, args ...interface{}) {
	logF(w, colorizeFunc(fmt.Sprintf(format, args...)))
}

func log(w io.Writer, msg string) {
	logF(w, "%s\n", msg)
}

func logF(w io.Writer, format string, args ...interface{}) {
	msg := fmt.Sprintf(format, args...)
	logBase(w, fmt.Sprintf("%s%s", logIndent(), msg))
}

func logBase(w io.Writer, msg string) {
	fmt.Fprintf(w, msg)
}

func logIndent() string {
	return strings.Repeat("  ", indent)
}

func withLogIndent(f func() error) error {
	indentUp()
	err := f()
	indentDown()

	return err
}

func indentUp() {
	indent += 1
}

func indentDown() {
	if indent == 0 {
		return
	}

	indent -= 1
}

func logProcessInlineBase(processMsg string, processFunc func() error, colorizeProcessMsgFunc, colorizeSuccessFunc func(string) string) error {
	processMsg = fmt.Sprintf(logProcessInlineProcessMsgFormat, processMsg)
	colorizeLogBaseF(os.Stdout, colorizeProcessMsgFunc, "%s", processMsg)

	resultStatus := logProcessSuccessStatus
	resultColorize := colorizeSuccessFunc
	start := time.Now()

	err := withLogIndent(processFunc)
	if err != nil {
		resultStatus = logProcessFailedStatus
		resultColorize = colorizeFail
	}

	elapsedSeconds := fmt.Sprintf(logProcessTimeFormat, time.Since(start).Seconds())

	rightPart := prepareLogStateRightPart(processMsg, resultStatus, elapsedSeconds, resultColorize)
	logBase(os.Stdout, fmt.Sprintf("%s\n", rightPart))

	return err
}

func logProcessBase(msg, processMsg string, processFunc func() error, colorizeMsgFunc, colorizeSuccessFunc func(string) string) error {
	if processMsg != "" {
		processMsg = logProcessDefaultProcessMsg
	}

	logStateBase(msg, processMsg, "", colorizeMsgFunc, colorizeSuccessFunc)

	start := time.Now()
	resultStatus := logProcessSuccessStatus

	err := withLogIndent(processFunc)

	elapsedSeconds := fmt.Sprintf(logProcessTimeFormat, time.Since(start).Seconds())

	if err != nil {
		resultStatus = logProcessFailedStatus
		logStateBase(msg, resultStatus, elapsedSeconds, colorizeFail, colorizeFail)

		return err
	}

	logStateBase(msg, resultStatus, elapsedSeconds, colorizeMsgFunc, colorizeSuccessFunc)

	return nil
}

func logStateBase(msg string, state, time string, colorizeLeftPartFunc, colorizeRightPartFunc func(string) string) {
	leftPart := prepareLogStateLeftPart(msg, state, time, colorizeLeftPartFunc)
	rightPart := prepareLogStateRightPart(msg, state, time, colorizeRightPartFunc)
	log(os.Stdout, fmt.Sprintf("%s%s", leftPart, rightPart))
}

func prepareLogStateLeftPart(msg, state, time string, colorizeFunc func(string) string) string {
	var result string

	spaceLength := availableTerminalLineSpace(state, timeOrStub(time))
	if spaceLength > 0 {
		if spaceLength > len(msg) {
			result = msg
		} else {
			result = msg[0:spaceLength]
		}
	} else {
		return ""
	}

	return colorizeFunc(result)
}

func prepareLogStateRightPart(msg, state, time string, colorizeFunc func(string) string) string {
	var result string
	spaceLength := availableTerminalLineSpace(msg)

	rightPartLength := len(state + timeOrStub(time) + logStateRightPartsSeparator)
	if spaceLength-rightPartLength > 0 {
		result += strings.Repeat(" ", spaceLength-rightPartLength)
	}

	var rightPart []string
	rightPart = append(rightPart, colorizeFunc(state))
	rightPart = append(rightPart, colorizeFunc(time))

	result += strings.Join(rightPart, logStateRightPartsSeparator)

	return result
}

func timeOrStub(time string) string {
	if time == "" {
		return fmt.Sprintf(logProcessTimeFormat, 0.0)
	}

	return time
}

func availableTerminalLineSpace(parts ...string) int {
	logIndentLength := len(logIndent())
	msgsLength := len(strings.Join(parts, " "))

	return terminalWidth() - logIndentLength - msgsLength
}

func terminalWidth() int {
	if terminal.IsTerminal(int(os.Stdout.Fd())) {
		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
		if err != nil {
			panic(err)
		}

		return w
	}

	return defaultTerminalWidth
}

func colorizeFail(msg string) string {
	return colorizeWarning(msg)
}

func colorizeSuccess(msg string) string {
	return colorize(msg, color.FgGreen, color.Bold)
}

func colorizeStep(msg string) string {
	return colorize(msg, color.FgYellow, color.Bold)
}

func colorizeService(msg string) string {
	return colorize(msg, color.FgWhite, color.Bold)
}

func colorizeInfo(msg string) string {
	return colorize(msg, color.FgBlue)
}

func colorizeWarning(msg string) string {
	return colorize(msg, color.FgRed, color.Bold)
}

func colorize(msg string, attributes ...color.Attribute) string {
	return color.New(attributes...).Sprint(msg)
}
<commit_msg>[go dapp] fix logger<commit_after>package logger

import (
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"github.com/fatih/color"
	"golang.org/x/crypto/ssh/terminal"
)

const (
	defaultTerminalWidth = 120

	logProcessDefaultProcessMsg      = "[RUNNING]"
	logProcessSuccessStatus          = "[OK]"
	logProcessFailedStatus           = "[FAILED]"
	logProcessTimeFormat             = "%5.2f sec "
	logProcessInlineProcessMsgFormat = "%s ..."

	logStateRightPartsSeparator = " "
)

var (
	indent = 0
)

func LogProcessInline(msg string, processFunc func() error) error {
	return logProcessInlineBase(msg, processFunc, colorizeStep, colorizeSuccess)
}

func LogServiceProcessInline(msg string, processFunc func() error) error {
	return logProcessInlineBase(msg, processFunc, colorizeService, colorizeService)
}

func LogProcess(msg, processMsg string, processFunc func() error) error {
	return logProcessBase(msg, processMsg, processFunc, colorizeStep, colorizeSuccess)
}

func LogServiceProcess(msg, processMsg string, processFunc func() error) error {
	return logProcessBase(msg, processMsg, processFunc, colorizeService, colorizeService)
}

func LogState(msg, state string) {
	logStateBase(msg, state, "", colorizeStep, colorizeService)
}

func LogServiceState(msg, state string) {
	logStateBase(msg, state, "", colorizeService, colorizeService)
}

func LogStep(msg string) {
	LogStepF("%s\n", msg)
}

func LogStepF(format string, args ...interface{}) {
	colorizeLogBaseF(os.Stdout, colorizeStep, format, args...)
}

func LogService(msg string) {
	LogServiceF("%s\n", msg)
}

func LogServiceF(format string, args ...interface{}) {
	colorizeLogBaseF(os.Stdout, colorizeService, format, args...)
}

func LogInfo(msg string) {
	LogInfoF("%s\n", msg)
}

func LogInfoF(format string, args ...interface{}) {
	colorizeLogBaseF(os.Stdout, colorizeInfo, format, args...)
}

func LogWarning(msg string) {
	LogWarningF("%s\n", msg)
}

func LogWarningF(format string, args ...interface{}) {
	colorizeLogBaseF(os.Stderr, colorizeWarning, format, args...)
}

func colorizeLogBaseF(w io.Writer, colorizeFunc func(string) string, format string, args ...interface{}) {
	logF(w, colorizeFunc(fmt.Sprintf(format, args...)))
}

func log(w io.Writer, msg string) {
	logF(w, "%s\n", msg)
}

func logF(w io.Writer, format string, args ...interface{}) {
	msg := fmt.Sprintf(format, args...)
	logBase(w, fmt.Sprintf("%s%s", logIndent(), msg))
}

func logBase(w io.Writer, msg string) {
	fmt.Fprintf(w, msg)
}

func logIndent() string {
	return strings.Repeat("  ", indent)
}

func withLogIndent(f func() error) error {
	indentUp()
	err := f()
	indentDown()

	return err
}

func indentUp() {
	indent += 1
}

func indentDown() {
	if indent == 0 {
		return
	}

	indent -= 1
}

func logProcessInlineBase(processMsg string, processFunc func() error, colorizeProcessMsgFunc, colorizeSuccessFunc func(string) string) error {
	processMsg = fmt.Sprintf(logProcessInlineProcessMsgFormat, processMsg)
	colorizeLogBaseF(os.Stdout, colorizeProcessMsgFunc, "%s", processMsg)

	resultStatus := logProcessSuccessStatus
	resultColorize := colorizeSuccessFunc
	start := time.Now()

	err := withLogIndent(processFunc)
	if err != nil {
		resultStatus = logProcessFailedStatus
		resultColorize = colorizeFail
	}

	elapsedSeconds := fmt.Sprintf(logProcessTimeFormat, time.Since(start).Seconds())

	rightPart := prepareLogStateRightPart(processMsg, resultStatus, elapsedSeconds, resultColorize)
	logBase(os.Stdout, fmt.Sprintf("%s\n", rightPart))

	return err
}

func logProcessBase(msg, processMsg string, processFunc func() error, colorizeMsgFunc, colorizeSuccessFunc func(string) string) error {
	if processMsg == "" {
		processMsg = logProcessDefaultProcessMsg
	}

	logStateBase(msg, processMsg, "", colorizeMsgFunc, colorizeSuccessFunc)

	start := time.Now()
	resultStatus := logProcessSuccessStatus

	err := withLogIndent(processFunc)

	elapsedSeconds := fmt.Sprintf(logProcessTimeFormat, time.Since(start).Seconds())

	if err != nil {
		resultStatus = logProcessFailedStatus
		logStateBase(msg, resultStatus, elapsedSeconds, colorizeFail, colorizeFail)

		return err
	}

	logStateBase(msg, resultStatus, elapsedSeconds, colorizeMsgFunc, colorizeSuccessFunc)

	return nil
}

func logStateBase(msg string, state, time string, colorizeLeftPartFunc, colorizeRightPartFunc func(string) string) {
	leftPart := prepareLogStateLeftPart(msg, state, time, colorizeLeftPartFunc)
	rightPart := prepareLogStateRightPart(msg, state, time, colorizeRightPartFunc)
	log(os.Stdout, fmt.Sprintf("%s%s", leftPart, rightPart))
}

func prepareLogStateLeftPart(msg, state, time string, colorizeFunc func(string) string) string {
	var result string

	spaceLength := availableTerminalLineSpace(state, timeOrStub(time))
	if spaceLength > 0 {
		if spaceLength > len(msg) {
			result = msg
		} else {
			result = msg[0:spaceLength]
		}
	} else {
		return ""
	}

	return colorizeFunc(result)
}

func prepareLogStateRightPart(msg, state, time string, colorizeFunc func(string) string) string {
	var result string
	spaceLength := availableTerminalLineSpace(msg)

	rightPartLength := len(state + timeOrStub(time) + logStateRightPartsSeparator)
	if spaceLength-rightPartLength > 0 {
		result += strings.Repeat(" ", spaceLength-rightPartLength)
	}

	var rightPart []string
	rightPart = append(rightPart, colorizeFunc(state))
	rightPart = append(rightPart, colorizeFunc(time))

	result += strings.Join(rightPart, logStateRightPartsSeparator)

	return result
}

func timeOrStub(time string) string {
	if time == "" {
		return fmt.Sprintf(logProcessTimeFormat, 0.0)
	}

	return time
}

func availableTerminalLineSpace(parts ...string) int {
	logIndentLength := len(logIndent())
	msgsLength := len(strings.Join(parts, " "))

	return terminalWidth() - logIndentLength - msgsLength
}

func terminalWidth() int {
	if terminal.IsTerminal(int(os.Stdout.Fd())) {
		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
		if err != nil {
			panic(err)
		}

		return w
	}

	return defaultTerminalWidth
}

func colorizeFail(msg string) string {
	return colorizeWarning(msg)
}

func colorizeSuccess(msg string) string {
	return colorize(msg, color.FgGreen, color.Bold)
}

func colorizeStep(msg string) string {
	return colorize(msg, color.FgYellow, color.Bold)
}

func colorizeService(msg string) string {
	return colorize(msg, color.FgWhite, color.Bold)
}

func colorizeInfo(msg string) string {
	return colorize(msg, color.FgBlue)
}

func colorizeWarning(msg string) string {
	return colorize(msg, color.FgRed, color.Bold)
}

func colorize(msg string, attributes ...color.Attribute) string {
	return color.New(attributes...).Sprint(msg)
}
<|endoftext|>