<commit_before>// Copyright 2013-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Convert and copy a file.
//
// Synopsis:
//     dd [OPTIONS...] [-inName FILE] [-outName FILE]
//
// Description:
//     dd is modeled after dd(1).
//
// Options:
//     -ibs n:   input block size (default=1)
//     -obs n:   output block size (default=1)
//     -bs n:    input and output block size (default=0)
//     -skip n:  skip n ibs-sized input blocks before reading (default=0)
//     -seek n:  seek n obs-sized output blocks before writing (default=0)
//     -conv s:  Convert the file on a specific way, like notrunc
//     -count n: copy only n ibs-sized input blocks
//     -inName:  defaults to stdin
//     -outName: defaults to stdout
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"math"
	"os"
	"strings"
	"sync"
)

var (
	ibs     = flag.Int64("ibs", 512, "Default input block size")
	obs     = flag.Int64("obs", 512, "Default output block size")
	bs      = flag.Int64("bs", 0, "Default input and output block size")
	skip    = flag.Int64("skip", 0, "skip N ibs-sized blocks before reading")
	seek    = flag.Int64("seek", 0, "seek N obs-sized blocks before writing")
	conv    = flag.String("conv", "none", "Convert the file on a specific way, like notrunc")
	count   = flag.Int64("count", math.MaxInt64, "copy only N input blocks")
	inName  = flag.String("if", "", "Input file")
	outName = flag.String("of", "", "Output file")
)

// intermediateBuffer is a buffer that one can write to and read from.
type intermediateBuffer interface {
	io.ReaderFrom
	io.WriterTo
}

// chunkedBuffer is an intermediateBuffer with a specific size.
type chunkedBuffer struct {
	outChunk  int64
	length    int64
	data      []byte
	transform func([]byte) []byte
}

// newChunkedBuffer returns an intermediateBuffer that stores inChunkSize-sized
// chunks of data and writes them to writers in outChunkSize-sized chunks.
func newChunkedBuffer(inChunkSize int64, outChunkSize int64, transform func([]byte) []byte) intermediateBuffer {
	return &chunkedBuffer{
		outChunk:  outChunkSize,
		length:    0,
		data:      make([]byte, inChunkSize),
		transform: transform,
	}
}

// ReadFrom reads an inChunkSize-sized chunk from r into the buffer.
func (cb *chunkedBuffer) ReadFrom(r io.Reader) (int64, error) {
	n, err := r.Read(cb.data)
	cb.length = int64(n)

	// Convert to EOF explicitly.
	if n == 0 && err == nil {
		return 0, io.EOF
	}
	return int64(n), err
}

// WriteTo writes from the buffer to w in outChunkSize-sized chunks.
func (cb *chunkedBuffer) WriteTo(w io.Writer) (int64, error) {
	var i int64
	for i = 0; i < int64(cb.length); {
		chunk := cb.outChunk
		if i+chunk > cb.length {
			chunk = cb.length - i
		}
		got, err := w.Write(cb.transform(cb.data[i : i+chunk]))
		// Ugh, Go cruft: io.Writer.Write returns (int, error).
		// io.WriterTo.WriteTo returns (int64, error). So we have to
		// cast.
		i += int64(got)
		if err != nil {
			return i, err
		}
		if int64(got) != chunk {
			return 0, io.ErrShortWrite
		}
	}
	return i, nil
}

// bufferPool is a pool of intermediateBuffers.
type bufferPool struct {
	f func() intermediateBuffer
	c chan intermediateBuffer
}

func newBufferPool(size int64, f func() intermediateBuffer) bufferPool {
	return bufferPool{
		f: f,
		c: make(chan intermediateBuffer, size),
	}
}

// Put returns a buffer to the pool for later use.
func (bp bufferPool) Put(b intermediateBuffer) {
	// Non-blocking write in case pool has filled up (too many buffers
	// returned, none being used).
	select {
	case bp.c <- b:
	default:
	}
}

// Get returns a buffer from the pool or allocates a new buffer if none is
// available.
func (bp bufferPool) Get() intermediateBuffer {
	select {
	case buf := <-bp.c:
		return buf
	default:
		return bp.f()
	}
}

func (bp bufferPool) Destroy() {
	close(bp.c)
}

func parallelChunkedCopy(r io.Reader, w io.Writer, inBufSize, outBufSize int64, transform func([]byte) []byte) error {
	// Make the channels deep enough to hold a total of 1GiB of data.
	depth := (1024 * 1024 * 1024) / inBufSize
	// But keep it reasonable!
	if depth > 8192 {
		depth = 8192
	}

	readyBufs := make(chan intermediateBuffer, depth)
	pool := newBufferPool(depth, func() intermediateBuffer {
		return newChunkedBuffer(inBufSize, outBufSize, transform)
	})
	defer pool.Destroy()

	// Closing quit makes both goroutines below exit.
	quit := make(chan struct{})

	// errs contains the error value to be returned.
	errs := make(chan error, 1)
	defer close(errs)

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		// Closing this unblocks the writing for-loop below.
		defer close(readyBufs)
		defer wg.Done()

		for {
			select {
			case <-quit:
				return
			default:
				buf := pool.Get()
				n, err := buf.ReadFrom(r)
				if n > 0 {
					readyBufs <- buf
				}
				if err == io.EOF {
					return
				}
				if n == 0 || err != nil {
					errs <- fmt.Errorf("input error: %v", err)
					return
				}
			}
		}
	}()

	var writeErr error
	for buf := range readyBufs {
		if _, err := buf.WriteTo(w); err != nil {
			writeErr = fmt.Errorf("output error: %v", err)
			break
		}
		pool.Put(buf)
	}

	// This will force the goroutine to quit if an error occured writing.
	close(quit)

	// Wait for goroutine to exit.
	wg.Wait()

	select {
	case readErr := <-errs:
		return readErr
	default:
		return writeErr
	}
}

// nothingWriter implements io.Writer and writes to nothing.
type nothingWriter struct{}

// Write implements io.Writer.
func (nothingWriter) Write(p []byte) (int, error) {
	// Pretend we wrote.
	return len(p), nil
}

// sectionReader implements a SectionReader on an underlying implementation of
// io.Reader (as opposed to io.SectionReader which uses io.ReaderAt).
type sectionReader struct {
	base   int64
	offset int64
	limit  int64
	io.Reader
}

// newStreamSectionReader uses an io.Reader to implement an io.Reader that
// seeks to offset and reads at most n bytes.
//
// This is useful if you want to use a NewSectionReader with stdin or other
// types of pipes (things that can't be seek'd or pread from).
func newStreamSectionReader(r io.Reader, offset int64, n int64) io.Reader {
	limit := offset + n
	if limit < 0 {
		limit = math.MaxInt64
	}
	return &sectionReader{offset, 0, limit, r}
}

// Read implements io.Reader.
func (s *sectionReader) Read(p []byte) (int, error) {
	if s.offset == 0 && s.base != 0 {
		if n, err := io.CopyN(&nothingWriter{}, s.Reader, s.base); err != nil {
			return 0, err
		} else if n != s.base {
			// Can't happen.
			return 0, fmt.Errorf("error skipping input bytes, short write.")
		}
		s.offset = s.base
	}

	if s.offset >= s.limit {
		return 0, io.EOF
	}

	if max := s.limit - s.offset; int64(len(p)) > max {
		p = p[0:max]
	}

	n, err := s.Reader.Read(p)
	s.offset += int64(n)

	// Convert to io.EOF explicitly.
	if n == 0 && err == nil {
		return 0, io.EOF
	}
	return n, err
}

// inFile opens the input file and seeks to the right position.
func inFile(name string, inputBytes int64, skip int64, count int64) (io.Reader, error) {
	maxRead := int64(math.MaxInt64)
	if count != math.MaxInt64 {
		maxRead = count * inputBytes
	}

	if name == "" {
		// os.Stdin is an io.ReaderAt, but you can't actually call
		// pread(2) on it, so use the copying section reader.
		return newStreamSectionReader(os.Stdin, inputBytes*skip, maxRead), nil
	}

	in, err := os.Open(name)
	if err != nil {
		return nil, fmt.Errorf("error opening input file %q: %v", name, err)
	}
	return io.NewSectionReader(in, inputBytes*skip, maxRead), nil
}

// outFile opens the output file and seeks to the right position.
func outFile(name string, outputBytes int64, seek int64) (io.Writer, error) {
	var out io.WriteSeeker
	var err error
	if name == "" {
		out = os.Stdout
	} else {
		if out, err = os.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666); err != nil {
			return nil, fmt.Errorf("error opening output file %q: %v", name, err)
		}
	}
	if seek*outputBytes != 0 {
		if _, err := out.Seek(seek*outputBytes, io.SeekCurrent); err != nil {
			return nil, fmt.Errorf("error seeking output file: %v", err)
		}
	}
	return out, nil
}

func usage() {
	// If the conversions get more complex we can dump
	// the convs map. For now, it's not really worth it.
	log.Fatal(`Usage: dd [if=file] [of=file] [conv=lcase|ucase] [seek=#] [skip=#] [count=#] [bs=#] [ibs=#] [obs=#]
		options may also be invoked Go-style as -opt value or -opt=value
		bs, if specified, overrides ibs and obs`)
}

func convertArgs(osArgs []string) []string {
	// EVERYTHING in dd follows x=y. So blindly split and convert.
	var args []string
	for _, v := range osArgs {
		l := strings.SplitN(v, "=", 2)

		// We only fix the exact case for x=y.
		if len(l) == 2 {
			l[0] = "-" + l[0]
		}

		args = append(args, l...)
	}
	return args
}

func main() {
	// rather than, in essence, recreating all the apparatus of flag.xxxx
	// with the if= bits, including dup checking, conversion, etc. we just
	// convert the arguments and then run flag.Parse. Gross, but hey, it
	// works.
	os.Args = convertArgs(os.Args)
	flag.Parse()

	if len(flag.Args()) > 0 {
		usage()
	}

	convs := map[string]func([]byte) []byte{
		"none":  func(b []byte) []byte { return b },
		"ucase": bytes.ToUpper,
		"lcase": bytes.ToLower,
	}
	convert, ok := convs[*conv]
	if !ok {
		usage()
	}

	// bs = both 'ibs' and 'obs' (IEEE Std 1003.1 - 2013)
	if *bs > 0 {
		*ibs = *bs
		*obs = *bs
	}

	in, err := inFile(*inName, *ibs, *skip, *count)
	if err != nil {
		log.Fatal(err)
	}
	out, err := outFile(*outName, *obs, *seek)
	if err != nil {
		log.Fatal(err)
	}
	if err := parallelChunkedCopy(in, out, *ibs, *obs, convert); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Use ioutil.Discard instead of self-written nothingWriter.<commit_after>// Copyright 2013-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Convert and copy a file.
//
// Synopsis:
//     dd [OPTIONS...] [-inName FILE] [-outName FILE]
//
// Description:
//     dd is modeled after dd(1).
//
// Options:
//     -ibs n:   input block size (default=1)
//     -obs n:   output block size (default=1)
//     -bs n:    input and output block size (default=0)
//     -skip n:  skip n ibs-sized input blocks before reading (default=0)
//     -seek n:  seek n obs-sized output blocks before writing (default=0)
//     -conv s:  Convert the file on a specific way, like notrunc
//     -count n: copy only n ibs-sized input blocks
//     -inName:  defaults to stdin
//     -outName: defaults to stdout
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"os"
	"strings"
	"sync"
)

var (
	ibs     = flag.Int64("ibs", 512, "Default input block size")
	obs     = flag.Int64("obs", 512, "Default output block size")
	bs      = flag.Int64("bs", 0, "Default input and output block size")
	skip    = flag.Int64("skip", 0, "skip N ibs-sized blocks before reading")
	seek    = flag.Int64("seek", 0, "seek N obs-sized blocks before writing")
	conv    = flag.String("conv", "none", "Convert the file on a specific way, like notrunc")
	count   = flag.Int64("count", math.MaxInt64, "copy only N input blocks")
	inName  = flag.String("if", "", "Input file")
	outName = flag.String("of", "", "Output file")
)

// intermediateBuffer is a buffer that one can write to and read from.
type intermediateBuffer interface {
	io.ReaderFrom
	io.WriterTo
}

// chunkedBuffer is an intermediateBuffer with a specific size.
type chunkedBuffer struct {
	outChunk  int64
	length    int64
	data      []byte
	transform func([]byte) []byte
}

// newChunkedBuffer returns an intermediateBuffer that stores inChunkSize-sized
// chunks of data and writes them to writers in outChunkSize-sized chunks.
func newChunkedBuffer(inChunkSize int64, outChunkSize int64, transform func([]byte) []byte) intermediateBuffer {
	return &chunkedBuffer{
		outChunk:  outChunkSize,
		length:    0,
		data:      make([]byte, inChunkSize),
		transform: transform,
	}
}

// ReadFrom reads an inChunkSize-sized chunk from r into the buffer.
func (cb *chunkedBuffer) ReadFrom(r io.Reader) (int64, error) {
	n, err := r.Read(cb.data)
	cb.length = int64(n)

	// Convert to EOF explicitly.
	if n == 0 && err == nil {
		return 0, io.EOF
	}
	return int64(n), err
}

// WriteTo writes from the buffer to w in outChunkSize-sized chunks.
func (cb *chunkedBuffer) WriteTo(w io.Writer) (int64, error) {
	var i int64
	for i = 0; i < int64(cb.length); {
		chunk := cb.outChunk
		if i+chunk > cb.length {
			chunk = cb.length - i
		}
		got, err := w.Write(cb.transform(cb.data[i : i+chunk]))
		// Ugh, Go cruft: io.Writer.Write returns (int, error).
		// io.WriterTo.WriteTo returns (int64, error). So we have to
		// cast.
		i += int64(got)
		if err != nil {
			return i, err
		}
		if int64(got) != chunk {
			return 0, io.ErrShortWrite
		}
	}
	return i, nil
}

// bufferPool is a pool of intermediateBuffers.
type bufferPool struct {
	f func() intermediateBuffer
	c chan intermediateBuffer
}

func newBufferPool(size int64, f func() intermediateBuffer) bufferPool {
	return bufferPool{
		f: f,
		c: make(chan intermediateBuffer, size),
	}
}

// Put returns a buffer to the pool for later use.
func (bp bufferPool) Put(b intermediateBuffer) {
	// Non-blocking write in case pool has filled up (too many buffers
	// returned, none being used).
	select {
	case bp.c <- b:
	default:
	}
}

// Get returns a buffer from the pool or allocates a new buffer if none is
// available.
func (bp bufferPool) Get() intermediateBuffer {
	select {
	case buf := <-bp.c:
		return buf
	default:
		return bp.f()
	}
}

func (bp bufferPool) Destroy() {
	close(bp.c)
}

func parallelChunkedCopy(r io.Reader, w io.Writer, inBufSize, outBufSize int64, transform func([]byte) []byte) error {
	// Make the channels deep enough to hold a total of 1GiB of data.
	depth := (1024 * 1024 * 1024) / inBufSize
	// But keep it reasonable!
	if depth > 8192 {
		depth = 8192
	}

	readyBufs := make(chan intermediateBuffer, depth)
	pool := newBufferPool(depth, func() intermediateBuffer {
		return newChunkedBuffer(inBufSize, outBufSize, transform)
	})
	defer pool.Destroy()

	// Closing quit makes both goroutines below exit.
	quit := make(chan struct{})

	// errs contains the error value to be returned.
	errs := make(chan error, 1)
	defer close(errs)

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		// Closing this unblocks the writing for-loop below.
		defer close(readyBufs)
		defer wg.Done()

		for {
			select {
			case <-quit:
				return
			default:
				buf := pool.Get()
				n, err := buf.ReadFrom(r)
				if n > 0 {
					readyBufs <- buf
				}
				if err == io.EOF {
					return
				}
				if n == 0 || err != nil {
					errs <- fmt.Errorf("input error: %v", err)
					return
				}
			}
		}
	}()

	var writeErr error
	for buf := range readyBufs {
		if _, err := buf.WriteTo(w); err != nil {
			writeErr = fmt.Errorf("output error: %v", err)
			break
		}
		pool.Put(buf)
	}

	// This will force the goroutine to quit if an error occured writing.
	close(quit)

	// Wait for goroutine to exit.
	wg.Wait()

	select {
	case readErr := <-errs:
		return readErr
	default:
		return writeErr
	}
}

// sectionReader implements a SectionReader on an underlying implementation of
// io.Reader (as opposed to io.SectionReader which uses io.ReaderAt).
type sectionReader struct {
	base   int64
	offset int64
	limit  int64
	io.Reader
}

// newStreamSectionReader uses an io.Reader to implement an io.Reader that
// seeks to offset and reads at most n bytes.
//
// This is useful if you want to use a NewSectionReader with stdin or other
// types of pipes (things that can't be seek'd or pread from).
func newStreamSectionReader(r io.Reader, offset int64, n int64) io.Reader {
	limit := offset + n
	if limit < 0 {
		limit = math.MaxInt64
	}
	return &sectionReader{offset, 0, limit, r}
}

// Read implements io.Reader.
func (s *sectionReader) Read(p []byte) (int, error) {
	if s.offset == 0 && s.base != 0 {
		if n, err := io.CopyN(ioutil.Discard, s.Reader, s.base); err != nil {
			return 0, err
		} else if n != s.base {
			// Can't happen.
			return 0, fmt.Errorf("error skipping input bytes, short write.")
		}
		s.offset = s.base
	}

	if s.offset >= s.limit {
		return 0, io.EOF
	}

	if max := s.limit - s.offset; int64(len(p)) > max {
		p = p[0:max]
	}

	n, err := s.Reader.Read(p)
	s.offset += int64(n)

	// Convert to io.EOF explicitly.
	if n == 0 && err == nil {
		return 0, io.EOF
	}
	return n, err
}

// inFile opens the input file and seeks to the right position.
func inFile(name string, inputBytes int64, skip int64, count int64) (io.Reader, error) {
	maxRead := int64(math.MaxInt64)
	if count != math.MaxInt64 {
		maxRead = count * inputBytes
	}

	if name == "" {
		// os.Stdin is an io.ReaderAt, but you can't actually call
		// pread(2) on it, so use the copying section reader.
		return newStreamSectionReader(os.Stdin, inputBytes*skip, maxRead), nil
	}

	in, err := os.Open(name)
	if err != nil {
		return nil, fmt.Errorf("error opening input file %q: %v", name, err)
	}
	return io.NewSectionReader(in, inputBytes*skip, maxRead), nil
}

// outFile opens the output file and seeks to the right position.
func outFile(name string, outputBytes int64, seek int64) (io.Writer, error) {
	var out io.WriteSeeker
	var err error
	if name == "" {
		out = os.Stdout
	} else {
		if out, err = os.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666); err != nil {
			return nil, fmt.Errorf("error opening output file %q: %v", name, err)
		}
	}
	if seek*outputBytes != 0 {
		if _, err := out.Seek(seek*outputBytes, io.SeekCurrent); err != nil {
			return nil, fmt.Errorf("error seeking output file: %v", err)
		}
	}
	return out, nil
}

func usage() {
	// If the conversions get more complex we can dump
	// the convs map. For now, it's not really worth it.
	log.Fatal(`Usage: dd [if=file] [of=file] [conv=lcase|ucase] [seek=#] [skip=#] [count=#] [bs=#] [ibs=#] [obs=#]
		options may also be invoked Go-style as -opt value or -opt=value
		bs, if specified, overrides ibs and obs`)
}

func convertArgs(osArgs []string) []string {
	// EVERYTHING in dd follows x=y. So blindly split and convert.
	var args []string
	for _, v := range osArgs {
		l := strings.SplitN(v, "=", 2)

		// We only fix the exact case for x=y.
		if len(l) == 2 {
			l[0] = "-" + l[0]
		}

		args = append(args, l...)
	}
	return args
}

func main() {
	// rather than, in essence, recreating all the apparatus of flag.xxxx
	// with the if= bits, including dup checking, conversion, etc. we just
	// convert the arguments and then run flag.Parse. Gross, but hey, it
	// works.
	os.Args = convertArgs(os.Args)
	flag.Parse()

	if len(flag.Args()) > 0 {
		usage()
	}

	convs := map[string]func([]byte) []byte{
		"none":  func(b []byte) []byte { return b },
		"ucase": bytes.ToUpper,
		"lcase": bytes.ToLower,
	}
	convert, ok := convs[*conv]
	if !ok {
		usage()
	}

	// bs = both 'ibs' and 'obs' (IEEE Std 1003.1 - 2013)
	if *bs > 0 {
		*ibs = *bs
		*obs = *bs
	}

	in, err := inFile(*inName, *ibs, *skip, *count)
	if err != nil {
		log.Fatal(err)
	}
	out, err := outFile(*outName, *obs, *seek)
	if err != nil {
		log.Fatal(err)
	}
	if err := parallelChunkedCopy(in, out, *ibs, *obs, convert); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package token

import (
	"gob"
	"io"
	"os"
)

type serializedFile struct {
	// fields correspond 1:1 to fields with same (lower-case) name in File
	Name  string
	Base  int
	Size  int
	Lines []int
	Infos []lineInfo
}

type serializedFileSet struct {
	Base  int
	Files []serializedFile
}

// Read reads the fileset from r into s; s must not be nil.
func (s *FileSet) Read(r io.Reader) os.Error {
	var ss serializedFileSet
	if err := gob.NewDecoder(r).Decode(&ss); err != nil {
		return err
	}

	s.mutex.Lock()
	s.base = ss.Base
	files := make([]*File, len(ss.Files))
	for i := 0; i < len(ss.Files); i++ {
		f := &ss.Files[i]
		files[i] = &File{s, f.Name, f.Base, f.Size, f.Lines, f.Infos}
	}
	s.files = files
	s.last = nil
	s.mutex.Unlock()

	return nil
}

// Write writes the fileset s to w.
func (s *FileSet) Write(w io.Writer) os.Error {
	var ss serializedFileSet

	s.mutex.Lock()
	ss.Base = s.base
	files := make([]serializedFile, len(s.files))
	for i, f := range s.files {
		files[i] = serializedFile{f.name, f.base, f.size, f.lines, f.infos}
	}
	ss.Files = files
	s.mutex.Unlock()

	return gob.NewEncoder(w).Encode(ss)
}
<commit_msg>go/token: document deserialization property<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package token

import (
	"gob"
	"io"
	"os"
)

type serializedFile struct {
	// fields correspond 1:1 to fields with same (lower-case) name in File
	Name  string
	Base  int
	Size  int
	Lines []int
	Infos []lineInfo
}

type serializedFileSet struct {
	Base  int
	Files []serializedFile
}

func (s *serializedFileSet) Read(r io.Reader) os.Error {
	return gob.NewDecoder(r).Decode(s)
}

func (s *serializedFileSet) Write(w io.Writer) os.Error {
	return gob.NewEncoder(w).Encode(s)
}

// Read reads the fileset from r into s; s must not be nil.
// If r does not also implement io.ByteReader, it will be wrapped in a bufio.Reader.
func (s *FileSet) Read(r io.Reader) os.Error {
	var ss serializedFileSet
	if err := ss.Read(r); err != nil {
		return err
	}

	s.mutex.Lock()
	s.base = ss.Base
	files := make([]*File, len(ss.Files))
	for i := 0; i < len(ss.Files); i++ {
		f := &ss.Files[i]
		files[i] = &File{s, f.Name, f.Base, f.Size, f.Lines, f.Infos}
	}
	s.files = files
	s.last = nil
	s.mutex.Unlock()

	return nil
}

// Write writes the fileset s to w.
func (s *FileSet) Write(w io.Writer) os.Error {
	var ss serializedFileSet

	s.mutex.Lock()
	ss.Base = s.base
	files := make([]serializedFile, len(s.files))
	for i, f := range s.files {
		files[i] = serializedFile{f.name, f.base, f.size, f.lines, f.infos}
	}
	ss.Files = files
	s.mutex.Unlock()

	return ss.Write(w)
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package source

import (
	"context"
	"time"

	"go.uber.org/zap"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/tools/cache"

	sourceinformer "knative.dev/pkg/client/injection/ducks/duck/v1/source"
	kubeclient "knative.dev/pkg/client/injection/kube/client"
	"knative.dev/pkg/configmap"
	"knative.dev/pkg/controller"
	"knative.dev/pkg/injection"
	"knative.dev/pkg/injection/clients/dynamicclient"
	"knative.dev/pkg/logging"
	pkgreconciler "knative.dev/pkg/reconciler"

	kedaresources "knative.dev/eventing-autoscaler-keda/pkg/reconciler/keda"
	kedaclient "knative.dev/eventing-autoscaler-keda/third_party/pkg/client/injection/client"
	scaledobjectinformer "knative.dev/eventing-autoscaler-keda/third_party/pkg/client/injection/informers/keda/v1alpha1/scaledobject"
)

const (
	// ReconcilerName is the name of the reconciler.
	ReconcilerName = "KEDASourceDucks"
)

// NewController returns a function that initializes the controller and
// Registers event handlers to enqueue events
func NewController(crd string, gvr schema.GroupVersionResource, gvk schema.GroupVersionKind) injection.ControllerConstructor {
	return func(ctx context.Context,
		cmw configmap.Watcher,
	) *controller.Impl {
		logger := logging.FromContext(ctx)
		sourceduckInformer := sourceinformer.Get(ctx)

		var sourceInformer cache.SharedIndexInformer
		var sourceLister cache.GenericLister

		var err error
		for i := 0; i < 10; i++ {
			sourceInformer, sourceLister, err = sourceduckInformer.Get(ctx, gvr)
			if err == nil {
				break
			} else if apierrors.IsNotFound(err) {
				logger.Debug("SourceDuckInformer not found -> waiting", zap.String("GVR", gvr.String()), zap.Error(err))
				time.Sleep(1 * time.Second)
			} else {
				logger.Errorw("Error getting source informer", zap.String("GVR", gvr.String()), zap.Error(err))
				return nil
			}
		}

		scaledobjectInformer := scaledobjectinformer.Get(ctx)

		r := &Reconciler{
			kubeClient:      kubeclient.Get(ctx),
			kedaClient:      kedaclient.Get(ctx),
			sourceInterface: dynamicclient.Get(ctx).Resource(gvr),
			sourceLister:    sourceLister,
			gvk:             gvk,
			gvr:             gvr,
		}
		impl := controller.NewImpl(r, logger, ReconcilerName)

		logger.Info("Setting up event handlers")
		sourceInformer.AddEventHandler(cache.FilteringResourceEventHandler{
			FilterFunc: pkgreconciler.AnnotationFilterFunc(kedaresources.AutoscalingClassAnnotation, kedaresources.KEDA, false),
			Handler:    controller.HandleAll(impl.Enqueue),
		})

		// don't handle updates on ScaledObject.Status field
		scaledobjectInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{
			FilterFunc: controller.FilterControllerGVK(gvk),
			Handler: cache.ResourceEventHandlerFuncs{
				AddFunc: impl.EnqueueControllerOf,
				UpdateFunc: func(old, new interface{}) {
					if mOld, ok := old.(metav1.Object); ok {
						if mNew, ok := new.(metav1.Object); ok {
							if mNew.GetGeneration() != mOld.GetGeneration() {
								impl.EnqueueControllerOf(new)
							}
						}
					}
				},
				DeleteFunc: impl.EnqueueControllerOf,
			},
		})

		return impl
	}
}
<commit_msg>Use `controller.NewContext` (#176)<commit_after>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package source

import (
	"context"
	"time"

	"go.uber.org/zap"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/tools/cache"

	sourceinformer "knative.dev/pkg/client/injection/ducks/duck/v1/source"
	kubeclient "knative.dev/pkg/client/injection/kube/client"
	"knative.dev/pkg/configmap"
	"knative.dev/pkg/controller"
	"knative.dev/pkg/injection"
	"knative.dev/pkg/injection/clients/dynamicclient"
	"knative.dev/pkg/logging"
	pkgreconciler "knative.dev/pkg/reconciler"

	kedaresources "knative.dev/eventing-autoscaler-keda/pkg/reconciler/keda"
	kedaclient "knative.dev/eventing-autoscaler-keda/third_party/pkg/client/injection/client"
	scaledobjectinformer "knative.dev/eventing-autoscaler-keda/third_party/pkg/client/injection/informers/keda/v1alpha1/scaledobject"
)

const (
	// ReconcilerName is the name of the reconciler.
	ReconcilerName = "KEDASourceDucks"
)

// NewController returns a function that initializes the controller and
// Registers event handlers to enqueue events
func NewController(crd string, gvr schema.GroupVersionResource, gvk schema.GroupVersionKind) injection.ControllerConstructor {
	return func(ctx context.Context,
		cmw configmap.Watcher,
	) *controller.Impl {
		logger := logging.FromContext(ctx)
		sourceduckInformer := sourceinformer.Get(ctx)

		var sourceInformer cache.SharedIndexInformer
		var sourceLister cache.GenericLister

		var err error
		for i := 0; i < 10; i++ {
			sourceInformer, sourceLister, err = sourceduckInformer.Get(ctx, gvr)
			if err == nil {
				break
			} else if apierrors.IsNotFound(err) {
				logger.Debug("SourceDuckInformer not found -> waiting", zap.String("GVR", gvr.String()), zap.Error(err))
				time.Sleep(1 * time.Second)
			} else {
				logger.Errorw("Error getting source informer", zap.String("GVR", gvr.String()), zap.Error(err))
				return nil
			}
		}

		scaledobjectInformer := scaledobjectinformer.Get(ctx)

		r := &Reconciler{
			kubeClient:      kubeclient.Get(ctx),
			kedaClient:      kedaclient.Get(ctx),
			sourceInterface: dynamicclient.Get(ctx).Resource(gvr),
			sourceLister:    sourceLister,
			gvk:             gvk,
			gvr:             gvr,
		}
		impl := controller.NewContext(ctx, r, controller.ControllerOptions{
			Logger:        logger,
			WorkQueueName: ReconcilerName,
		})

		logger.Info("Setting up event handlers")
		sourceInformer.AddEventHandler(cache.FilteringResourceEventHandler{
			FilterFunc: pkgreconciler.AnnotationFilterFunc(kedaresources.AutoscalingClassAnnotation, kedaresources.KEDA, false),
			Handler:    controller.HandleAll(impl.Enqueue),
		})

		// don't handle updates on ScaledObject.Status field
		scaledobjectInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{
			FilterFunc: controller.FilterControllerGVK(gvk),
			Handler: cache.ResourceEventHandlerFuncs{
				AddFunc: impl.EnqueueControllerOf,
				UpdateFunc: func(old, new interface{}) {
					if mOld, ok := old.(metav1.Object); ok {
						if mNew, ok := new.(metav1.Object); ok {
							if mNew.GetGeneration() != mOld.GetGeneration() {
								impl.EnqueueControllerOf(new)
							}
						}
					}
				},
				DeleteFunc: impl.EnqueueControllerOf,
			},
		})

		return impl
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"context"
	"fmt"
	"io"
	"time"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	runcontext "github.com/GoogleContainerTools/skaffold/pkg/skaffold/runner/context"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/pkg/errors"
)

// Builder builds docker artifacts on Kubernetes.
type Builder struct {
	*latest.ClusterDetails

	timeout            time.Duration
	insecureRegistries map[string]bool
}

// NewBuilder creates a new Builder that builds artifacts on cluster.
func NewBuilder(runCtx *runcontext.RunContext) (*Builder, error) {
	timeout, err := time.ParseDuration(runCtx.Cfg.Build.Cluster.Timeout)
	if err != nil {
		return nil, errors.Wrap(err, "parsing timeout")
	}

	return &Builder{
		ClusterDetails: runCtx.Cfg.Build.Cluster,
		timeout:        timeout,
	}, nil
}

// Labels are labels specific to cluster builder.
func (b *Builder) Labels() map[string]string {
	return map[string]string{
		constants.Labels.Builder: "cluster",
	}
}

// DependenciesForArtifact returns the Dockerfile dependencies for this artifact
func (b *Builder) DependenciesForArtifact(ctx context.Context, a *latest.Artifact) ([]string, error) {
	var (
		paths []string
		err   error
	)
	switch {
	case a.KanikoArtifact != nil:
		paths, err = docker.GetDependencies(ctx, a.Workspace, a.KanikoArtifact.DockerfilePath, a.KanikoArtifact.BuildArgs, b.insecureRegistries)

	default:
		return nil, fmt.Errorf("undefined artifact type: %+v", a.ArtifactType)
	}

	if err != nil {
		return nil, err
	}
	return util.AbsolutePaths(a.Workspace, paths), nil
}

func (b *Builder) Prune(ctx context.Context, out io.Writer) error {
	return nil
}

func (b *Builder) SyncMap(ctx context.Context, artifact *latest.Artifact) (map[string][]string, error) {
	return nil, build.ErrSyncMapNotSupported{}
}
<commit_msg>cluster builder fails to detect insecure registries<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"context"
	"fmt"
	"io"
	"time"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	runcontext "github.com/GoogleContainerTools/skaffold/pkg/skaffold/runner/context"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/pkg/errors"
)

// Builder builds docker artifacts on Kubernetes.
type Builder struct {
	*latest.ClusterDetails

	timeout            time.Duration
	insecureRegistries map[string]bool
}

// NewBuilder creates a new Builder that builds artifacts on cluster.
func NewBuilder(runCtx *runcontext.RunContext) (*Builder, error) {
	timeout, err := time.ParseDuration(runCtx.Cfg.Build.Cluster.Timeout)
	if err != nil {
		return nil, errors.Wrap(err, "parsing timeout")
	}

	return &Builder{
		ClusterDetails: runCtx.Cfg.Build.Cluster,
		timeout:        timeout,
		insecureRegistries: runCtx.InsecureRegistries,
	}, nil
}

// Labels are labels specific to cluster builder.
func (b *Builder) Labels() map[string]string {
	return map[string]string{
		constants.Labels.Builder: "cluster",
	}
}

// DependenciesForArtifact returns the Dockerfile dependencies for this artifact
func (b *Builder) DependenciesForArtifact(ctx context.Context, a *latest.Artifact) ([]string, error) {
	var (
		paths []string
		err   error
	)
	switch {
	case a.KanikoArtifact != nil:
		paths, err = docker.GetDependencies(ctx, a.Workspace, a.KanikoArtifact.DockerfilePath, a.KanikoArtifact.BuildArgs, b.insecureRegistries)

	default:
		return nil, fmt.Errorf("undefined artifact type: %+v", a.ArtifactType)
	}

	if err != nil {
		return nil, err
	}
	return util.AbsolutePaths(a.Workspace, paths), nil
}

func (b *Builder) Prune(ctx context.Context, out io.Writer) error {
	return nil
}

func (b *Builder) SyncMap(ctx context.Context, artifact *latest.Artifact) (map[string][]string, error) {
	return nil, build.ErrSyncMapNotSupported{}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kaniko

import (
	"context"
	"io"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/tag"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/pkg/errors"
)

// Build builds a list of artifacts with Kaniko.
func (b *Builder) Build(ctx context.Context, out io.Writer, tagger tag.Tagger, artifacts []*latest.Artifact) ([]build.Artifact, error) {
	teardown, err := b.setupSecret(out)
	if err != nil {
		return nil, errors.Wrap(err, "setting up secret")
	}
	defer teardown()

	return build.InParallel(ctx, out, tagger, artifacts, b.buildArtifact)
}

func (b *Builder) buildArtifact(ctx context.Context, out io.Writer, tagger tag.Tagger, artifact *latest.Artifact) (string, error) {
	initialTag, err := b.run(ctx, out, artifact, b.KanikoBuild)
	if err != nil {
		return "", errors.Wrapf(err, "kaniko build for [%s]", artifact.ImageName)
	}

	digest, err := docker.RemoteDigest(initialTag)
	if err != nil {
		return "", errors.Wrap(err, "getting digest")
	}

	tag, err := tagger.GenerateFullyQualifiedImageName(artifact.Workspace, tag.Options{
		ImageName: artifact.ImageName,
		Digest:    digest,
	})
	if err != nil {
		return "", errors.Wrap(err, "generating tag")
	}

	if err := docker.AddTag(initialTag, tag); err != nil {
		return "", errors.Wrap(err, "tagging image")
	}

	return tag, nil
}
<commit_msg>Rename main function<commit_after>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kaniko

import (
	"context"
	"io"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/tag"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/pkg/errors"
)

// Build builds a list of artifacts with Kaniko.
func (b *Builder) Build(ctx context.Context, out io.Writer, tagger tag.Tagger, artifacts []*latest.Artifact) ([]build.Artifact, error) {
	teardown, err := b.setupSecret(out)
	if err != nil {
		return nil, errors.Wrap(err, "setting up secret")
	}
	defer teardown()

	return build.InParallel(ctx, out, tagger, artifacts, b.buildArtifactWithKaniko)
}

func (b *Builder) buildArtifactWithKaniko(ctx context.Context, out io.Writer, tagger tag.Tagger, artifact *latest.Artifact) (string, error) {
	initialTag, err := b.run(ctx, out, artifact, b.KanikoBuild)
	if err != nil {
		return "", errors.Wrapf(err, "kaniko build for [%s]", artifact.ImageName)
	}

	digest, err := docker.RemoteDigest(initialTag)
	if err != nil {
		return "", errors.Wrap(err, "getting digest")
	}

	tag, err := tagger.GenerateFullyQualifiedImageName(artifact.Workspace, tag.Options{
		ImageName: artifact.ImageName,
		Digest:    digest,
	})
	if err != nil {
		return "", errors.Wrap(err, "generating tag")
	}

	if err := docker.AddTag(initialTag, tag); err != nil {
		return "", errors.Wrap(err, "tagging image")
	}

	return tag, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package deploy

import (
	"context"
	"errors"
	"fmt"
	"io"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/deploy/resource"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/event"
	pkgkubernetes "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/runner/runcontext"
)

var (
	defaultStatusCheckDeadline = 2 * time.Minute

	// Poll period for checking set to 100 milliseconds
	defaultPollPeriodInMilliseconds = 100

	// report resource status for pending resources 0.5 second.
	reportStatusTime = 500 * time.Millisecond
)

const (
	tabHeader             = " -"
	kubernetesMaxDeadline = 600
)

type resourceCounter struct {
	deployments *counter
	pods        *counter
}

type counter struct {
	total   int
	pending int32
	failed  int32
}

func StatusCheck(ctx context.Context, defaultLabeller *DefaultLabeller, runCtx *runcontext.RunContext, out io.Writer) error {
	event.StatusCheckEventStarted()
	if err := statusCheck(ctx, defaultLabeller, runCtx, out); err != nil {
		event.StatusCheckEventFailed(err)
		return err
	}
	event.StatusCheckEventSucceeded()
	return nil
}

func statusCheck(ctx context.Context, defaultLabeller *DefaultLabeller, runCtx *runcontext.RunContext, out io.Writer) error {
	client, err := pkgkubernetes.Client()
	if err != nil {
		return fmt.Errorf("getting Kubernetes client: %w", err)
	}

	deployments, err := getDeployments(client, runCtx.Opts.Namespace, defaultLabeller,
		getDeadline(runCtx.Cfg.Deploy.StatusCheckDeadlineSeconds))
	if err != nil {
		return fmt.Errorf("could not fetch deployments: %w", err)
	}
	deadline := statusCheckMaxDeadline(runCtx.Cfg.Deploy.StatusCheckDeadlineSeconds, deployments)

	var wg sync.WaitGroup

	rc := newResourceCounter(len(deployments))

	for _, d := range deployments {
		wg.Add(1)
		go func(r Resource) {
			defer wg.Done()
			pollResourceStatus(ctx, runCtx, r)
			rcCopy := rc.markProcessed(r.Status().Error())
			printStatusCheckSummary(out, r, rcCopy)
		}(d)
	}

	// Retrieve pending resource states
	go func() {
		printResourceStatus(ctx, out, deployments, deadline)
	}()

	// Wait for all deployment status to be fetched
	wg.Wait()
	err = getSkaffoldDeployStatus(rc.deployments)
	return err
}

func getDeployments(client kubernetes.Interface, ns string, l *DefaultLabeller, deadlineDuration time.Duration) ([]Resource, error) {
	deps, err := client.AppsV1().Deployments(ns).List(metav1.ListOptions{
		LabelSelector: l.RunIDSelector(),
	})
	if err != nil {
		return nil, fmt.Errorf("could not fetch deployments: %w", err)
	}

	deployments := make([]Resource, 0, len(deps.Items))
	for _, d := range deps.Items {
		var deadline time.Duration
		if d.Spec.ProgressDeadlineSeconds == nil || *d.Spec.ProgressDeadlineSeconds == kubernetesMaxDeadline {
			deadline = deadlineDuration
		} else {
			deadline = time.Duration(*d.Spec.ProgressDeadlineSeconds) * time.Second
		}
		deployments = append(deployments, resource.NewDeployment(d.Name, d.Namespace, deadline))
	}

	return deployments, nil
}

func pollResourceStatus(ctx context.Context, runCtx *runcontext.RunContext, r Resource) {
	pollDuration := time.Duration(defaultPollPeriodInMilliseconds) * time.Millisecond
	// Add poll duration to account for one last attempt after progressDeadlineSeconds.
	timeoutContext, cancel := context.WithTimeout(ctx, r.Deadline()+pollDuration)
	logrus.Debugf("checking status %s", r)
	defer cancel()
	for {
		select {
		case <-timeoutContext.Done():
			err := fmt.Errorf("could not stabilize within %v: %w", r.Deadline(), timeoutContext.Err())
			r.UpdateStatus(err.Error(), err)
			return
		case <-time.After(pollDuration):
			r.CheckStatus(timeoutContext, runCtx)
			if r.IsStatusCheckComplete() {
				return
			}
		}
	}
}

func getSkaffoldDeployStatus(c *counter) error {
	if c.failed == 0 {
		return nil
	}
	err := fmt.Errorf("%d/%d deployment(s) failed", c.failed, c.total)

	return err
}

func getDeadline(d int) time.Duration {
	if d > 0 {
		return time.Duration(d) * time.Second
	}
	return defaultStatusCheckDeadline
}

func printStatusCheckSummary(out io.Writer, r Resource, rc resourceCounter) {
	err := r.Status().Error()
	if errors.Is(err, context.Canceled) {
		// Don't print the status summary if the user ctrl-Cd
		return
	}

	status := fmt.Sprintf("%s %s", tabHeader, r)
	if err != nil {
		event.ResourceStatusCheckEventFailed(r.String(), err)
		status = fmt.Sprintf("%s failed.%s Error: %s.",
			status,
			trimNewLine(getPendingMessage(rc.deployments.pending, rc.deployments.total)),
			trimNewLine(err.Error()),
		)
	} else {
		event.ResourceStatusCheckEventSucceeded(r.String())
		status = fmt.Sprintf("%s is ready.%s", status, getPendingMessage(rc.deployments.pending, rc.deployments.total))
	}

	fmt.Fprintln(out, status)
}

// Print resource statuses until all status check are completed or context is cancelled.
func printResourceStatus(ctx context.Context, out io.Writer, resources []Resource, deadline time.Duration) {
	timeoutContext, cancel := context.WithTimeout(ctx, deadline)
	defer cancel()
	for {
		var allResourcesCheckComplete bool
		select {
		case <-timeoutContext.Done():
			return
		case <-time.After(reportStatusTime):
			allResourcesCheckComplete = printStatus(resources, out)
		}
		if allResourcesCheckComplete {
			return
		}
	}
}

func printStatus(resources []Resource, out io.Writer) bool {
	allResourcesCheckComplete := true
	for _, r := range resources {
		if r.IsStatusCheckComplete() {
			continue
		}
		allResourcesCheckComplete = false
		if str := r.ReportSinceLastUpdated(); str != "" {
			event.ResourceStatusCheckEventUpdated(r.String(), str)
			fmt.Fprintln(out, tabHeader, trimNewLine(str))
		}
	}
	return allResourcesCheckComplete
}

func getPendingMessage(pending int32, total int) string {
	if pending > 0 {
		return fmt.Sprintf(" [%d/%d deployment(s) still pending]", pending, total)
	}
	return ""
}

func trimNewLine(msg string) string {
	return strings.TrimSuffix(msg, "\n")
}

func newCounter(i int) *counter {
	return &counter{
		total:   i,
		pending: int32(i),
	}
}

func (c *counter) markProcessed(err error) counter {
	if err != nil {
		atomic.AddInt32(&c.failed, 1)
	}
	atomic.AddInt32(&c.pending, -1)
	return c.copy()
}

func (c *counter) copy() counter {
	return counter{
		total:   c.total,
		pending: c.pending,
		failed:  c.failed,
	}
}

func newResourceCounter(d int) *resourceCounter {
	return &resourceCounter{
		deployments: newCounter(d),
		pods:        newCounter(0),
	}
}

func (c *resourceCounter) markProcessed(err error) resourceCounter {
	depCp := c.deployments.markProcessed(err)
	podCp := c.pods.copy()
	return resourceCounter{
		deployments: &depCp,
		pods:        &podCp,
	}
}

func statusCheckMaxDeadline(value int, deployments []Resource) time.Duration {
	if value > 0 {
		return time.Duration(value) * time.Second
	}
	d := time.Duration(0)
	for _, r := range deployments {
		if r.Deadline() > d {
			d = r.Deadline()
		}
	}
	return d
}
<commit_msg>revert return<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package deploy

import (
	"context"
	"errors"
	"fmt"
	"io"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/deploy/resource"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/event"
	pkgkubernetes "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/runner/runcontext"
)

var (
	defaultStatusCheckDeadline = 2 * time.Minute

	// Poll period for checking set to 100 milliseconds
	defaultPollPeriodInMilliseconds = 100

	// report resource status for pending resources 0.5 second.
	reportStatusTime = 500 * time.Millisecond
)

const (
	tabHeader             = " -"
	kubernetesMaxDeadline = 600
)

type resourceCounter struct {
	deployments *counter
	pods        *counter
}

type counter struct {
	total   int
	pending int32
	failed  int32
}

func StatusCheck(ctx context.Context, defaultLabeller *DefaultLabeller, runCtx *runcontext.RunContext, out io.Writer) error {
	event.StatusCheckEventStarted()
	if err := statusCheck(ctx, defaultLabeller, runCtx, out); err != nil {
		event.StatusCheckEventFailed(err)
		return err
	}
	event.StatusCheckEventSucceeded()
	return nil
}

func statusCheck(ctx context.Context, defaultLabeller *DefaultLabeller, runCtx *runcontext.RunContext, out io.Writer) error {
	client, err := pkgkubernetes.Client()
	if err != nil {
		return fmt.Errorf("getting Kubernetes client: %w", err)
	}

	deployments, err := getDeployments(client, runCtx.Opts.Namespace, defaultLabeller,
		getDeadline(runCtx.Cfg.Deploy.StatusCheckDeadlineSeconds))
	if err != nil {
		return fmt.Errorf("could not fetch deployments: %w", err)
	}
	deadline := statusCheckMaxDeadline(runCtx.Cfg.Deploy.StatusCheckDeadlineSeconds, deployments)

	var wg sync.WaitGroup

	rc := newResourceCounter(len(deployments))

	for _, d := range deployments {
		wg.Add(1)
		go func(r Resource) {
			defer wg.Done()
			pollResourceStatus(ctx, runCtx, r)
			rcCopy := rc.markProcessed(r.Status().Error())
			printStatusCheckSummary(out, r, rcCopy)
		}(d)
	}

	// Retrieve pending resource states
	go func() {
		printResourceStatus(ctx, out, deployments, deadline)
	}()

	// Wait for all deployment status to be fetched
	wg.Wait()
	return getSkaffoldDeployStatus(rc.deployments)
}

func getDeployments(client kubernetes.Interface, ns string, l *DefaultLabeller, deadlineDuration time.Duration) ([]Resource, error) {
	deps, err := client.AppsV1().Deployments(ns).List(metav1.ListOptions{
		LabelSelector: l.RunIDSelector(),
	})
	if err != nil {
		return nil, fmt.Errorf("could not fetch deployments: %w", err)
	}

	deployments := make([]Resource, 0, len(deps.Items))
	for _, d := range deps.Items {
		var deadline time.Duration
		if d.Spec.ProgressDeadlineSeconds == nil || *d.Spec.ProgressDeadlineSeconds == kubernetesMaxDeadline {
			deadline = deadlineDuration
		} else {
			deadline = time.Duration(*d.Spec.ProgressDeadlineSeconds) * time.Second
		}
		deployments = append(deployments, resource.NewDeployment(d.Name, d.Namespace, deadline))
	}

	return deployments, nil
}

func pollResourceStatus(ctx context.Context, runCtx *runcontext.RunContext, r Resource) {
	pollDuration := time.Duration(defaultPollPeriodInMilliseconds) * time.Millisecond
	// Add poll duration to account for one last attempt after progressDeadlineSeconds.
	timeoutContext, cancel := context.WithTimeout(ctx, r.Deadline()+pollDuration)
	logrus.Debugf("checking status %s", r)
	defer cancel()
	for {
		select {
		case <-timeoutContext.Done():
			err := fmt.Errorf("could not stabilize within %v: %w", r.Deadline(), timeoutContext.Err())
			r.UpdateStatus(err.Error(), err)
			return
		case <-time.After(pollDuration):
			r.CheckStatus(timeoutContext, runCtx)
			if r.IsStatusCheckComplete() {
				return
			}
		}
	}
}

func getSkaffoldDeployStatus(c *counter) error {
	if c.failed == 0 {
		return nil
	}
	err := fmt.Errorf("%d/%d deployment(s) failed", c.failed, c.total)

	return err
}

func getDeadline(d int) time.Duration {
	if d > 0 {
		return time.Duration(d) * time.Second
	}
	return defaultStatusCheckDeadline
}

func printStatusCheckSummary(out io.Writer, r Resource, rc resourceCounter) {
	err := r.Status().Error()
	if errors.Is(err, context.Canceled) {
		// Don't print the status summary if the user ctrl-Cd
		return
	}

	status := fmt.Sprintf("%s %s", tabHeader, r)
	if err != nil {
		event.ResourceStatusCheckEventFailed(r.String(), err)
		status = fmt.Sprintf("%s failed.%s Error: %s.",
			status,
			trimNewLine(getPendingMessage(rc.deployments.pending, rc.deployments.total)),
			trimNewLine(err.Error()),
		)
	} else {
		event.ResourceStatusCheckEventSucceeded(r.String())
		status = fmt.Sprintf("%s is ready.%s", status, getPendingMessage(rc.deployments.pending, rc.deployments.total))
	}

	fmt.Fprintln(out, status)
}

// Print resource statuses until all status check are completed or context is cancelled.
func printResourceStatus(ctx context.Context, out io.Writer, resources []Resource, deadline time.Duration) {
	timeoutContext, cancel := context.WithTimeout(ctx, deadline)
	defer cancel()
	for {
		var allResourcesCheckComplete bool
		select {
		case <-timeoutContext.Done():
			return
		case <-time.After(reportStatusTime):
			allResourcesCheckComplete = printStatus(resources, out)
		}
		if allResourcesCheckComplete {
			return
		}
	}
}

func printStatus(resources []Resource, out io.Writer) bool {
	allResourcesCheckComplete := true
	for _, r := range resources {
		if r.IsStatusCheckComplete() {
			continue
		}
		allResourcesCheckComplete = false
		if str := r.ReportSinceLastUpdated(); str != "" {
			event.ResourceStatusCheckEventUpdated(r.String(), str)
			fmt.Fprintln(out, tabHeader, trimNewLine(str))
		}
	}
	return allResourcesCheckComplete
}

func getPendingMessage(pending int32, total int) string {
	if pending > 0 {
		return fmt.Sprintf(" [%d/%d deployment(s) still pending]", pending, total)
	}
	return ""
}

func trimNewLine(msg string) string {
	return strings.TrimSuffix(msg, "\n")
}

func newCounter(i int) *counter {
	return &counter{
		total:   i,
		pending: int32(i),
	}
}

func (c *counter) markProcessed(err error) counter {
	if err != nil {
		atomic.AddInt32(&c.failed, 1)
	}
	atomic.AddInt32(&c.pending, -1)
	return c.copy()
}

func (c *counter) copy() counter {
	return counter{
		total:   c.total,
		pending: c.pending,
		failed:  c.failed,
	}
}

func newResourceCounter(d int) *resourceCounter {
	return &resourceCounter{
		deployments: newCounter(d),
		pods:        newCounter(0),
	}
}

func (c *resourceCounter) markProcessed(err error) resourceCounter {
	depCp := c.deployments.markProcessed(err)
	podCp := c.pods.copy()
	return resourceCounter{
		deployments: &depCp,
		pods:        &podCp,
	}
}

func statusCheckMaxDeadline(value int, deployments []Resource) time.Duration {
	if value > 0 {
		return time.Duration(value) * time.Second
	}
	d := time.Duration(0)
	for _, r := range deployments {
		if r.Deadline() > d {
			d = r.Deadline()
		}
	}
	return d
}
<|endoftext|>


<commit_before>package konnectors

import (
	"archive/tar"
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"runtime"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/cozy/cozy-stack/pkg/stack"
	"github.com/cozy/cozy-stack/pkg/workers/mails"
	"github.com/spf13/afero"
)

func init() {
	jobs.AddWorker("konnector", &jobs.WorkerConfig{
		Concurrency:  runtime.NumCPU(),
		MaxExecCount: 2,
		MaxExecTime:  120 * time.Second,
		Timeout:      60 * time.Second,
		WorkerFunc:   Worker,
		WorkerCommit: commit,
	})
}

// Options contains the options to execute a konnector.
type Options struct {
	Konnector    string `json:"konnector"`
	Account      string `json:"account"`
	FolderToSave string `json:"folder_to_save"`
}

// result stores the result of a konnector execution.
type result struct {
	DocID     string    `json:"_id,omitempty"`
	DocRev    string    `json:"_rev,omitempty"`
	CreatedAt time.Time `json:"last_execution"`
	State     string    `json:"state"`
	Error     string    `json:"error"`
}

func (r *result) ID() string         { return r.DocID }
func (r *result) Rev() string        { return r.DocRev }
func (r *result) DocType() string    { return consts.KonnectorResults }
func (r *result) Clone() couchdb.Doc { return r }
func (r *result) SetID(id string)    { r.DocID = id }
func (r *result) SetRev(rev string)  { r.DocRev = rev }

const konnectorMsgTypeError string = "error"
const konnectorMsgTypeDebug string = "debug"
const konnectorMsgTypeWarning string = "warning"
const konnectorMsgTypeProgress string = "progress"

type konnectorMsg struct {
	Type    string `json:"type"`
	Message string `json:"message"`
}

// Worker is the worker that runs a konnector by executing an external process.
func Worker(ctx context.Context, m *jobs.Message) error {
	opts := &Options{}
	if err := m.Unmarshal(&opts); err != nil {
		return err
	}

	slug := opts.Konnector
	fields := struct {
		Account      string `json:"account"`
		FolderToSave string `json:"folder_to_save"`
	}{
		Account:      opts.Account,
		FolderToSave: opts.FolderToSave,
	}
	domain := ctx.Value(jobs.ContextDomainKey).(string)
	worker := ctx.Value(jobs.ContextWorkerKey).(string)
	jobID := fmt.Sprintf("%s/%s/%s", worker, slug, domain)

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	man, err := apps.GetKonnectorBySlug(inst, slug)
	if err != nil {
		return err
	}
	if man.State() != apps.Ready {
		return errors.New("Konnector is not ready")
	}

	token := inst.BuildKonnectorToken(man)

	osFS := afero.NewOsFs()
	workDir, err := afero.TempDir(osFS, "", "konnector-"+slug)
	if err != nil {
		return err
	}
	defer osFS.RemoveAll(workDir)
	workFS := afero.NewBasePathFs(osFS, workDir)

	fileServer := inst.KonnectorsFileServer()
	tarFile, err := fileServer.Open(slug, man.Version(), apps.KonnectorArchiveName)
	if err != nil {
		return err
	}

	tr := tar.NewReader(tarFile)
	for {
		var hdr *tar.Header
		hdr, err = tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		dirname := path.Dir(hdr.Name)
		if dirname != "." {
			if err = workFS.MkdirAll(dirname, 0755); err != nil {
				return nil
			}
		}
		var f afero.File
		f, err = workFS.OpenFile(hdr.Name, os.O_CREATE|os.O_WRONLY, os.FileMode(hdr.Mode))
		if err != nil {
			return err
		}
		_, err = io.Copy(f, tr)
		if err != nil {
			return err
		}
	}

	fieldsJSON, err := json.Marshal(fields)
	if err != nil {
		return err
	}

	konnCmd := config.GetConfig().Konnectors.Cmd
	cmd := exec.CommandContext(ctx, konnCmd, workDir) // #nosec
	cmd.Env = []string{
		"COZY_URL=" + inst.PageURL("/", nil),
		"COZY_CREDENTIALS=" + token,
		"COZY_FIELDS=" + string(fieldsJSON),
		"COZY_TYPE=" + man.Type,
		"COZY_JOB_ID=" + jobID,
	}

	cmdErr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}
	cmdOut, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	scanErr := bufio.NewScanner(cmdErr)
	scanOut := bufio.NewScanner(cmdOut)
	scanOut.Buffer(nil, 256*1024)

	var msgChan = make(chan konnectorMsg)
	var messages []konnectorMsg

	go doScanOut(jobID, scanOut, domain, msgChan)
	go doScanErr(jobID, scanErr)
	go func() {
		hub := realtime.GetHub()
		for msg := range msgChan {
			messages = append(messages, msg)
			hub.Publish(&realtime.Event{
				Type: realtime.EventCreate,
				Doc: couchdb.JSONDoc{Type: consts.JobEvents, M: map[string]interface{}{
					"type":    msg.Type,
					"message": msg.Message,
				}},
				Domain: domain,
			})
		}
	}()

	if err = cmd.Start(); err != nil {
		return wrapErr(ctx, err)
	}
	if err = cmd.Wait(); err != nil {
		return wrapErr(ctx, err)
	}
	for _, msg := range messages {
		if msg.Type == konnectorMsgTypeError {
			err = errors.New(msg.Message)
			return err
		}
	}
	close(msgChan)
	return nil
}

func doScanOut(jobID string, scanner *bufio.Scanner, domain string, msgs chan konnectorMsg) {
	for scanner.Scan() {
		linebb := scanner.Bytes()
		from := bytes.IndexByte(linebb, '{')
		to := bytes.LastIndexByte(linebb, '}')
		var msg konnectorMsg
		log.Infof("[konnector] %s: Stdout: %s", jobID, string(linebb[from:to+1]))
		if from > -1 && to > -1 {
			err := json.Unmarshal(linebb[from:to+1], &msg)
			if err == nil {
				msgs <- msg
				continue
			}
		}
		log.Warnf("[konnector] %s: Could not parse as JSON", jobID)
	}
	if err := scanner.Err(); err != nil {
		log.Errorf("[konnector] %s: Error while reading stdout: %s", jobID, err)
	}
}

func doScanErr(jobID string, scanner *bufio.Scanner) {
	for scanner.Scan() {
		log.Errorf("[konnector] %s: Stderr: %s", jobID, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		log.Errorf("[konnector] %s: Error while reading stderr: %s", jobID, err)
	}
}

func commit(ctx context.Context, m *jobs.Message, errjob error) error {
	opts := &Options{}
	if err := m.Unmarshal(&opts); err != nil {
		return err
	}

	slug := opts.Konnector
	domain := ctx.Value(jobs.ContextDomainKey).(string)

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	lastResult := &result{}
	err = couchdb.GetDoc(inst, consts.KonnectorResults, slug, lastResult)
	if err != nil {
		if !couchdb.IsNotFoundError(err) {
			return err
		}
		lastResult = nil
	}

	var state, errstr string
	if errjob != nil {
		errstr = errjob.Error()
		state = jobs.Errored
	} else {
		state = jobs.Done
	}
	result := &result{
		DocID:     slug,
		CreatedAt: time.Now(),
		State:     state,
		Error:     errstr,
	}
	if lastResult == nil {
		err = couchdb.CreateNamedDocWithDB(inst, result)
	} else {
		result.SetRev(lastResult.Rev())
		err = couchdb.UpdateDoc(inst, result)
	}
	if err != nil {
		return err
	}

	// if it is the first try we do not take into account an error, we bail.
	if lastResult == nil {
		return nil
	}
	// if the job has not errored, or the last one was already errored, we bail.
	if state != jobs.Errored || lastResult.State == jobs.Errored {
		return nil
	}

	msg, err := jobs.NewMessage(jobs.JSONEncoding, &mails.Options{
		Mode:           mails.ModeNoReply,
		Subject:        inst.Translate("Konnector execution error"),
		TemplateName:   "konnector_error_" + inst.Locale,
		TemplateValues: map[string]string{"KonnectorName": slug},
	})
	if err != nil {
		return err
	}
	_, err = stack.GetBroker().PushJob(&jobs.JobRequest{
		Domain:     domain,
		WorkerType: "sendmail",
		Message:    msg,
	})
	return err
}

func wrapErr(ctx context.Context, err error) error {
	if ctx.Err() == context.DeadlineExceeded {
		return context.DeadlineExceeded
	}
	return err
}
<commit_msg>gometalinter<commit_after>package konnectors

import (
	"archive/tar"
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"runtime"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/cozy/cozy-stack/pkg/stack"
	"github.com/cozy/cozy-stack/pkg/workers/mails"
	"github.com/spf13/afero"
)

func init() {
	jobs.AddWorker("konnector", &jobs.WorkerConfig{
		Concurrency:  runtime.NumCPU(),
		MaxExecCount: 2,
		MaxExecTime:  120 * time.Second,
		Timeout:      60 * time.Second,
		WorkerFunc:   Worker,
		WorkerCommit: commit,
	})
}

// Options contains the options to execute a konnector.
type Options struct {
	Konnector    string `json:"konnector"`
	Account      string `json:"account"`
	FolderToSave string `json:"folder_to_save"`
}

// result stores the result of a konnector execution.
type result struct {
	DocID     string    `json:"_id,omitempty"`
	DocRev    string    `json:"_rev,omitempty"`
	CreatedAt time.Time `json:"last_execution"`
	State     string    `json:"state"`
	Error     string    `json:"error"`
}

func (r *result) ID() string         { return r.DocID }
func (r *result) Rev() string        { return r.DocRev }
func (r *result) DocType() string    { return consts.KonnectorResults }
func (r *result) Clone() couchdb.Doc { return r }
func (r *result) SetID(id string)    { r.DocID = id }
func (r *result) SetRev(rev string)  { r.DocRev = rev }

const konnectorMsgTypeError string = "error"

// const konnectorMsgTypeDebug string = "debug"
// const konnectorMsgTypeWarning string = "warning"
// const konnectorMsgTypeProgress string = "progress"

type konnectorMsg struct {
	Type    string `json:"type"`
	Message string `json:"message"`
}

// Worker is the worker that runs a konnector by executing an external process.
func Worker(ctx context.Context, m *jobs.Message) error {
	opts := &Options{}
	if err := m.Unmarshal(&opts); err != nil {
		return err
	}

	slug := opts.Konnector
	fields := struct {
		Account      string `json:"account"`
		FolderToSave string `json:"folder_to_save"`
	}{
		Account:      opts.Account,
		FolderToSave: opts.FolderToSave,
	}
	domain := ctx.Value(jobs.ContextDomainKey).(string)
	worker := ctx.Value(jobs.ContextWorkerKey).(string)
	jobID := fmt.Sprintf("%s/%s/%s", worker, slug, domain)

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	man, err := apps.GetKonnectorBySlug(inst, slug)
	if err != nil {
		return err
	}
	if man.State() != apps.Ready {
		return errors.New("Konnector is not ready")
	}

	token := inst.BuildKonnectorToken(man)

	osFS := afero.NewOsFs()
	workDir, err := afero.TempDir(osFS, "", "konnector-"+slug)
	if err != nil {
		return err
	}
	defer osFS.RemoveAll(workDir)
	workFS := afero.NewBasePathFs(osFS, workDir)

	fileServer := inst.KonnectorsFileServer()
	tarFile, err := fileServer.Open(slug, man.Version(), apps.KonnectorArchiveName)
	if err != nil {
		return err
	}

	tr := tar.NewReader(tarFile)
	for {
		var hdr *tar.Header
		hdr, err = tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		dirname := path.Dir(hdr.Name)
		if dirname != "." {
			if err = workFS.MkdirAll(dirname, 0755); err != nil {
				return nil
			}
		}
		var f afero.File
		f, err = workFS.OpenFile(hdr.Name, os.O_CREATE|os.O_WRONLY, os.FileMode(hdr.Mode))
		if err != nil {
			return err
		}
		_, err = io.Copy(f, tr)
		if err != nil {
			return err
		}
	}

	fieldsJSON, err := json.Marshal(fields)
	if err != nil {
		return err
	}

	konnCmd := config.GetConfig().Konnectors.Cmd
	cmd := exec.CommandContext(ctx, konnCmd, workDir) // #nosec
	cmd.Env = []string{
		"COZY_URL=" + inst.PageURL("/", nil),
		"COZY_CREDENTIALS=" + token,
		"COZY_FIELDS=" + string(fieldsJSON),
		"COZY_TYPE=" + man.Type,
		"COZY_JOB_ID=" + jobID,
	}

	cmdErr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}
	cmdOut, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	scanErr := bufio.NewScanner(cmdErr)
	scanOut := bufio.NewScanner(cmdOut)
	scanOut.Buffer(nil, 256*1024)

	var msgChan = make(chan konnectorMsg)
	var messages []konnectorMsg

	go doScanOut(jobID, scanOut, domain, msgChan)
	go doScanErr(jobID, scanErr)
	go func() {
		hub := realtime.GetHub()
		for msg := range msgChan {
			messages = append(messages, msg)
			hub.Publish(&realtime.Event{
				Type: realtime.EventCreate,
				Doc: couchdb.JSONDoc{Type: consts.JobEvents, M: map[string]interface{}{
					"type":    msg.Type,
					"message": msg.Message,
				}},
				Domain: domain,
			})
		}
	}()

	if err = cmd.Start(); err != nil {
		return wrapErr(ctx, err)
	}
	if err = cmd.Wait(); err != nil {
		return wrapErr(ctx, err)
	}
	for _, msg := range messages {
		if msg.Type == konnectorMsgTypeError {
			err = errors.New(msg.Message)
			return err
		}
	}
	close(msgChan)
	return nil
}

func doScanOut(jobID string, scanner *bufio.Scanner, domain string, msgs chan konnectorMsg) {
	for scanner.Scan() {
		linebb := scanner.Bytes()
		from := bytes.IndexByte(linebb, '{')
		to := bytes.LastIndexByte(linebb, '}')
		var msg konnectorMsg
		log.Infof("[konnector] %s: Stdout: %s", jobID, string(linebb[from:to+1]))
		if from > -1 && to > -1 {
			err := json.Unmarshal(linebb[from:to+1], &msg)
			if err == nil {
				msgs <- msg
				continue
			}
		}
		log.Warnf("[konnector] %s: Could not parse as JSON", jobID)
	}
	if err := scanner.Err(); err != nil {
		log.Errorf("[konnector] %s: Error while reading stdout: %s", jobID, err)
	}
}

func doScanErr(jobID string, scanner *bufio.Scanner) {
	for scanner.Scan() {
		log.Errorf("[konnector] %s: Stderr: %s", jobID, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		log.Errorf("[konnector] %s: Error while reading stderr: %s", jobID, err)
	}
}

func commit(ctx context.Context, m *jobs.Message, errjob error) error {
	opts := &Options{}
	if err := m.Unmarshal(&opts); err != nil {
		return err
	}

	slug := opts.Konnector
	domain := ctx.Value(jobs.ContextDomainKey).(string)

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	lastResult := &result{}
	err = couchdb.GetDoc(inst, consts.KonnectorResults, slug, lastResult)
	if err != nil {
		if !couchdb.IsNotFoundError(err) {
			return err
		}
		lastResult = nil
	}

	var state, errstr string
	if errjob != nil {
		errstr = errjob.Error()
		state = jobs.Errored
	} else {
		state = jobs.Done
	}
	result := &result{
		DocID:     slug,
		CreatedAt: time.Now(),
		State:     state,
		Error:     errstr,
	}
	if lastResult == nil {
		err = couchdb.CreateNamedDocWithDB(inst, result)
	} else {
		result.SetRev(lastResult.Rev())
		err = couchdb.UpdateDoc(inst, result)
	}
	if err != nil {
		return err
	}

	// if it is the first try we do not take into account an error, we bail.
	if lastResult == nil {
		return nil
	}
	// if the job has not errored, or the last one was already errored, we bail.
	if state != jobs.Errored || lastResult.State == jobs.Errored {
		return nil
	}

	msg, err := jobs.NewMessage(jobs.JSONEncoding, &mails.Options{
		Mode:           mails.ModeNoReply,
		Subject:        inst.Translate("Konnector execution error"),
		TemplateName:   "konnector_error_" + inst.Locale,
		TemplateValues: map[string]string{"KonnectorName": slug},
	})
	if err != nil {
		return err
	}
	_, err = stack.GetBroker().PushJob(&jobs.JobRequest{
		Domain:     domain,
		WorkerType: "sendmail",
		Message:    msg,
	})
	return err
}

func wrapErr(ctx context.Context, err error) error {
	if ctx.Err() == context.DeadlineExceeded {
		return context.DeadlineExceeded
	}
	return err
}
<|endoftext|>


<commit_before>package gopenid

import (
	"crypto/sha1"
	"crypto/sha256"
	"errors"
	"github.com/nu7hatch/gouuid"
	"hash"
	"io"
	"time"
)

const (
	ASSOCIATION_LIFETIME = 60 * 60 * 24 * 1
)

var (
	ErrGeneratingAssociationFailed = errors.New("generating association failed")
	ErrAssociationNotFound         = errors.New("association not found")

	ASSOC_HMAC_SHA1 = AssocType{
		name:       "HMAC-SHA1",
		hashFunc:   sha1.New,
		secretSize: sha1.Size,
	}
	ASSOC_HMAC_SHA256 = AssocType{
		name:       "HMAC-SHA256",
		hashFunc:   sha256.New,
		secretSize: sha256.Size,
	}

	SESSION_DH_SHA1 = SessionType{
		name: "DH-SHA1",
		assocTypes: []AssocType{
			ASSOC_HMAC_SHA1,
		},
	}
	SESSION_DH_SHA256 = SessionType{
		name: "DH-SHA256",
		assocTypes: []AssocType{
			ASSOC_HMAC_SHA256,
		},
	}
	SESSION_NO_ENCRYPTION = SessionType{
		name: "no-encryption",
		assocTypes: []AssocType{
			ASSOC_HMAC_SHA1,
			ASSOC_HMAC_SHA256,
		},
	}
)

type AssocType struct {
	name       string
	hashFunc   func() hash.Hash
	secretSize int
}

func (t *AssocType) Name() string {
	return t.name
}

func (t *AssocType) GetSecretSize() int {
	return t.secretSize
}

type SessionType struct {
	name       string
	assocTypes []AssocType
}

func (t *SessionType) Name() string {
	return t.name
}

type Association struct {
	assocType   AssocType
	handle      string
	secret      []byte
	expires     int64
	isStateless bool
}

func NewAssociation(assocType AssocType, handle string, secret []byte, expires int64, isStateless bool) *Association {
	if expires < 1 {
		expires = time.Now().Unix() + ASSOCIATION_LIFETIME
	}

	return &Association{
		assocType:   assocType,
		handle:      handle,
		secret:      secret,
		expires:     expires,
		isStateless: isStateless,
	}
}

func CreateAssociation(random io.Reader, assocType AssocType, expires int64, isStateless bool) (assoc *Association, err error) {
	handle, err := uuid.NewV4()
	if err != nil {
		err = ErrGeneratingAssociationFailed
		return
	}

	secret := make([]byte, assocType.GetSecretSize())
	_, err = io.ReadFull(random, secret)
	if err != nil {
		err = ErrGeneratingAssociationFailed
		return
	}

	assoc = NewAssociation(assocType, handle.String(), secret, expires, isStateless)
	return
}

func (assoc *Association) GetAssocType() AssocType {
	return assoc.assocType
}

func (assoc *Association) GetHandle() string {
	return assoc.handle
}

func (assoc *Association) GetSecret() []byte {
	return assoc.secret
}

func (assoc *Association) GetExpires() int64 {
	return assoc.expires
}

func (assoc *Association) IsValid() bool {
	return time.Now().Before(time.Unix(assoc.GetExpires(), 0))
}

func (assoc *Association) IsStateless() bool {
	return assoc.isStateless
}
<commit_msg>add GetAssocTypeByName, GetSessionTypeByName<commit_after>package gopenid

import (
	"crypto/sha1"
	"crypto/sha256"
	"errors"
	"github.com/nu7hatch/gouuid"
	"hash"
	"io"
	"time"
)

const (
	ASSOCIATION_LIFETIME = 60 * 60 * 24 * 1
)

var (
	ErrGeneratingAssociationFailed = errors.New("generating association failed")
	ErrAssociationNotFound         = errors.New("association not found")
	ErrUnknownSessionType          = errors.New("unknown session type")
	ErrUnknownAssocType            = errors.New("unknown association type")

	ASSOC_HMAC_SHA1 = AssocType{
		name:       "HMAC-SHA1",
		hashFunc:   sha1.New,
		secretSize: sha1.Size,
	}
	ASSOC_HMAC_SHA256 = AssocType{
		name:       "HMAC-SHA256",
		hashFunc:   sha256.New,
		secretSize: sha256.Size,
	}

	SESSION_DH_SHA1 = SessionType{
		name: "DH-SHA1",
		assocTypes: []AssocType{
			ASSOC_HMAC_SHA1,
		},
	}
	SESSION_DH_SHA256 = SessionType{
		name: "DH-SHA256",
		assocTypes: []AssocType{
			ASSOC_HMAC_SHA256,
		},
	}
	SESSION_NO_ENCRYPTION = SessionType{
		name: "no-encryption",
		assocTypes: []AssocType{
			ASSOC_HMAC_SHA1,
			ASSOC_HMAC_SHA256,
		},
	}
)

type AssocType struct {
	name       string
	hashFunc   func() hash.Hash
	secretSize int
}

func (t *AssocType) Name() string {
	return t.name
}

func (t *AssocType) GetSecretSize() int {
	return t.secretSize
}

func GetAssocTypeByName(name string) (assocType AssocType, err error) {
	switch name {
	case "HMAC-SHA1":
		assocType = ASSOC_HMAC_SHA1
	case "HMAC-SHA256":
		assocType = ASSOC_HMAC_SHA256
	default:
		err = ErrUnknownAssocType
	}

	return
}

type SessionType struct {
	name       string
	assocTypes []AssocType
}

func (t *SessionType) Name() string {
	return t.name
}

func GetSessionTypeByName(name string) (sessionType SessionType, err error) {
	switch name {
	case "no-encryption":
		sessionType = SESSION_NO_ENCRYPTION
	case "DH-SHA1":
		sessionType = SESSION_DH_SHA1
	case "DH-SHA256":
		sessionType = SESSION_DH_SHA256
	default:
		err = ErrUnknownSessionType
	}

	return
}

type Association struct {
	assocType   AssocType
	handle      string
	secret      []byte
	expires     int64
	isStateless bool
}

func NewAssociation(assocType AssocType, handle string, secret []byte, expires int64, isStateless bool) *Association {
	if expires < 1 {
		expires = time.Now().Unix() + ASSOCIATION_LIFETIME
	}

	return &Association{
		assocType:   assocType,
		handle:      handle,
		secret:      secret,
		expires:     expires,
		isStateless: isStateless,
	}
}

func CreateAssociation(random io.Reader, assocType AssocType, expires int64, isStateless bool) (assoc *Association, err error) {
	handle, err := uuid.NewV4()
	if err != nil {
		err = ErrGeneratingAssociationFailed
		return
	}

	secret := make([]byte, assocType.GetSecretSize())
	_, err = io.ReadFull(random, secret)
	if err != nil {
		err = ErrGeneratingAssociationFailed
		return
	}

	assoc = NewAssociation(assocType, handle.String(), secret, expires, isStateless)
	return
}

func (assoc *Association) GetAssocType() AssocType {
	return assoc.assocType
}

func (assoc *Association) GetHandle() string {
	return assoc.handle
}

func (assoc *Association) GetSecret() []byte {
	return assoc.secret
}

func (assoc *Association) GetExpires() int64 {
	return assoc.expires
}

func (assoc *Association) IsValid() bool {
	return time.Now().Before(time.Unix(assoc.GetExpires(), 0))
}

func (assoc *Association) IsStateless() bool {
	return assoc.isStateless
}
<|endoftext|>


<commit_before>package disk

import (
	"fmt"
	"strconv"
	"strings"

	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"
	boshsys "github.com/cloudfoundry/bosh-utils/system"
)

type sfdiskPartitioner struct {
	logger    boshlog.Logger
	cmdRunner boshsys.CmdRunner
	logTag    string
}

func NewSfdiskPartitioner(logger boshlog.Logger, cmdRunner boshsys.CmdRunner) Partitioner {
	return sfdiskPartitioner{
		logger:    logger,
		cmdRunner: cmdRunner,
		logTag:    "SfdiskPartitioner",
	}
}

func (p sfdiskPartitioner) Partition(devicePath string, partitions []Partition) error {
	if p.diskMatchesPartitions(devicePath, partitions) {
		p.logger.Info(p.logTag, "%s already partitioned as expected, skipping", devicePath)
		return nil
	}

	sfdiskPartitionTypes := map[PartitionType]string{
		PartitionTypeSwap:  "S",
		PartitionTypeLinux: "L",
	}

	sfdiskInput := ""
	for index, partition := range partitions {
		sfdiskPartitionType := sfdiskPartitionTypes[partition.Type]
		partitionSize := fmt.Sprintf("%d", p.convertFromBytesToMb(partition.SizeInBytes))

		if index == len(partitions)-1 {
			partitionSize = ""
		}

		sfdiskInput = sfdiskInput + fmt.Sprintf(",%s,%s\n", partitionSize, sfdiskPartitionType)
	}
	p.logger.Info(p.logTag, "Partitioning %s with %s", devicePath, sfdiskInput)

	_, _, _, err := p.cmdRunner.RunCommandWithInput(sfdiskInput, "sfdisk", "-uM", devicePath)
	if err != nil {
		return bosherr.WrapError(err, "Shelling out to sfdisk")
	}

	return nil
}

func (p sfdiskPartitioner) GetDeviceSizeInBytes(devicePath string) (uint64, error) {
	stdout, _, _, err := p.cmdRunner.RunCommand("sfdisk", "-s", devicePath)
	if err != nil {
		return 0, bosherr.WrapError(err, "Shelling out to sfdisk")
	}

	sizeInKb, err := strconv.ParseUint(strings.Trim(stdout, "\n"), 10, 64)
	if err != nil {
		return 0, bosherr.WrapError(err, "Converting disk size to integer")
	}

	return p.convertFromKbToBytes(sizeInKb), nil
}

func (p sfdiskPartitioner) diskMatchesPartitions(devicePath string, partitionsToMatch []Partition) (result bool) {
	existingPartitions, err := p.getPartitions(devicePath)
	if err != nil {
		err = bosherr.WrapErrorf(err, "Getting partitions for %s", devicePath)
		return
	}

	if len(existingPartitions) < len(partitionsToMatch) {
		return
	}

	remainingDiskSpace, err := p.GetDeviceSizeInBytes(devicePath)
	if err != nil {
		err = bosherr.WrapErrorf(err, "Getting device size for %s", devicePath)
		return
	}

	for index, partitionToMatch := range partitionsToMatch {
		if index == len(partitionsToMatch)-1 {
			partitionToMatch.SizeInBytes = remainingDiskSpace
		}

		existingPartition := existingPartitions[index]
		switch {
		case existingPartition.Type != partitionToMatch.Type:
			return
		case !withinDelta(existingPartition.SizeInBytes, partitionToMatch.SizeInBytes, p.convertFromMbToBytes(20)):
			return
		}

		remainingDiskSpace = remainingDiskSpace - partitionToMatch.SizeInBytes
	}

	return true
}

func (p sfdiskPartitioner) getPartitions(devicePath string) (partitions []Partition, err error) {
	stdout, _, _, err := p.cmdRunner.RunCommand("sfdisk", "-d", devicePath)
	if err != nil {
		err = bosherr.WrapError(err, "Shelling out to sfdisk")
		return
	}

	allLines := strings.Split(stdout, "\n")
	if len(allLines) < 4 {
		return
	}

	partitionLines := allLines[3 : len(allLines)-1]

	for _, partitionLine := range partitionLines {
		partitionPath, partitionType := extractPartitionPathAndType(partitionLine)
		partition := Partition{Type: partitionType}

		if partition.Type != PartitionTypeEmpty {
			size, err := p.GetDeviceSizeInBytes(partitionPath)
			if err == nil {
				partition.SizeInBytes = size
			}
		}

		partitions = append(partitions, partition)
	}
	return
}

var partitionTypesMap = map[string]PartitionType{
	"82": PartitionTypeSwap,
	"83": PartitionTypeLinux,
	"0":  PartitionTypeEmpty,
}

func extractPartitionPathAndType(line string) (partitionPath string, partitionType PartitionType) {
	partitionFields := strings.Fields(line)
	lastField := partitionFields[len(partitionFields)-1]

	sfdiskPartitionType := strings.Replace(lastField, "Id=", "", 1)

	partitionPath = partitionFields[0]
	partitionType = partitionTypesMap[sfdiskPartitionType]
	return
}

func (p sfdiskPartitioner) convertFromBytesToMb(sizeInBytes uint64) uint64 {
	return sizeInBytes / (1024 * 1024)
}

func (p sfdiskPartitioner) convertFromMbToBytes(sizeInMb uint64) uint64 {
	return sizeInMb * 1024 * 1024
}

func (p sfdiskPartitioner) convertFromKbToBytes(sizeInKb uint64) uint64 {
	return sizeInKb * 1024
}
<commit_msg>Retry to format disk when an error occurs.<commit_after>package disk

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"
	boshsys "github.com/cloudfoundry/bosh-utils/system"
)

type sfdiskPartitioner struct {
	logger    boshlog.Logger
	cmdRunner boshsys.CmdRunner
	logTag    string
}

func NewSfdiskPartitioner(logger boshlog.Logger, cmdRunner boshsys.CmdRunner) Partitioner {
	return sfdiskPartitioner{
		logger:    logger,
		cmdRunner: cmdRunner,
		logTag:    "SfdiskPartitioner",
	}
}

func (p sfdiskPartitioner) Partition(devicePath string, partitions []Partition) error {
	if p.diskMatchesPartitions(devicePath, partitions) {
		p.logger.Info(p.logTag, "%s already partitioned as expected, skipping", devicePath)
		return nil
	}

	sfdiskPartitionTypes := map[PartitionType]string{
		PartitionTypeSwap:  "S",
		PartitionTypeLinux: "L",
	}

	sfdiskInput := ""
	for index, partition := range partitions {
		sfdiskPartitionType := sfdiskPartitionTypes[partition.Type]
		partitionSize := fmt.Sprintf("%d", p.convertFromBytesToMb(partition.SizeInBytes))

		if index == len(partitions)-1 {
			partitionSize = ""
		}

		sfdiskInput = sfdiskInput + fmt.Sprintf(",%s,%s\n", partitionSize, sfdiskPartitionType)
	}

	maxFormatRetries := 20
	for i := 1; ; i++ {
		p.logger.Info(p.logTag, "%d: Partitioning %s with %s", i, devicePath, sfdiskInput)
		_, _, _, err := p.cmdRunner.RunCommandWithInput(sfdiskInput, "sfdisk", "-uM", devicePath)
		if err == nil {
			p.logger.Info(p.logTag, "%d: Succeed in partitioning %s with %s", i, devicePath, sfdiskInput)
			break
		}
		if i == maxFormatRetries {
			return bosherr.WrapError(err, "Shelling out to sfdisk")
		}
		p.logger.Error(p.logTag, "%d: Failed with an error: %s", i, err)
		time.Sleep(3 * time.Second)
	}

	return nil
}

func (p sfdiskPartitioner) GetDeviceSizeInBytes(devicePath string) (uint64, error) {
	stdout, _, _, err := p.cmdRunner.RunCommand("sfdisk", "-s", devicePath)
	if err != nil {
		return 0, bosherr.WrapError(err, "Shelling out to sfdisk")
	}

	sizeInKb, err := strconv.ParseUint(strings.Trim(stdout, "\n"), 10, 64)
	if err != nil {
		return 0, bosherr.WrapError(err, "Converting disk size to integer")
	}

	return p.convertFromKbToBytes(sizeInKb), nil
}

func (p sfdiskPartitioner) diskMatchesPartitions(devicePath string, partitionsToMatch []Partition) (result bool) {
	existingPartitions, err := p.getPartitions(devicePath)
	if err != nil {
		err = bosherr.WrapErrorf(err, "Getting partitions for %s", devicePath)
		return
	}

	if len(existingPartitions) < len(partitionsToMatch) {
		return
	}

	remainingDiskSpace, err := p.GetDeviceSizeInBytes(devicePath)
	if err != nil {
		err = bosherr.WrapErrorf(err, "Getting device size for %s", devicePath)
		return
	}

	for index, partitionToMatch := range partitionsToMatch {
		if index == len(partitionsToMatch)-1 {
			partitionToMatch.SizeInBytes = remainingDiskSpace
		}

		existingPartition := existingPartitions[index]
		switch {
		case existingPartition.Type != partitionToMatch.Type:
			return
		case !withinDelta(existingPartition.SizeInBytes, partitionToMatch.SizeInBytes, p.convertFromMbToBytes(20)):
			return
		}

		remainingDiskSpace = remainingDiskSpace - partitionToMatch.SizeInBytes
	}

	return true
}

func (p sfdiskPartitioner) getPartitions(devicePath string) (partitions []Partition, err error) {
	stdout, _, _, err := p.cmdRunner.RunCommand("sfdisk", "-d", devicePath)
	if err != nil {
		err = bosherr.WrapError(err, "Shelling out to sfdisk")
		return
	}

	allLines := strings.Split(stdout, "\n")
	if len(allLines) < 4 {
		return
	}

	partitionLines := allLines[3 : len(allLines)-1]

	for _, partitionLine := range partitionLines {
		partitionPath, partitionType := extractPartitionPathAndType(partitionLine)
		partition := Partition{Type: partitionType}

		if partition.Type != PartitionTypeEmpty {
			size, err := p.GetDeviceSizeInBytes(partitionPath)
			if err == nil {
				partition.SizeInBytes = size
			}
		}

		partitions = append(partitions, partition)
	}
	return
}

var partitionTypesMap = map[string]PartitionType{
	"82": PartitionTypeSwap,
	"83": PartitionTypeLinux,
	"0":  PartitionTypeEmpty,
}

func extractPartitionPathAndType(line string) (partitionPath string, partitionType PartitionType) {
	partitionFields := strings.Fields(line)
	lastField := partitionFields[len(partitionFields)-1]

	sfdiskPartitionType := strings.Replace(lastField, "Id=", "", 1)

	partitionPath = partitionFields[0]
	partitionType = partitionTypesMap[sfdiskPartitionType]
	return
}

func (p sfdiskPartitioner) convertFromBytesToMb(sizeInBytes uint64) uint64 {
	return sizeInBytes / (1024 * 1024)
}

func (p sfdiskPartitioner) convertFromMbToBytes(sizeInMb uint64) uint64 {
	return sizeInMb * 1024 * 1024
}

func (p sfdiskPartitioner) convertFromKbToBytes(sizeInKb uint64) uint64 {
	return sizeInKb * 1024
}
<|endoftext|>


<commit_before>package wellington

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"testing"
	"time"
)

func TestRebuild(t *testing.T) {
	var f *os.File
	log.SetOutput(f)
	pmap := NewPartialMap()
	path := "test/file1a.scss"
	pmap.Add("file/event", []string{path})
	wc, err := NewWatcher(&WatchOptions{
		PartialMap: pmap,
	})
	if err != nil {
		t.Fatal(err)
	}

	rebuildMu.Lock()
	rebuildChan = make(chan []string, 1)
	rebuildMu.Unlock()
	defer func() {
		rebuildMu.Lock()
		rebuildChan = nil
		rebuildMu.Unlock()
	}()

	done := make(chan error)
	go func(t *testing.T) {
		select {
		case p := <-rebuildChan:
			if p[0] != path {
				t.Errorf("got: %s wanted: %s", p[0], path)
			}
			done <- nil
		case err := <-wc.errChan:
			if err == nil {
				done <- err
			}
			if e := fmt.Errorf("build args"); e != err {
				done <- fmt.Errorf("got: %s wanted: %s", e, err)
			}
		case <-time.After(5000 * time.Millisecond):
			done <- fmt.Errorf("timeout waiting for load error")
		}
	}(t)

	// rebuild doesn't throw errors ever
	wc.rebuild("file/event")

	err = <-done
	if err != nil {
		t.Fatal(err)
	}
}

func TestRebuild_watch(t *testing.T) {
	tdir, err := ioutil.TempDir(os.TempDir(), "testwatch_")
	if err != nil {
		t.Fatal(err)
	}
	tfile := filepath.Join(tdir, "_new.scss")
	fh, err := os.Create(tfile)
	if err != nil {
		t.Fatal(err)
	}

	rebuildMu.Lock()
	rebuildChan = make(chan []string, 1)
	rebuildMu.Unlock()
	defer func() {
		rebuildMu.Lock()
		rebuildChan = nil
		rebuildMu.Unlock()
	}()

	pMap := NewPartialMap()
	pMap.AddRelation("tswif", tfile)
	w, err := NewWatcher(&WatchOptions{
		Paths:      []string{tdir},
		PartialMap: pMap,
	})
	if err != nil {
		t.Fatal(err)
	}
	err = w.Watch()
	if err != nil {
		t.Fatal(err)
	}

	done := make(chan bool, 1)
	go func(t *testing.T) {
		select {
		case <-rebuildChan:
			done <- true
		case <-time.After(2 * time.Second):
			done <- false
		}
		done <- true
	}(t)
	fh.WriteString("boom")
	success := <-done
	if !success {
		t.Fatal("Timeout waiting for rebuild")
	}

}

func TestWatch(t *testing.T) {
	w, err := NewWatcher(NewWatchOptions())
	if err != nil {
		t.Fatal(err)
	}
	err = w.Watch()
	if err == nil {
		t.Error("No errors thrown for nil directories")
	}
	// w.fw.Close()

	watcherChan = make(chan string, 1)
	w, err = NewWatcher(&WatchOptions{
		Paths:      []string{"test"},
		PartialMap: NewPartialMap(),
	})
	if err != nil {
		t.Fatal(err)
	}
	err = w.Watch()

	// Test file creation event
	go func() {
		select {
		case <-watcherChan:
			break
		case <-time.After(5 * time.Second):
			fmt.Printf("timeout %d\n", len(watcherChan))
			t.Error("Timeout without creating file")
		}
	}()

	testFile := "test/watchfile.lock"
	f, err := os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666)
	defer func() {
		// Give time for filesystem to sync before deleting file
		time.Sleep(50 * time.Millisecond)
		os.Remove(testFile)
		f.Close()
	}()
	if err != nil {
		t.Fatalf("creating test file failed: %s", err)
	}
	f.Sync()

	// Test file modification event
	go func() {
		select {
		case <-watcherChan:
			break
		case <-time.After(2 * time.Second):
			fmt.Printf("timeout %d\n", len(watcherChan))
			t.Error("Timeout without detecting write")
		}
	}()

	f.WriteString("data")
	f.Sync()
}

func TestWatch_errors(t *testing.T) {
	w, err := NewWatcher(nil)
	if err != nil {
		t.Fatal(err)
	}

	if w.opts == nil {
		t.Fatal("unexpected nil")
	}
}

func TestAppendUnique(t *testing.T) {
	lst := []string{"a", "b", "c"}
	new := appendUnique(lst, "a")
	if len(new) != len(lst) {
		t.Errorf("got: %d wanted: %d", len(new), len(lst))
	}

	new = appendUnique(lst, "d")
	if len(new) != len(lst)+1 {
		t.Errorf("got: %d wanted: %d", len(new), len(lst)+1)
	}
}
<commit_msg>paths must be available now with fsevents<commit_after>package wellington

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"testing"
	"time"
)

func TestRebuild(t *testing.T) {
	var f *os.File
	log.SetOutput(f)
	pmap := NewPartialMap()
	path := "test/file1a.scss"
	pmap.Add("file/event", []string{path})
	wc, err := NewWatcher(&WatchOptions{
		PartialMap: pmap,
	})
	if err != nil {
		t.Fatal(err)
	}

	rebuildMu.Lock()
	rebuildChan = make(chan []string, 1)
	rebuildMu.Unlock()
	defer func() {
		rebuildMu.Lock()
		rebuildChan = nil
		rebuildMu.Unlock()
	}()

	done := make(chan error)
	go func(t *testing.T) {
		select {
		case p := <-rebuildChan:
			if p[0] != path {
				t.Errorf("got: %s wanted: %s", p[0], path)
			}
			done <- nil
		case err := <-wc.errChan:
			if err == nil {
				done <- err
			}
			if e := fmt.Errorf("build args"); e != err {
				done <- fmt.Errorf("got: %s wanted: %s", e, err)
			}
		case <-time.After(5000 * time.Millisecond):
			done <- fmt.Errorf("timeout waiting for load error")
		}
	}(t)

	// rebuild doesn't throw errors ever
	wc.rebuild("file/event")

	err = <-done
	if err != nil {
		t.Fatal(err)
	}
}

func TestRebuild_watch(t *testing.T) {
	tdir, err := ioutil.TempDir(os.TempDir(), "testwatch_")
	if err != nil {
		t.Fatal(err)
	}
	tfile := filepath.Join(tdir, "_new.scss")
	fh, err := os.Create(tfile)
	if err != nil {
		t.Fatal(err)
	}

	rebuildMu.Lock()
	rebuildChan = make(chan []string, 1)
	rebuildMu.Unlock()
	defer func() {
		rebuildMu.Lock()
		rebuildChan = nil
		rebuildMu.Unlock()
	}()

	pMap := NewPartialMap()
	pMap.AddRelation("tswif", tfile)
	w, err := NewWatcher(&WatchOptions{
		Paths:      []string{tdir},
		PartialMap: pMap,
	})
	if err != nil {
		t.Fatal(err)
	}
	err = w.Watch()
	if err != nil {
		t.Fatal(err)
	}

	done := make(chan bool, 1)
	go func(t *testing.T) {
		select {
		case <-rebuildChan:
			done <- true
		case <-time.After(2 * time.Second):
			done <- false
		}
		done <- true
	}(t)
	fh.WriteString("boom")
	success := <-done
	if !success {
		t.Fatal("Timeout waiting for rebuild")
	}

}

func TestWatch_success(t *testing.T) {
	pmap := NewPartialMap()
	path := "test/file1a.scss"
	pmap.Add("file/event", []string{path})
	opts := &WatchOptions{
		Paths:      []string{"test"},
		PartialMap: pmap,
	}
	w, err := NewWatcher(opts)
	if err != nil {
		t.Fatal(err)
	}
	err = w.Watch()
	if err == nil {
		t.Error("No errors thrown for nil directories")
	}
	// w.fw.Close()

	watcherChan = make(chan string, 1)
	w, err = NewWatcher(&WatchOptions{
		Paths:      []string{"test"},
		PartialMap: NewPartialMap(),
	})
	if err != nil {
		t.Fatal(err)
	}
	err = w.Watch()

	// Test file creation event
	go func() {
		select {
		case <-watcherChan:
			break
		case <-time.After(5 * time.Second):
			fmt.Printf("timeout %d\n", len(watcherChan))
			t.Error("Timeout without creating file")
		}
	}()

	testFile := "test/watchfile.lock"
	f, err := os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666)
	defer func() {
		// Give time for filesystem to sync before deleting file
		time.Sleep(50 * time.Millisecond)
		os.Remove(testFile)
		f.Close()
	}()
	if err != nil {
		t.Fatalf("creating test file failed: %s", err)
	}
	f.Sync()

	// Test file modification event
	go func() {
		select {
		case <-watcherChan:
			break
		case <-time.After(2 * time.Second):
			fmt.Printf("timeout %d\n", len(watcherChan))
			t.Error("Timeout without detecting write")
		}
	}()

	f.WriteString("data")
	f.Sync()
}

func TestWatch_errors(t *testing.T) {
	w, err := NewWatcher(nil)
	if err != nil {
		t.Fatal(err)
	}

	if w.opts == nil {
		t.Fatal("unexpected nil")
	}
}

func TestAppendUnique(t *testing.T) {
	lst := []string{"a", "b", "c"}
	new := appendUnique(lst, "a")
	if len(new) != len(lst) {
		t.Errorf("got: %d wanted: %d", len(new), len(lst))
	}

	new = appendUnique(lst, "d")
	if len(new) != len(lst)+1 {
		t.Errorf("got: %d wanted: %d", len(new), len(lst)+1)
	}
}
<|endoftext|>