<commit_before>// Copyright 2009 The Go9p Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The clnt package provides definitions and functions used to implement
// a 9P2000 file client.
package clnt

import (
	"code.google.com/p/go9p/p"
	"fmt"
	"log"
	"net"
	"sync"
	"syscall"
)

// Debug flags
const (
	DbgPrintFcalls  = (1 << iota) // print all 9P messages on stderr
	DbgPrintPackets               // print the raw packets on stderr
	DbgLogFcalls                  // keep the last N 9P messages (can be accessed over http)
	DbgLogPackets                 // keep the last N 9P messages (can be accessed over http)
)

type StatsOps interface {
	statsRegister()
	statsUnregister()
}

// The Clnt type represents a 9P2000 client. The client is connected to
// a 9P2000 file server and its methods can be used to access and manipulate
// the files exported by the server.
type Clnt struct {
	sync.Mutex
	Debuglevel int    // =0 don't print anything, >0 print Fcalls, >1 print raw packets
	Msize      uint32 // Maximum size of the 9P messages
	Dotu       bool   // If true, 9P2000.u protocol is spoken
	Root       *Fid   // Fid that points to the root directory
	Id         string // Used when printing debug messages
	Log        *p.Logger

	conn     net.Conn
	tagpool  *pool
	fidpool  *pool
	reqout   chan *Req
	done     chan bool
	reqfirst *Req
	reqlast  *Req
	err      error

	reqchan chan *Req
	tchan   chan *p.Fcall

	next, prev *Clnt
}

// A Fid type represents a file on the server. Fids are used for the
// low level methods that correspond directly to the 9P2000 message requests
type Fid struct {
	sync.Mutex
	Clnt   *Clnt // Client the fid belongs to
	Iounit uint32
	p.Qid         // The Qid description for the file
	Mode   uint8  // Open mode (one of p.O* values) (if file is open)
	Fid    uint32 // Fid number
	p.User        // The user the fid belongs to
	walked bool   // true if the fid points to a walked file on the server
}

// The file is similar to the Fid, but is used in the high-level client
// interface.
type File struct {
	fid    *Fid
	offset uint64
}

type pool struct {
	sync.Mutex
	need  int
	nchan chan uint32
	maxid uint32
	imap  []byte
}

type Req struct {
	sync.Mutex
	Clnt       *Clnt
	Tc         *p.Fcall
	Rc         *p.Fcall
	Err        error
	Done       chan *Req
	tag        uint16
	prev, next *Req
	fid        *Fid
}

type ClntList struct {
	sync.Mutex
	clntList, clntLast *Clnt
}

var clnts *ClntList
var DefaultDebuglevel int
var DefaultLogger *p.Logger

func (clnt *Clnt) Rpcnb(r *Req) error {
	var tag uint16

	if r.Tc.Type == p.Tversion {
		tag = p.NOTAG
	} else {
		tag = r.tag
	}

	p.SetTag(r.Tc, tag)
	clnt.Lock()
	if clnt.err != nil {
		clnt.Unlock()
		return clnt.err
	}

	if clnt.reqlast != nil {
		clnt.reqlast.next = r
	} else {
		clnt.reqfirst = r
	}

	r.prev = clnt.reqlast
	clnt.reqlast = r
	clnt.Unlock()

	clnt.reqout <- r
	return nil
}

func (clnt *Clnt) Rpc(tc *p.Fcall) (rc *p.Fcall, err error) {
	r := clnt.ReqAlloc()
	r.Tc = tc
	r.Done = make(chan *Req)
	err = clnt.Rpcnb(r)
	if err != nil {
		return
	}

	<-r.Done
	rc = r.Rc
	err = r.Err
	clnt.ReqFree(r)
	return
}

func (clnt *Clnt) recv() {
	var err error

	err = nil
	buf := make([]byte, clnt.Msize*8)
	pos := 0
	for {
		if len(buf) < int(clnt.Msize) {
			b := make([]byte, clnt.Msize*8)
			copy(b, buf[0:pos])
			buf = b
			b = nil
		}

		n, oerr := clnt.conn.Read(buf[pos:len(buf)])
		if oerr != nil || n == 0 {
			err = &p.Error{oerr.Error(), p.EIO}
			clnt.Lock()
			clnt.err = err
			clnt.Unlock()
			goto closed
		}

		pos += n
		for pos > 4 {
			sz, _ := p.Gint32(buf)
			if pos < int(sz) {
				if len(buf) < int(sz) {
					b := make([]byte, clnt.Msize*8)
					copy(b, buf[0:pos])
					buf = b
					b = nil
				}

				break
			}

			fc, err, fcsize := p.Unpack(buf, clnt.Dotu)
			clnt.Lock()
			if err != nil {
				clnt.err = err
				clnt.conn.Close()
				clnt.Unlock()
				goto closed
			}

			if clnt.Debuglevel > 0 {
				clnt.logFcall(fc)
				if clnt.Debuglevel&DbgPrintPackets != 0 {
					log.Println("}-}", clnt.Id, fmt.Sprint(fc.Pkt))
				}

				if clnt.Debuglevel&DbgPrintFcalls != 0 {
					log.Println("}}}", clnt.Id, fc.String())
				}
			}

			var r *Req = nil
			for r = clnt.reqfirst; r != nil; r = r.next {
				if r.Tc.Tag == fc.Tag {
					break
				}
			}

			if r == nil {
				clnt.err = &p.Error{"unexpected response", p.EINVAL}
				clnt.conn.Close()
				clnt.Unlock()
				goto closed
			}

			r.Rc = fc
			if r.prev != nil {
				r.prev.next = r.next
			} else {
				clnt.reqfirst = r.next
			}

			if r.next != nil {
				r.next.prev = r.prev
			} else {
				clnt.reqlast = r.prev
			}
			clnt.Unlock()

			if r.Tc.Type != r.Rc.Type-1 {
				if r.Rc.Type != p.Rerror {
					r.Err = &p.Error{"invalid response", p.EINVAL}
					log.Println(fmt.Sprintf("TTT %v", r.Tc))
					log.Println(fmt.Sprintf("RRR %v", r.Rc))
				} else {
					if r.Err == nil {
						r.Err = &p.Error{r.Rc.Error, syscall.Errno(r.Rc.Errornum)}
					}
				}
			}

			if r.Done != nil {
				r.Done <- r
			}

			pos -= fcsize
			buf = buf[fcsize:]
		}
	}

closed:
	clnt.done <- true

	/* send error to all pending requests */
	clnt.Lock()
	r := clnt.reqfirst
	clnt.reqfirst = nil
	clnt.reqlast = nil
	if err == nil {
		err = clnt.err
	}
	clnt.Unlock()
	for ; r != nil; r = r.next {
		r.Err = err
		if r.Done != nil {
			r.Done <- r
		}
	}

	clnts.Lock()
	if clnt.prev != nil {
		clnt.prev.next = clnt.next
	} else {
		clnts.clntList = clnt.next
	}

	if clnt.next != nil {
		clnt.next.prev = clnt.prev
	} else {
		clnts.clntLast = clnt.prev
	}
	clnts.Unlock()

	if sop, ok := (interface{}(clnt)).(StatsOps); ok {
		sop.statsUnregister()
	}
}

func (clnt *Clnt) send() {
	for {
		select {
		case <-clnt.done:
			return

		case req := <-clnt.reqout:
			if clnt.Debuglevel > 0 {
				clnt.logFcall(req.Tc)
				if clnt.Debuglevel&DbgPrintPackets != 0 {
					log.Println("{-{", clnt.Id, fmt.Sprint(req.Tc.Pkt))
				}

				if clnt.Debuglevel&DbgPrintFcalls != 0 {
					log.Println("{{{", clnt.Id, req.Tc.String())
				}
			}

			for buf := req.Tc.Pkt; len(buf) > 0; {
				n, err := clnt.conn.Write(buf)
				if err != nil {
					/* just close the socket, will get signal on clnt.done */
					clnt.conn.Close()
					break
				}

				buf = buf[n:len(buf)]
			}
		}
	}
}

// Creates and initializes a new Clnt object. Doesn't send any data
// on the wire.
func NewClnt(c net.Conn, msize uint32, dotu bool) *Clnt {
	clnt := new(Clnt)
	clnt.conn = c
	clnt.Msize = msize
	clnt.Dotu = dotu
	clnt.Debuglevel = DefaultDebuglevel
	clnt.Log = DefaultLogger
	clnt.tagpool = newPool(uint32(p.NOTAG))
	clnt.fidpool = newPool(p.NOFID)
	clnt.reqout = make(chan *Req)
	clnt.done = make(chan bool)
	clnt.reqchan = make(chan *Req, 16)
	clnt.tchan = make(chan *p.Fcall, 16)

	go clnt.recv()
	go clnt.send()

	clnts.Lock()
	if clnts.clntLast != nil {
		clnts.clntLast.next = clnt
	} else {
		clnts.clntList = clnt
	}

	clnt.prev = clnts.clntLast
	clnts.clntLast = clnt
	clnts.Unlock()

	if sop, ok := (interface{}(clnt)).(StatsOps); ok {
		sop.statsRegister()
	}

	return clnt
}

// Establishes a new socket connection to the 9P server and creates
// a client object for it. Negotiates the dialect and msize for the
// connection. Returns a Clnt object, or Error.
func Connect(c net.Conn, msize uint32, dotu bool) (*Clnt, error) {
	clnt := NewClnt(c, msize, dotu)
	clnt.Id = c.RemoteAddr().String() + ":"
	ver := "9P2000"
	if clnt.Dotu {
		ver = "9P2000.u"
	}

	tc := p.NewFcall(clnt.Msize)
	err := p.PackTversion(tc, clnt.Msize, ver)
	if err != nil {
		return nil, err
	}

	rc, err := clnt.Rpc(tc)
	if err != nil {
		return nil, err
	}

	if rc.Msize < clnt.Msize {
		clnt.Msize = rc.Msize
	}

	clnt.Dotu = rc.Version == "9P2000.u" && clnt.Dotu
	return clnt, nil
}

// Creates a new Fid object for the client
func (clnt *Clnt) FidAlloc() *Fid {
	fid := new(Fid)
	fid.Fid = clnt.fidpool.getId()
	fid.Clnt = clnt

	return fid
}

func (clnt *Clnt) NewFcall() *p.Fcall {
	select {
	case tc := <-clnt.tchan:
		return tc
	default:
	}
	return p.NewFcall(clnt.Msize)
}

func (clnt *Clnt) FreeFcall(fc *p.Fcall) {
	if fc != nil && len(fc.Buf) >= int(clnt.Msize) {
		select {
		case clnt.tchan <- fc:
			break
		default:
		}
	}
}

func (clnt *Clnt) ReqAlloc() *Req {
	var req *Req
	select {
	case req = <-clnt.reqchan:
		break
	default:
		req = new(Req)
		req.Clnt = clnt
		req.tag = uint16(clnt.tagpool.getId())
	}
	return req
}

func (clnt *Clnt) ReqFree(req *Req) {
	clnt.FreeFcall(req.Tc)
	req.Tc = nil
	req.Rc = nil
	req.Err = nil
	req.Done = nil
	req.next = nil
	req.prev = nil

	select {
	case clnt.reqchan <- req:
		break
	default:
		clnt.tagpool.putId(uint32(req.tag))
	}
}

func (clnt *Clnt) logFcall(fc *p.Fcall) {
	if clnt.Debuglevel&DbgLogPackets != 0 {
		pkt := make([]byte, len(fc.Pkt))
		copy(pkt, fc.Pkt)
		clnt.Log.Log(pkt, clnt, DbgLogPackets)
	}

	if clnt.Debuglevel&DbgLogFcalls != 0 {
		f := new(p.Fcall)
		*f = *fc
		f.Pkt = nil
		clnt.Log.Log(f, clnt, DbgLogFcalls)
	}
}

func init() {
	clnts = new(ClntList)
	if sop, ok := (interface{}(clnts)).(StatsOps); ok {
		sop.statsRegister()
	}
}
<commit_msg>Removed executable bit from clnt.go.<commit_after>// Copyright 2009 The Go9p Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The clnt package provides definitions and functions used to implement
// a 9P2000 file client.
package clnt

import (
	"code.google.com/p/go9p/p"
	"fmt"
	"log"
	"net"
	"sync"
	"syscall"
)

// Debug flags
const (
	DbgPrintFcalls  = (1 << iota) // print all 9P messages on stderr
	DbgPrintPackets               // print the raw packets on stderr
	DbgLogFcalls                  // keep the last N 9P messages (can be accessed over http)
	DbgLogPackets                 // keep the last N 9P messages (can be accessed over http)
)

type StatsOps interface {
	statsRegister()
	statsUnregister()
}

// The Clnt type represents a 9P2000 client. The client is connected to
// a 9P2000 file server and its methods can be used to access and manipulate
// the files exported by the server.
type Clnt struct {
	sync.Mutex
	Debuglevel int    // =0 don't print anything, >0 print Fcalls, >1 print raw packets
	Msize      uint32 // Maximum size of the 9P messages
	Dotu       bool   // If true, 9P2000.u protocol is spoken
	Root       *Fid   // Fid that points to the root directory
	Id         string // Used when printing debug messages
	Log        *p.Logger

	conn     net.Conn
	tagpool  *pool
	fidpool  *pool
	reqout   chan *Req
	done     chan bool
	reqfirst *Req
	reqlast  *Req
	err      error

	reqchan chan *Req
	tchan   chan *p.Fcall

	next, prev *Clnt
}

// A Fid type represents a file on the server. Fids are used for the
// low level methods that correspond directly to the 9P2000 message requests
type Fid struct {
	sync.Mutex
	Clnt   *Clnt // Client the fid belongs to
	Iounit uint32
	p.Qid         // The Qid description for the file
	Mode   uint8  // Open mode (one of p.O* values) (if file is open)
	Fid    uint32 // Fid number
	p.User        // The user the fid belongs to
	walked bool   // true if the fid points to a walked file on the server
}

// The file is similar to the Fid, but is used in the high-level client
// interface.
type File struct {
	fid    *Fid
	offset uint64
}

type pool struct {
	sync.Mutex
	need  int
	nchan chan uint32
	maxid uint32
	imap  []byte
}

type Req struct {
	sync.Mutex
	Clnt       *Clnt
	Tc         *p.Fcall
	Rc         *p.Fcall
	Err        error
	Done       chan *Req
	tag        uint16
	prev, next *Req
	fid        *Fid
}

type ClntList struct {
	sync.Mutex
	clntList, clntLast *Clnt
}

var clnts *ClntList
var DefaultDebuglevel int
var DefaultLogger *p.Logger

func (clnt *Clnt) Rpcnb(r *Req) error {
	var tag uint16

	if r.Tc.Type == p.Tversion {
		tag = p.NOTAG
	} else {
		tag = r.tag
	}

	p.SetTag(r.Tc, tag)
	clnt.Lock()
	if clnt.err != nil {
		clnt.Unlock()
		return clnt.err
	}

	if clnt.reqlast != nil {
		clnt.reqlast.next = r
	} else {
		clnt.reqfirst = r
	}

	r.prev = clnt.reqlast
	clnt.reqlast = r
	clnt.Unlock()

	clnt.reqout <- r
	return nil
}

func (clnt *Clnt) Rpc(tc *p.Fcall) (rc *p.Fcall, err error) {
	r := clnt.ReqAlloc()
	r.Tc = tc
	r.Done = make(chan *Req)
	err = clnt.Rpcnb(r)
	if err != nil {
		return
	}

	<-r.Done
	rc = r.Rc
	err = r.Err
	clnt.ReqFree(r)
	return
}

func (clnt *Clnt) recv() {
	var err error

	err = nil
	buf := make([]byte, clnt.Msize*8)
	pos := 0
	for {
		if len(buf) < int(clnt.Msize) {
			b := make([]byte, clnt.Msize*8)
			copy(b, buf[0:pos])
			buf = b
			b = nil
		}

		n, oerr := clnt.conn.Read(buf[pos:len(buf)])
		if oerr != nil || n == 0 {
			err = &p.Error{oerr.Error(), p.EIO}
			clnt.Lock()
			clnt.err = err
			clnt.Unlock()
			goto closed
		}

		pos += n
		for pos > 4 {
			sz, _ := p.Gint32(buf)
			if pos < int(sz) {
				if len(buf) < int(sz) {
					b := make([]byte, clnt.Msize*8)
					copy(b, buf[0:pos])
					buf = b
					b = nil
				}

				break
			}

			fc, err, fcsize := p.Unpack(buf, clnt.Dotu)
			clnt.Lock()
			if err != nil {
				clnt.err = err
				clnt.conn.Close()
				clnt.Unlock()
				goto closed
			}

			if clnt.Debuglevel > 0 {
				clnt.logFcall(fc)
				if clnt.Debuglevel&DbgPrintPackets != 0 {
					log.Println("}-}", clnt.Id, fmt.Sprint(fc.Pkt))
				}

				if clnt.Debuglevel&DbgPrintFcalls != 0 {
					log.Println("}}}", clnt.Id, fc.String())
				}
			}

			var r *Req = nil
			for r = clnt.reqfirst; r != nil; r = r.next {
				if r.Tc.Tag == fc.Tag {
					break
				}
			}

			if r == nil {
				clnt.err = &p.Error{"unexpected response", p.EINVAL}
				clnt.conn.Close()
				clnt.Unlock()
				goto closed
			}

			r.Rc = fc
			if r.prev != nil {
				r.prev.next = r.next
			} else {
				clnt.reqfirst = r.next
			}

			if r.next != nil {
				r.next.prev = r.prev
			} else {
				clnt.reqlast = r.prev
			}
			clnt.Unlock()

			if r.Tc.Type != r.Rc.Type-1 {
				if r.Rc.Type != p.Rerror {
					r.Err = &p.Error{"invalid response", p.EINVAL}
					log.Println(fmt.Sprintf("TTT %v", r.Tc))
					log.Println(fmt.Sprintf("RRR %v", r.Rc))
				} else {
					if r.Err == nil {
						r.Err = &p.Error{r.Rc.Error, syscall.Errno(r.Rc.Errornum)}
					}
				}
			}

			if r.Done != nil {
				r.Done <- r
			}

			pos -= fcsize
			buf = buf[fcsize:]
		}
	}

closed:
	clnt.done <- true

	/* send error to all pending requests */
	clnt.Lock()
	r := clnt.reqfirst
	clnt.reqfirst = nil
	clnt.reqlast = nil
	if err == nil {
		err = clnt.err
	}
	clnt.Unlock()
	for ; r != nil; r = r.next {
		r.Err = err
		if r.Done != nil {
			r.Done <- r
		}
	}

	clnts.Lock()
	if clnt.prev != nil {
		clnt.prev.next = clnt.next
	} else {
		clnts.clntList = clnt.next
	}

	if clnt.next != nil {
		clnt.next.prev = clnt.prev
	} else {
		clnts.clntLast = clnt.prev
	}
	clnts.Unlock()

	if sop, ok := (interface{}(clnt)).(StatsOps); ok {
		sop.statsUnregister()
	}
}

func (clnt *Clnt) send() {
	for {
		select {
		case <-clnt.done:
			return

		case req := <-clnt.reqout:
			if clnt.Debuglevel > 0 {
				clnt.logFcall(req.Tc)
				if clnt.Debuglevel&DbgPrintPackets != 0 {
					log.Println("{-{", clnt.Id, fmt.Sprint(req.Tc.Pkt))
				}

				if clnt.Debuglevel&DbgPrintFcalls != 0 {
					log.Println("{{{", clnt.Id, req.Tc.String())
				}
			}

			for buf := req.Tc.Pkt; len(buf) > 0; {
				n, err := clnt.conn.Write(buf)
				if err != nil {
					/* just close the socket, will get signal on clnt.done */
					clnt.conn.Close()
					break
				}

				buf = buf[n:len(buf)]
			}
		}
	}
}

// Creates and initializes a new Clnt object. Doesn't send any data
// on the wire.
func NewClnt(c net.Conn, msize uint32, dotu bool) *Clnt {
	clnt := new(Clnt)
	clnt.conn = c
	clnt.Msize = msize
	clnt.Dotu = dotu
	clnt.Debuglevel = DefaultDebuglevel
	clnt.Log = DefaultLogger
	clnt.tagpool = newPool(uint32(p.NOTAG))
	clnt.fidpool = newPool(p.NOFID)
	clnt.reqout = make(chan *Req)
	clnt.done = make(chan bool)
	clnt.reqchan = make(chan *Req, 16)
	clnt.tchan = make(chan *p.Fcall, 16)

	go clnt.recv()
	go clnt.send()

	clnts.Lock()
	if clnts.clntLast != nil {
		clnts.clntLast.next = clnt
	} else {
		clnts.clntList = clnt
	}

	clnt.prev = clnts.clntLast
	clnts.clntLast = clnt
	clnts.Unlock()

	if sop, ok := (interface{}(clnt)).(StatsOps); ok {
		sop.statsRegister()
	}

	return clnt
}

// Establishes a new socket connection to the 9P server and creates
// a client object for it. Negotiates the dialect and msize for the
// connection. Returns a Clnt object, or Error.
func Connect(c net.Conn, msize uint32, dotu bool) (*Clnt, error) {
	clnt := NewClnt(c, msize, dotu)
	clnt.Id = c.RemoteAddr().String() + ":"
	ver := "9P2000"
	if clnt.Dotu {
		ver = "9P2000.u"
	}

	tc := p.NewFcall(clnt.Msize)
	err := p.PackTversion(tc, clnt.Msize, ver)
	if err != nil {
		return nil, err
	}

	rc, err := clnt.Rpc(tc)
	if err != nil {
		return nil, err
	}

	if rc.Msize < clnt.Msize {
		clnt.Msize = rc.Msize
	}

	clnt.Dotu = rc.Version == "9P2000.u" && clnt.Dotu
	return clnt, nil
}

// Creates a new Fid object for the client
func (clnt *Clnt) FidAlloc() *Fid {
	fid := new(Fid)
	fid.Fid = clnt.fidpool.getId()
	fid.Clnt = clnt

	return fid
}

func (clnt *Clnt) NewFcall() *p.Fcall {
	select {
	case tc := <-clnt.tchan:
		return tc
	default:
	}
	return p.NewFcall(clnt.Msize)
}

func (clnt *Clnt) FreeFcall(fc *p.Fcall) {
	if fc != nil && len(fc.Buf) >= int(clnt.Msize) {
		select {
		case clnt.tchan <- fc:
			break
		default:
		}
	}
}

func (clnt *Clnt) ReqAlloc() *Req {
	var req *Req
	select {
	case req = <-clnt.reqchan:
		break
	default:
		req = new(Req)
		req.Clnt = clnt
		req.tag = uint16(clnt.tagpool.getId())
	}
	return req
}

func (clnt *Clnt) ReqFree(req *Req) {
	clnt.FreeFcall(req.Tc)
	req.Tc = nil
	req.Rc = nil
	req.Err = nil
	req.Done = nil
	req.next = nil
	req.prev = nil

	select {
	case clnt.reqchan <- req:
		break
	default:
		clnt.tagpool.putId(uint32(req.tag))
	}
}

func (clnt *Clnt) logFcall(fc *p.Fcall) {
	if clnt.Debuglevel&DbgLogPackets != 0 {
		pkt := make([]byte, len(fc.Pkt))
		copy(pkt, fc.Pkt)
		clnt.Log.Log(pkt, clnt, DbgLogPackets)
	}

	if clnt.Debuglevel&DbgLogFcalls != 0 {
		f := new(p.Fcall)
		*f = *fc
		f.Pkt = nil
		clnt.Log.Log(f, clnt, DbgLogFcalls)
	}
}

func init() {
	clnts = new(ClntList)
	if sop, ok := (interface{}(clnts)).(StatsOps); ok {
		sop.statsRegister()
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package buffer

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestGrowth(t *testing.T) {
	t.Parallel()
	x := 10
	g := NewRingGrowing(1)
	for i := 0; i < x; i++ {
		assert.Equal(t, i, g.readable)
		g.WriteOne(i)
	}
	read := 0
	for g.readable > 0 {
		v, ok := g.ReadOne()
		assert.True(t, ok)
		assert.Equal(t, read, v)
		read++
	}
	assert.Equalf(t, x, read, "expected to have read %d items: %d", x, read)
	assert.Zerof(t, g.readable, "expected readable to be zero: %d", g.readable)
	assert.Equalf(t, g.n, 16, "expected N to be 16: %d", g.n)
}

func TestEmpty(t *testing.T) {
	t.Parallel()
	g := NewRingGrowing(1)
	_, ok := g.ReadOne()
	assert.False(t, ok)
}
<commit_msg>fix assert.Equal argument order<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package buffer

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestGrowth(t *testing.T) {
	t.Parallel()
	x := 10
	g := NewRingGrowing(1)
	for i := 0; i < x; i++ {
		assert.Equal(t, i, g.readable)
		g.WriteOne(i)
	}
	read := 0
	for g.readable > 0 {
		v, ok := g.ReadOne()
		assert.True(t, ok)
		assert.Equal(t, read, v)
		read++
	}
	assert.Equalf(t, x, read, "expected to have read %d items: %d", x, read)
	assert.Zerof(t, g.readable, "expected readable to be zero: %d", g.readable)
	assert.Equalf(t, 16, g.n, "expected N to be 16: %d", g.n)
}

func TestEmpty(t *testing.T) {
	t.Parallel()
	g := NewRingGrowing(1)
	_, ok := g.ReadOne()
	assert.False(t, ok)
}
<|endoftext|>


<commit_before>package packit

import (
	"bytes"
	"io"
	"os"
	"testing"

	"github.com/hooklift/assert"
)

func TestZip(t *testing.T) {
	buf := new(bytes.Buffer)
	Zip("fixtures/myfiles", buf)

	f, err := os.Create("boom.zip")
	assert.Ok(t, err)
	defer func() {
		err := f.Close()
		assert.Ok(t, err)
	}()

	_, err = io.Copy(f, buf)
	assert.Ok(t, err)
}

func TestTar(t *testing.T) {
	buf := new(bytes.Buffer)
	Tar("fixtures/myfiles", buf)

	f, err := os.Create("boom.tar")
	assert.Ok(t, err)
	defer func() {
		err := f.Close()
		assert.Ok(t, err)
	}()

	_, err = io.Copy(f, buf)
	assert.Ok(t, err)
}

func TestGzip(t *testing.T) {
	tar := new(bytes.Buffer)
	targz := new(bytes.Buffer)

	Tar("fixtures/myfiles", tar)
	err := Gzip(tar, targz)
	assert.Ok(t, err)
}

func TestBzip2(t *testing.T) {

}

func TestXz(t *testing.T) {

}
<commit_msg>Add tests for each supported compression and archive format<commit_after>package packit

import (
	"bytes"
	"testing"

	"github.com/c4milo/unzipit"
	"github.com/hooklift/assert"
)

func TestZip(t *testing.T) {
	buf := new(bytes.Buffer)
	Zip("fixtures/myfiles", buf)

	_, err := unpackit.Unpack(buf, "")
	assert.Ok(t, err)
}

func TestTar(t *testing.T) {
	buf := new(bytes.Buffer)
	Tar("fixtures/myfiles", buf)

	_, err := unpackit.Unpack(buf, "")
	assert.Ok(t, err)
}

func TestGzip(t *testing.T) {
	tar := new(bytes.Buffer)
	tarGz := new(bytes.Buffer)

	Tar("fixtures/myfiles", tar)
	err := Gzip(tar, tarGz)
	assert.Ok(t, err)

	_, err = unpackit.Unpack(tarGz, "")
	assert.Ok(t, err)
}

func TestBzip2(t *testing.T) {
	tar := new(bytes.Buffer)
	tarBzip2 := new(bytes.Buffer)
	Tar("fixtures/myfiles", tar)
	err := Bzip2(tar, tarBzip2)
	assert.Ok(t, err)

	_, err = unpackit.Unpack(tarBzip2, "")
	assert.Ok(t, err)
}

func TestXz(t *testing.T) {
	tar := new(bytes.Buffer)
	tarXz := new(bytes.Buffer)
	Tar("fixtures/myfiles", tar)
	err := Xz(tar, tarXz)
	assert.Ok(t, err)

	_, err = unpackit.Unpack(tarXz, "")
	assert.Ok(t, err)
}
<|endoftext|>


<commit_before>package pages

import (
	"fmt"
	"path"
	"path/filepath"

	"github.com/osteele/gojekyll/templates"
	"github.com/osteele/gojekyll/utils"
	"github.com/osteele/liquid"
)

// ToLiquid is part of the liquid.Drop interface.
func (d *StaticFile) ToLiquid() interface{} {
	return liquid.IterationKeyedMap(map[string]interface{}{
		"name":          d.relpath,
		"basename":      utils.TrimExt(d.relpath),
		"path":          d.Permalink(),
		"modified_time": d.fileModTime,
		"extname":       d.OutputExt(),
	})
}

func (f *file) ToLiquid() interface{} {
	var (
		relpath = "/" + filepath.ToSlash(f.relpath)
		base    = path.Base(relpath)
		ext     = path.Ext(relpath)
	)

	return liquid.IterationKeyedMap(templates.MergeVariableMaps(f.frontMatter, map[string]interface{}{
		"path":          relpath,
		"modified_time": f.fileModTime,
		"name":          base,
		"basename":      utils.TrimExt(base),
		"extname":       ext,
	}))
}

// ToLiquid is in the liquid.Drop interface.
func (p *page) ToLiquid() interface{} {
	var (
		relpath = p.relpath
		ext     = filepath.Ext(relpath)
		root    = utils.TrimExt(p.relpath)
		base    = filepath.Base(root)
	)
	data := map[string]interface{}{
		"content": p.maybeContent(),
		"excerpt": p.Excerpt(),
		"path":    relpath,
		"url":     p.Permalink(),
		// "output": // TODO; includes layouts

		// not documented, but present in both collection and non-collection pages
		"permalink": p.Permalink(),

		// TODO only in non-collection pages:
		"dir":  fmt.Sprintf("%c%s", filepath.Separator, path.Dir(relpath)),
		"name": path.Base(relpath),

		// TODO documented as present in all pages, but de facto only defined for collection pages
		"id":         base,
		"categories": p.Categories(),
		"tags":       p.Tags(),
		// "title": base,

		// TODO Only present in collection pages https://jekyllrb.com/docs/collections/#documents
		"relative_path": filepath.ToSlash(p.site.RelativePath(p.filename)),
		// TODO collection(name)

		// TODO undocumented; only present in collection pages:
		"ext": ext,
	}
	for k, v := range p.frontMatter {
		switch k {
		// doc implies these aren't present, but they appear to be present in a collection page:
		// case "layout", "published":
		case "permalink":
		// omit this, in order to use the value above
		default:
			data[k] = v
		}
	}
	return liquid.IterationKeyedMap(data)
}

func (p *page) maybeContent() interface{} {
	p.RLock()
	defer p.RUnlock()
	if p.rendered {
		return p.content
	}
	return p.raw
}
<commit_msg>Remove page.{dir,name,permalink}<commit_after>package pages

import (
	"path"
	"path/filepath"

	"github.com/osteele/gojekyll/templates"
	"github.com/osteele/gojekyll/utils"
	"github.com/osteele/liquid"
)

// ToLiquid is part of the liquid.Drop interface.
func (d *StaticFile) ToLiquid() interface{} {
	return liquid.IterationKeyedMap(map[string]interface{}{
		"name":          d.relpath,
		"basename":      utils.TrimExt(d.relpath),
		"path":          d.Permalink(),
		"modified_time": d.fileModTime,
		"extname":       d.OutputExt(),
	})
}

func (f *file) ToLiquid() interface{} {
	var (
		relpath = "/" + filepath.ToSlash(f.relpath)
		base    = path.Base(relpath)
		ext     = path.Ext(relpath)
	)

	return liquid.IterationKeyedMap(templates.MergeVariableMaps(f.frontMatter, map[string]interface{}{
		"path":          relpath,
		"modified_time": f.fileModTime,
		"name":          base,
		"basename":      utils.TrimExt(base),
		"extname":       ext,
	}))
}

// ToLiquid is in the liquid.Drop interface.
func (p *page) ToLiquid() interface{} {
	var (
		relpath = p.relpath
		ext     = filepath.Ext(relpath)
		root    = utils.TrimExt(p.relpath)
		base    = filepath.Base(root)
	)
	data := map[string]interface{}{
		"content": p.maybeContent(),
		"excerpt": p.Excerpt(),
		"path":    relpath,
		"url":     p.Permalink(),
		// "output": // TODO; includes layouts

		// TODO documented as present in all pages, but de facto only defined for collection pages
		"id":         base,
		"categories": p.Categories(),
		"tags":       p.Tags(),
		// "title": base,

		// TODO Only present in collection pages https://jekyllrb.com/docs/collections/#documents
		"relative_path": filepath.ToSlash(p.site.RelativePath(p.filename)),
		// TODO collection(name)

		// TODO undocumented; only present in collection pages:
		"ext": ext,
	}
	for k, v := range p.frontMatter {
		switch k {
		// doc implies these aren't present, but they appear to be present in a collection page:
		// case "layout", "published":
		case "permalink":
		// omit this, in order to use the value above
		default:
			data[k] = v
		}
	}
	return liquid.IterationKeyedMap(data)
}

func (p *page) maybeContent() interface{} {
	p.RLock()
	defer p.RUnlock()
	if p.rendered {
		return p.content
	}
	return p.raw
}
<|endoftext|>


<commit_before>package pages

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/datasektionen/taitan/anchor"

	"github.com/BurntSushi/toml"
	"github.com/russross/blackfriday"
	log "github.com/sirupsen/logrus"
)

// Resp is the response we serve for file queries.
type Resp struct {
	Title     string          `json:"title"` // Human-readable title.
	Slug      string          `json:"slug"`  // URL-slug.
	URL       string          `json:"url"`
	UpdatedAt string          `json:"updated_at"` // Body update time.
	Image     string          `json:"image"`      // Path/URL/Placeholder to image.
	Message   string          `json:"message"`    // Message to show at top
	Body      string          `json:"body"`       // Main content of the page.
	Sidebar   string          `json:"sidebar"`    // The sidebar of the page.
	Anchors   []anchor.Anchor `json:"anchors"`    // The list of anchors to headers in the body.
	Nav       []*Node         `json:"nav,omitempty"`
}

// Node is a recursive node in a page tree.
type Node struct {
	path     string
	Slug     string  `json:"slug"`
	Title    string  `json:"title"`
	Active   bool    `json:"active,omitempty"`
	Expanded bool    `json:"expanded,omitempty"`
	Nav      []*Node `json:"nav,omitempty"`
}

// NewNode creates a new node with it's path, slug and page title.
func NewNode(path, slug, title string) *Node {
	return &Node{path: path, Slug: slug, Title: title, Nav: make([]*Node, 0)}
}

func (n *Node) getNode(path string) *Node {
	for _, c := range n.Nav {
		if c.path == path {
			return c
		}
	}
	log.Fatalf("Expected nested Node %v in %v\n", path, n.path)
	return &Node{} // cannot happen
}

func (n *Node) hasNode(path string) bool {
	for _, c := range n.Nav {
		if c.path == path {
			return true
		}
	}
	return false
}

// AddNode adds a node to the node tree.
func (n *Node) AddNode(root []string, p string, title string, paths []string, active bool, expanded bool) {
	// Yay! Create us!
	if len(paths) == 0 {
		n.Active = active
		n.Expanded = expanded
		n.Title = title
		n.Slug = p
		return
	}
	// Parent folder.
	parent := paths[0]

	// Have we already created the parent?
	if n.hasNode(parent) {
		if len(root) == 0 {
			return
		}
		if root[0] == parent {
			n.getNode(parent).AddNode(root[1:], p, title, paths[1:], false, false)
		} else if len(paths) == 1 {
			n.getNode(parent).AddNode(root, p, title, []string{}, false, false)
		}
		return
	}
	// Create it and move on.
	n.Nav = append(n.Nav, NewNode(parent, p, title))
	n.getNode(parent).AddNode(root, p, title, paths[1:], len(root) == 1 && root[0] == parent, len(root) > 1 && root[0] == parent)
}

// Num returns the recursive number of pages under this node.
func (n *Node) Num() int {
	sum := 1
	for _, c := range n.Nav {
		sum += c.Num()
	}
	return sum
}

// Load intializes a root directory and serves all sub-folders.
func Load(root string) (pages map[string]*Resp, err error) {
	var dirs []string
	err = filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
		// We only search for article directories.
		if !fi.IsDir() {
			return nil
		}

		// Ignore our .git folder.
		if fi.IsDir() && fi.Name() == ".git" {
			return filepath.SkipDir
		}
		dirs = append(dirs, path)
		return nil
	})
	if err != nil {
		return nil, err
	}
	return parseDirs(root, dirs)
}

// stripRoot removes root level of a directory.
// This is because when a user requests:
// `/sektionen/om-oss` the actual path is: `root/sektionen/om-oss`
func stripRoot(root string, dir string) string {
	return filepath.Clean(strings.Replace(dir, root, "/", 1))
}

// parseDirs parses each directory into a response. Returns a map from requested
// urls into responses.
func parseDirs(root string, dirs []string) (pages map[string]*Resp, err error) {
	pages = map[string]*Resp{}
	for _, dir := range dirs {
		r, err := parseDir(root, dir)
		if err != nil {
			log.Warnln(err)
			return nil, err
		}
		pages[stripRoot(root, dir)] = r
		log.WithFields(log.Fields{
			"Resp": r,
			"dir":  dir,
		}).Debug("Our parsed response\n")
	}
	return pages, nil
}

// toHTML reads a markdown file and returns a HTML string.
func toHTML(filename string) (string, error) {
	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		return "", err
	}
	// Use standard HTML rendering.
	renderer := blackfriday.HtmlRenderer(blackfriday.HTML_USE_XHTML, "", "")
	// Parse markdown where all id's are created from the values inside
	// the element tag.
	buf = blackfriday.MarkdownOptions(buf, renderer, blackfriday.Options{
		Extensions: blackfriday.EXTENSION_AUTO_HEADER_IDS | blackfriday.EXTENSION_TABLES | blackfriday.EXTENSION_FENCED_CODE,
	})
	return string(buf), nil
}

// parseDir creates a response for a directory.
func parseDir(root, dir string) (*Resp, error) {
	log.WithField("dir", dir).Debug("Parsing directory:")

	// Our content files.
	var (
		bodyPath    = filepath.Join(dir, "body.md")
		sidebarPath = filepath.Join(dir, "sidebar.md")
		metaPath    = filepath.Join(dir, "meta.toml")
	)

	// Parse markdown to HTML.
	body, err := toHTML(bodyPath)
	if err != nil {
		return nil, err
	}
	log.WithField("body", body).Debug("HTML of body.md")

	// Parse sidebar to HTML.
	sidebar, err := toHTML(sidebarPath)
	if err != nil {
		return nil, err
	}
	log.WithField("sidebar", sidebar).Debug("HTML of sidebar.md")

	// Parse modified at.
	fi, err := os.Stat(bodyPath)
	if err != nil {
		return nil, err
	}
	// Parse anchors in the body.
	anchs, err := anchor.Anchors(body)
	if err != nil {
		return nil, err
	}

	// Parse meta data from a toml file.
	var meta struct {
		Image   string
		Title   string
		Message string
	}
	if _, err := toml.DecodeFile(metaPath, &meta); err != nil {
		return nil, err
	}

	const iso8601DateTime = "2006-01-02T15:04:05Z"
	return &Resp{
		Title:     meta.Title,
		Slug:      filepath.Base(stripRoot(root, dir)),
		UpdatedAt: fi.ModTime().Format(iso8601DateTime),
		Image:     meta.Image,
		Message:   meta.Message,
		Body:      body,
		Sidebar:   sidebar,
		Anchors:   anchs,
	}, nil
}
<commit_msg>Increase the range of ignored pages to include all hidden directories<commit_after>package pages

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/datasektionen/taitan/anchor"

	"github.com/BurntSushi/toml"
	"github.com/russross/blackfriday"
	log "github.com/sirupsen/logrus"
)

// Resp is the response we serve for file queries.
type Resp struct {
	Title     string          `json:"title"` // Human-readable title.
	Slug      string          `json:"slug"`  // URL-slug.
	URL       string          `json:"url"`
	UpdatedAt string          `json:"updated_at"` // Body update time.
	Image     string          `json:"image"`      // Path/URL/Placeholder to image.
	Message   string          `json:"message"`    // Message to show at top
	Body      string          `json:"body"`       // Main content of the page.
	Sidebar   string          `json:"sidebar"`    // The sidebar of the page.
	Anchors   []anchor.Anchor `json:"anchors"`    // The list of anchors to headers in the body.
	Nav       []*Node         `json:"nav,omitempty"`
}

// Node is a recursive node in a page tree.
type Node struct {
	path     string
	Slug     string  `json:"slug"`
	Title    string  `json:"title"`
	Active   bool    `json:"active,omitempty"`
	Expanded bool    `json:"expanded,omitempty"`
	Nav      []*Node `json:"nav,omitempty"`
}

// NewNode creates a new node with it's path, slug and page title.
func NewNode(path, slug, title string) *Node {
	return &Node{path: path, Slug: slug, Title: title, Nav: make([]*Node, 0)}
}

func (n *Node) getNode(path string) *Node {
	for _, c := range n.Nav {
		if c.path == path {
			return c
		}
	}
	log.Fatalf("Expected nested Node %v in %v\n", path, n.path)
	return &Node{} // cannot happen
}

func (n *Node) hasNode(path string) bool {
	for _, c := range n.Nav {
		if c.path == path {
			return true
		}
	}
	return false
}

// AddNode adds a node to the node tree.
func (n *Node) AddNode(root []string, p string, title string, paths []string, active bool, expanded bool) {
	// Yay! Create us!
	if len(paths) == 0 {
		n.Active = active
		n.Expanded = expanded
		n.Title = title
		n.Slug = p
		return
	}
	// Parent folder.
	parent := paths[0]

	// Have we already created the parent?
	if n.hasNode(parent) {
		if len(root) == 0 {
			return
		}
		if root[0] == parent {
			n.getNode(parent).AddNode(root[1:], p, title, paths[1:], false, false)
		} else if len(paths) == 1 {
			n.getNode(parent).AddNode(root, p, title, []string{}, false, false)
		}
		return
	}
	// Create it and move on.
	n.Nav = append(n.Nav, NewNode(parent, p, title))
	n.getNode(parent).AddNode(root, p, title, paths[1:], len(root) == 1 && root[0] == parent, len(root) > 1 && root[0] == parent)
}

// Num returns the recursive number of pages under this node.
func (n *Node) Num() int {
	sum := 1
	for _, c := range n.Nav {
		sum += c.Num()
	}
	return sum
}

// Load intializes a root directory and serves all sub-folders.
func Load(root string) (pages map[string]*Resp, err error) {
	var dirs []string
	err = filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
		// We only search for article directories.
		if !fi.IsDir() {
			return nil
		}

		// Ignore our .git folder.
		if fi.IsDir() && fi.Name()[0] == '.' {
			return filepath.SkipDir
		}
		dirs = append(dirs, path)
		return nil
	})
	if err != nil {
		return nil, err
	}
	return parseDirs(root, dirs)
}

// stripRoot removes root level of a directory.
// This is because when a user requests:
// `/sektionen/om-oss` the actual path is: `root/sektionen/om-oss`
func stripRoot(root string, dir string) string {
	return filepath.Clean(strings.Replace(dir, root, "/", 1))
}

// parseDirs parses each directory into a response. Returns a map from requested
// urls into responses.
func parseDirs(root string, dirs []string) (pages map[string]*Resp, err error) {
	pages = map[string]*Resp{}
	for _, dir := range dirs {
		r, err := parseDir(root, dir)
		if err != nil {
			log.Warnln(err)
			return nil, err
		}
		pages[stripRoot(root, dir)] = r
		log.WithFields(log.Fields{
			"Resp": r,
			"dir":  dir,
		}).Debug("Our parsed response\n")
	}
	return pages, nil
}

// toHTML reads a markdown file and returns a HTML string.
func toHTML(filename string) (string, error) {
	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		return "", err
	}
	// Use standard HTML rendering.
	renderer := blackfriday.HtmlRenderer(blackfriday.HTML_USE_XHTML, "", "")
	// Parse markdown where all id's are created from the values inside
	// the element tag.
	buf = blackfriday.MarkdownOptions(buf, renderer, blackfriday.Options{
		Extensions: blackfriday.EXTENSION_AUTO_HEADER_IDS | blackfriday.EXTENSION_TABLES | blackfriday.EXTENSION_FENCED_CODE,
	})
	return string(buf), nil
}

// parseDir creates a response for a directory.
func parseDir(root, dir string) (*Resp, error) {
	log.WithField("dir", dir).Debug("Parsing directory:")

	// Our content files.
	var (
		bodyPath    = filepath.Join(dir, "body.md")
		sidebarPath = filepath.Join(dir, "sidebar.md")
		metaPath    = filepath.Join(dir, "meta.toml")
	)

	// Parse markdown to HTML.
	body, err := toHTML(bodyPath)
	if err != nil {
		return nil, err
	}
	log.WithField("body", body).Debug("HTML of body.md")

	// Parse sidebar to HTML.
	sidebar, err := toHTML(sidebarPath)
	if err != nil {
		return nil, err
	}
	log.WithField("sidebar", sidebar).Debug("HTML of sidebar.md")

	// Parse modified at.
	fi, err := os.Stat(bodyPath)
	if err != nil {
		return nil, err
	}
	// Parse anchors in the body.
	anchs, err := anchor.Anchors(body)
	if err != nil {
		return nil, err
	}

	// Parse meta data from a toml file.
	var meta struct {
		Image   string
		Title   string
		Message string
	}
	if _, err := toml.DecodeFile(metaPath, &meta); err != nil {
		return nil, err
	}

	const iso8601DateTime = "2006-01-02T15:04:05Z"
	return &Resp{
		Title:     meta.Title,
		Slug:      filepath.Base(stripRoot(root, dir)),
		UpdatedAt: fi.ModTime().Format(iso8601DateTime),
		Image:     meta.Image,
		Message:   meta.Message,
		Body:      body,
		Sidebar:   sidebar,
		Anchors:   anchs,
	}, nil
}
<|endoftext|>


<commit_before>//pansi provides functions to aid in parsing ansi escape code
//All rules for parsing are pulled from:
//http://ascii-table.com/ansi-escape-sequences.php
//http://www.vt100.net/emu/dec_ansi_parser
package pansi

type AnsiEscapeType int

const (
	Invalid AnsiEscapeType = iota
	CursorPosition
	CursorUp
	CursorDown
	CursorForward
	CursorBackward
	SaveCursorPosition
	RestoreCursorPosition
	EraseDisplay
	EraseLine
	SetGraphicMode
	SetMode
	ResetMode
	SetKeyboardStrings
)

type State interface {
	Entry(*Parser)
	Execute(*Parser, byte) (State, *AnsiEscapeCode)
	Exit(*Parser)
}

type (
	nullState  struct{}
	clearEntry struct{ nullState }

	AnsiEscapeCode struct {
		Type   AnsiEscapeType
		Values []int
		// Params []byte
	}

	Parser struct {
		state  State
		mode   byte
		params []byte
		result *AnsiEscapeCode
	}
)

func (s nullState) Entry(_ *Parser)  {}
func (s nullState) Exit(_ *Parser)   {}
func (s clearEntry) Entry(p *Parser) { p.Clear() }

func (p *Parser) Clear() {
	p.params = nil
	p.mode = 0x0
	p.state = nil
}

func (p *Parser) Feed(b byte) {
	switch b {
	case 0x1B:
		p.state, p.result = &escape{}, nil
	default:
		if p.state != nil {
			p.state, p.result = p.state.Execute(p, b)
		}
	}
}

func (p *Parser) FeedRune(r rune) {
	p.Feed(byte(r))
}

func (p *Parser) Collect(b byte) {
	p.mode = b
}

func (p *Parser) Param(b byte) {
	p.params = append(p.params, b)
}

func (p *Parser) Result() *AnsiEscapeCode {
	return p.result
}

func NewParser() Parser {
	return Parser{nil, 0x0, make([]byte, 0, 16), nil}
}

func Parse(s string) *AnsiEscapeCode {
	p := NewParser()
	for _, ch := range s {
		p.FeedRune(ch)
	}
	return p.Result()
}
<commit_msg>result needs to be cleared out as well<commit_after>//pansi provides functions to aid in parsing ansi escape code
//All rules for parsing are pulled from:
//http://ascii-table.com/ansi-escape-sequences.php
//http://www.vt100.net/emu/dec_ansi_parser
package pansi

type AnsiEscapeType int

const (
	Invalid AnsiEscapeType = iota
	CursorPosition
	CursorUp
	CursorDown
	CursorForward
	CursorBackward
	SaveCursorPosition
	RestoreCursorPosition
	EraseDisplay
	EraseLine
	SetGraphicMode
	SetMode
	ResetMode
	SetKeyboardStrings
)

type State interface {
	Entry(*Parser)
	Execute(*Parser, byte) (State, *AnsiEscapeCode)
	Exit(*Parser)
}

type (
	nullState  struct{}
	clearEntry struct{ nullState }

	AnsiEscapeCode struct {
		Type   AnsiEscapeType
		Values []int
		// Params []byte
	}

	Parser struct {
		state  State
		mode   byte
		params []byte
		result *AnsiEscapeCode
	}
)

func (s nullState) Entry(_ *Parser)  {}
func (s nullState) Exit(_ *Parser)   {}
func (s clearEntry) Entry(p *Parser) { p.Clear() }

func (p *Parser) Clear() {
	p.params = nil
	p.mode = 0x0
	p.state = nil
	p.result = nil
}

func (p *Parser) Feed(b byte) {
	switch b {
	case 0x1B:
		p.state, p.result = &escape{}, nil
	default:
		if p.state != nil {
			p.state, p.result = p.state.Execute(p, b)
		}
	}
}

func (p *Parser) FeedRune(r rune) {
	p.Feed(byte(r))
}

func (p *Parser) Collect(b byte) {
	p.mode = b
}

func (p *Parser) Param(b byte) {
	p.params = append(p.params, b)
}

func (p *Parser) Result() *AnsiEscapeCode {
	return p.result
}

func NewParser() Parser {
	return Parser{nil, 0x0, make([]byte, 0, 16), nil}
}

func Parse(s string) *AnsiEscapeCode {
	p := NewParser()
	for _, ch := range s {
		p.FeedRune(ch)
	}
	return p.Result()
}
<|endoftext|>


<commit_before>package parse_graphql

import (
	"encoding/json"
	"fmt"

	"github.com/tmc/graphql"
	"github.com/tmc/graphql/executor/resolver"
	"github.com/tmc/graphql/schema"
	"github.com/tmc/parse"
)

type ParseClass struct {
	client *parse.Client
	class  *parse.Schema
	schema map[string]*parse.Schema
	Data   map[string]interface{}
}

func NewParseClass(client *parse.Client, className string, schema map[string]*parse.Schema) (*ParseClass, error) {
	class, ok := schema[className]
	if !ok {
		return nil, fmt.Errorf("class '%s' not found in schema.", className)
	}
	return &ParseClass{
		client: client,
		class:  class,
		schema: schema,
	}, nil
}

func (p *ParseClass) GraphQLTypeInfo() schema.GraphQLTypeInfo {
	className := p.class.ClassName
	ti := schema.GraphQLTypeInfo{
		Name:        p.class.ClassName,
		Description: fmt.Sprintf("Parse Class %v", className),
		Fields: schema.GraphQLFieldSpecMap{
			className: &schema.GraphQLFieldSpec{
				Name:        p.class.ClassName,
				Description: fmt.Sprintf("Root call to fetch %s", className),
				Func:        p.get,
				IsRootCall:  true,
			},
		},
	}

	// generate basic value accessors
	for fieldName, fieldSchema := range p.class.Fields {
		fn := fieldName
		ti.Fields[fieldName] = &schema.GraphQLFieldSpec{
			Name:        fn,
			Description: fmt.Sprintf("Accessor for %s field (%v)", fn, fieldSchema.Type),
			Func: func(r resolver.Resolver, f *graphql.Field) (interface{}, error) {
				partial, err := p.resolve(r, fn)
				if err != nil {
					return nil, err
				}
				return r.Resolve(partial, f)
			},
		}
	}
	return ti
}

func (p *ParseClass) resolve(r resolver.Resolver, fieldName string) (interface{}, error) {
	fieldInfo := p.class.Fields[fieldName]
	if fieldInfo.Type == "Pointer" {
		pc, err := NewParseClass(p.client, fieldInfo.TargetClass, p.schema)
		if err != nil {
			return nil, err
		}
		objectID := p.Data[fieldName].(map[string]interface{})["objectId"].(string)
		err = pc.client.GetClass(fieldInfo.TargetClass, objectID, &pc.Data)
		return pc, err
	} else if fieldInfo.Type == "ReversePointer" {
		pc, err := NewParseClass(p.client, fieldInfo.TargetClass, p.schema)
		fieldName := fieldName[len(fieldInfo.TargetClass)+1:]
		// TODO(tmc): optimize
		query := []byte(fmt.Sprintf(`{"__type":"Pointer","className":"%s","objectId":"%s"}`,
			p.class.ClassName,
			p.Data["objectId"]))
		queryEncoded := json.RawMessage(query)
		return pc.get(r, &graphql.Field{
			Arguments: []graphql.Argument{
				{
					Name:  fieldName,
					Value: &queryEncoded,
				},
			},
		})
		if err != nil {
			return nil, err
		}
		objectID := p.Data[fieldName].(map[string]interface{})["objectId"].(string)
		err = pc.client.GetClass(fieldInfo.TargetClass, objectID, &pc.Data)
		return pc, err
	} else {
		return p.Data[fieldName], nil
	}
}

func (p *ParseClass) get(r resolver.Resolver, f *graphql.Field) (interface{}, error) {
	var results []map[string]interface{}

	whereClause := make(map[string]interface{})
	for _, a := range f.Arguments {
		whereClause[a.Name] = a.Value
	}
	whereJSON, err := json.Marshal(whereClause)
	if err != nil {
		return nil, err
	}
	query := &parse.QueryOptions{
		Where: string(whereJSON),
	}
	if err := p.client.QueryClass(p.class.ClassName, query, &results); err != nil {
		return nil, err
	}
	typedResults := make([]*ParseClass, 0, len(results))

	for _, r := range results {
		pc, err := NewParseClass(p.client, p.class.ClassName, p.schema)
		if err != nil {
			return nil, err
		}
		pc.Data = r
		typedResults = append(typedResults, pc)
	}

	return typedResults, err
}
<commit_msg>reflect new upstream api<commit_after>package parse_graphql

import (
	"encoding/json"
	"fmt"

	"github.com/tmc/graphql"
	"github.com/tmc/graphql/executor/resolver"
	"github.com/tmc/graphql/schema"
	"github.com/tmc/parse"
)

type ParseClass struct {
	client *parse.Client
	class  *parse.Schema
	schema map[string]*parse.Schema
	Data   map[string]interface{}
}

func NewParseClass(client *parse.Client, className string, schema map[string]*parse.Schema) (*ParseClass, error) {
	class, ok := schema[className]
	if !ok {
		return nil, fmt.Errorf("class '%s' not found in schema.", className)
	}
	return &ParseClass{
		client: client,
		class:  class,
		schema: schema,
	}, nil
}

func (p *ParseClass) GraphQLTypeInfo() schema.GraphQLTypeInfo {
	className := p.class.ClassName
	ti := schema.GraphQLTypeInfo{
		Name:        p.class.ClassName,
		Description: fmt.Sprintf("Parse Class %v", className),
		Fields: schema.GraphQLFieldSpecMap{
			className: &schema.GraphQLFieldSpec{
				Name:        p.class.ClassName,
				Description: fmt.Sprintf("Root call to fetch %s", className),
				Func:        p.get,
				IsRoot:      true,
			},
		},
	}

	// generate basic value accessors
	for fieldName, fieldSchema := range p.class.Fields {
		fn := fieldName
		ti.Fields[fieldName] = &schema.GraphQLFieldSpec{
			Name:        fn,
			Description: fmt.Sprintf("Accessor for %s field (%v)", fn, fieldSchema.Type),
			Func: func(r resolver.Resolver, f *graphql.Field) (interface{}, error) {
				partial, err := p.resolve(r, fn)
				if err != nil {
					return nil, err
				}
				return r.Resolve(partial, f)
			},
		}
	}
	return ti
}

func (p *ParseClass) resolve(r resolver.Resolver, fieldName string) (interface{}, error) {
	fieldInfo := p.class.Fields[fieldName]
	if fieldInfo.Type == "Pointer" {
		pc, err := NewParseClass(p.client, fieldInfo.TargetClass, p.schema)
		if err != nil {
			return nil, err
		}
		objectID := p.Data[fieldName].(map[string]interface{})["objectId"].(string)
		err = pc.client.GetClass(fieldInfo.TargetClass, objectID, &pc.Data)
		return pc, err
	} else if fieldInfo.Type == "ReversePointer" {
		pc, err := NewParseClass(p.client, fieldInfo.TargetClass, p.schema)
		fieldName := fieldName[len(fieldInfo.TargetClass)+1:]
		// TODO(tmc): optimize
		query := []byte(fmt.Sprintf(`{"__type":"Pointer","className":"%s","objectId":"%s"}`,
			p.class.ClassName,
			p.Data["objectId"]))
		queryEncoded := json.RawMessage(query)
		return pc.get(r, &graphql.Field{
			Arguments: []graphql.Argument{
				{
					Name:  fieldName,
					Value: &queryEncoded,
				},
			},
		})
		if err != nil {
			return nil, err
		}
		objectID := p.Data[fieldName].(map[string]interface{})["objectId"].(string)
		err = pc.client.GetClass(fieldInfo.TargetClass, objectID, &pc.Data)
		return pc, err
	} else {
		return p.Data[fieldName], nil
	}
}

func (p *ParseClass) get(r resolver.Resolver, f *graphql.Field) (interface{}, error) {
	var results []map[string]interface{}

	whereClause := make(map[string]interface{})
	for _, a := range f.Arguments {
		whereClause[a.Name] = a.Value
	}
	whereJSON, err := json.Marshal(whereClause)
	if err != nil {
		return nil, err
	}
	query := &parse.QueryOptions{
		Where: string(whereJSON),
	}
	if err := p.client.QueryClass(p.class.ClassName, query, &results); err != nil {
		return nil, err
	}
	typedResults := make([]*ParseClass, 0, len(results))

	for _, r := range results {
		pc, err := NewParseClass(p.client, p.class.ClassName, p.schema)
		if err != nil {
			return nil, err
		}
		pc.Data = r
		typedResults = append(typedResults, pc)
	}

	return typedResults, err
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"

	lib "github.com/bazooka-ci/bazooka/commons"
	"github.com/bazooka-ci/bazooka/commons/mongo"
	docker "github.com/bywan/go-dockercommand"
)

const (
	buildFolderPattern = "%s/build/%s/%s"     // $bzk_home/build/$projectId/$buildId
	logFolderPattern   = "%s/build/%s/%s/log" // $bzk_home/build/$projectId/$buildId/log
)

func (c *context) startBitbucketJob(params map[string]string, body bodyFunc) (*response, error) {
	var bitbucketPayload BitbucketPayload

	body(&bitbucketPayload)

	if len(bitbucketPayload.Commits) == 0 {
		return badRequest("no commit found in Bitbucket payload")
	}

	//TODO(julienvey) Order by timestamp to find the last commit instead of trusting
	// Bitbucket to give us the commits in the right order

	if len(bitbucketPayload.Commits[0].RawNode) == 0 {
		return badRequest("RawNode is empty in Bitbucket payload")
	}

	return c.startJob(params, lib.StartJob{
		ScmReference: bitbucketPayload.Commits[0].RawNode,
	})

}

func (c *context) startGithubJob(params map[string]string, body bodyFunc) (*response, error) {
	var githubPayload GithubPayload

	body(&githubPayload)

	if len(githubPayload.HeadCommit.ID) == 0 {
		return badRequest("HeadCommit is empty in Github payload")
	}

	return c.startJob(params, lib.StartJob{
		ScmReference: githubPayload.HeadCommit.ID,
	})

}

func (c *context) startStandardJob(params map[string]string, body bodyFunc) (*response, error) {

	var startJob lib.StartJob

	body(&startJob)

	if len(startJob.ScmReference) == 0 {
		return badRequest("reference is mandatory")
	}

	return c.startJob(params, startJob)
}

func (c *context) startJob(params map[string]string, startJob lib.StartJob) (*response, error) {

	project, err := c.Connector.GetProjectById(params["id"])
	if err != nil {
		if err.Error() != "not found" {
			return nil, err
		}
		return notFound("project not found")
	}

	client, err := docker.NewDocker(c.DockerEndpoint)
	if err != nil {
		return nil, err
	}

	orchestrationImage, err := c.Connector.GetImage("orchestration")
	if err != nil {
		return nil, &errorResponse{500, fmt.Sprintf("Failed to retrieve the orchestration image: %v", err)}
	}

	runningJob := &lib.Job{
		ProjectID:  project.ID,
		Started:    time.Now(),
		Parameters: startJob.Parameters,
	}

	if err := c.Connector.AddJob(runningJob); err != nil {
		return nil, err
	}

	for _, v := range startJob.Parameters {
		if !strings.Contains(v, "=") {
			return nil, &errorResponse{400, fmt.Sprintf("Environment variable %v is empty", v)}
		}
	}
	jobParameters, err := json.Marshal(startJob.Parameters)
	if err != nil {
		return nil, err
	}

	buildFolder := fmt.Sprintf(buildFolderPattern, c.Env[BazookaEnvHome], runningJob.ProjectID, runningJob.ID)
	orchestrationEnv := map[string]string{
		"BZK_SCM":            project.ScmType,
		"BZK_SCM_URL":        project.ScmURI,
		"BZK_SCM_REFERENCE":  startJob.ScmReference,
		"BZK_HOME":           buildFolder,
		"BZK_PROJECT_ID":     project.ID,
		"BZK_JOB_ID":         runningJob.ID,
		"BZK_DOCKERSOCK":     c.Env[BazookaEnvDockerSock],
		"BZK_JOB_PARAMETERS": string(jobParameters),
		BazookaEnvMongoAddr:  c.Env[BazookaEnvMongoAddr],
		BazookaEnvMongoPort:  c.Env[BazookaEnvMongoPort],
	}

	buildFolderLocal := fmt.Sprintf(buildFolderPattern, "/bazooka", runningJob.ProjectID, runningJob.ID)

	projectSSHKey, err := c.Connector.GetProjectKey(project.ID)
	if err != nil {
		_, keyNotFound := err.(*mongo.NotFoundError)
		if !keyNotFound {
			return nil, err
		}
		//Use Global Key if provided
		if len(c.Env[BazookaEnvSCMKeyfile]) > 0 {
			orchestrationEnv["BZK_SCM_KEYFILE"] = c.Env[BazookaEnvSCMKeyfile]
		}
	} else {
		err = os.MkdirAll(buildFolderLocal, 0644)
		if err != nil {
			return nil, err
		}

		err = ioutil.WriteFile(fmt.Sprintf("%s/key", buildFolderLocal), []byte(projectSSHKey.Content), 0600)
		if err != nil {
			return nil, err
		}
		orchestrationEnv["BZK_SCM_KEYFILE"] = fmt.Sprintf("%s/key", buildFolder)
	}

	projectCryptoKey, err := c.Connector.GetProjectCryptoKey(project.ID)

	if err != nil {
		_, keyNotFound := err.(*mongo.NotFoundError)
		if !keyNotFound {
			return nil, err
		}
	} else {
		err = os.MkdirAll(buildFolderLocal, 0644)
		if err != nil {
			return nil, err
		}

		err = ioutil.WriteFile(fmt.Sprintf("%s/crypto-key", buildFolderLocal), []byte(projectCryptoKey.Content), 0600)
		if err != nil {
			return nil, err
		}
		orchestrationEnv["BZK_CRYPTO_KEYFILE"] = fmt.Sprintf("%s/crypto-key", buildFolder)
	}

	container, err := client.Run(&docker.RunOptions{
		Image: orchestrationImage,
		VolumeBinds: []string{
			fmt.Sprintf("%s:/bazooka", buildFolder),
			fmt.Sprintf("%s:/var/run/docker.sock", c.Env[BazookaEnvDockerSock]),
		},
		Env:    orchestrationEnv,
		Detach: true,
	})

	runningJob.OrchestrationID = container.ID()
	log.WithFields(log.Fields{
		"job_id":           runningJob.ID,
		"project_id":       runningJob.ProjectID,
		"orchestration_id": runningJob.OrchestrationID,
	}).Info("Starting job")

	err = c.Connector.SetJobOrchestrationId(runningJob.ID, container.ID())
	if err != nil {
		log.Error(err.Error())
		return nil, err
	}

	return accepted(runningJob, "/job/"+runningJob.ID)
}

func (c *context) getJob(params map[string]string, body bodyFunc) (*response, error) {

	job, err := c.Connector.GetJobByID(params["id"])
	if err != nil {
		if err.Error() != "not found" {
			return nil, err
		}
		return notFound("job not found")
	}

	return ok(&job)
}

func (c *context) getJobs(params map[string]string, body bodyFunc) (*response, error) {

	jobs, err := c.Connector.GetJobs(params["id"])
	if err != nil {
		return nil, err
	}

	return ok(&jobs)
}

func (c *context) getAllJobs(params map[string]string, body bodyFunc) (*response, error) {

	jobs, err := c.Connector.GetAllJobs()
	if err != nil {
		return nil, err
	}

	return ok(&jobs)
}

func (c *context) getJobLog(params map[string]string, body bodyFunc) (*response, error) {

	log, err := c.Connector.GetLog(&mongo.LogExample{
		JobID: params["id"],
	})
	if err != nil {
		if err.Error() != "not found" {
			return nil, err
		}
		return notFound("log not found")
	}

	return ok(&log)
}
<commit_msg>Clean containers at the end of theirs execution<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"

	lib "github.com/bazooka-ci/bazooka/commons"
	"github.com/bazooka-ci/bazooka/commons/mongo"
	docker "github.com/bywan/go-dockercommand"
)

const (
	buildFolderPattern = "%s/build/%s/%s"     // $bzk_home/build/$projectId/$buildId
	logFolderPattern   = "%s/build/%s/%s/log" // $bzk_home/build/$projectId/$buildId/log
)

func (c *context) startBitbucketJob(params map[string]string, body bodyFunc) (*response, error) {
	var bitbucketPayload BitbucketPayload

	body(&bitbucketPayload)

	if len(bitbucketPayload.Commits) == 0 {
		return badRequest("no commit found in Bitbucket payload")
	}

	//TODO(julienvey) Order by timestamp to find the last commit instead of trusting
	// Bitbucket to give us the commits in the right order

	if len(bitbucketPayload.Commits[0].RawNode) == 0 {
		return badRequest("RawNode is empty in Bitbucket payload")
	}

	return c.startJob(params, lib.StartJob{
		ScmReference: bitbucketPayload.Commits[0].RawNode,
	})

}

func (c *context) startGithubJob(params map[string]string, body bodyFunc) (*response, error) {
	var githubPayload GithubPayload

	body(&githubPayload)

	if len(githubPayload.HeadCommit.ID) == 0 {
		return badRequest("HeadCommit is empty in Github payload")
	}

	return c.startJob(params, lib.StartJob{
		ScmReference: githubPayload.HeadCommit.ID,
	})

}

func (c *context) startStandardJob(params map[string]string, body bodyFunc) (*response, error) {

	var startJob lib.StartJob

	body(&startJob)

	if len(startJob.ScmReference) == 0 {
		return badRequest("reference is mandatory")
	}

	return c.startJob(params, startJob)
}

func (c *context) startJob(params map[string]string, startJob lib.StartJob) (*response, error) {

	project, err := c.Connector.GetProjectById(params["id"])
	if err != nil {
		if err.Error() != "not found" {
			return nil, err
		}
		return notFound("project not found")
	}

	client, err := docker.NewDocker(c.DockerEndpoint)
	if err != nil {
		return nil, err
	}

	orchestrationImage, err := c.Connector.GetImage("orchestration")
	if err != nil {
		return nil, &errorResponse{500, fmt.Sprintf("Failed to retrieve the orchestration image: %v", err)}
	}

	runningJob := &lib.Job{
		ProjectID:  project.ID,
		Started:    time.Now(),
		Parameters: startJob.Parameters,
	}

	if err := c.Connector.AddJob(runningJob); err != nil {
		return nil, err
	}

	for _, v := range startJob.Parameters {
		if !strings.Contains(v, "=") {
			return nil, &errorResponse{400, fmt.Sprintf("Environment variable %v is empty", v)}
		}
	}
	jobParameters, err := json.Marshal(startJob.Parameters)
	if err != nil {
		return nil, err
	}

	buildFolder := fmt.Sprintf(buildFolderPattern, c.Env[BazookaEnvHome], runningJob.ProjectID, runningJob.ID)
	orchestrationEnv := map[string]string{
		"BZK_SCM":            project.ScmType,
		"BZK_SCM_URL":        project.ScmURI,
		"BZK_SCM_REFERENCE":  startJob.ScmReference,
		"BZK_HOME":           buildFolder,
		"BZK_PROJECT_ID":     project.ID,
		"BZK_JOB_ID":         runningJob.ID,
		"BZK_DOCKERSOCK":     c.Env[BazookaEnvDockerSock],
		"BZK_JOB_PARAMETERS": string(jobParameters),
		BazookaEnvMongoAddr:  c.Env[BazookaEnvMongoAddr],
		BazookaEnvMongoPort:  c.Env[BazookaEnvMongoPort],
	}

	buildFolderLocal := fmt.Sprintf(buildFolderPattern, "/bazooka", runningJob.ProjectID, runningJob.ID)

	projectSSHKey, err := c.Connector.GetProjectKey(project.ID)
	if err != nil {
		_, keyNotFound := err.(*mongo.NotFoundError)
		if !keyNotFound {
			return nil, err
		}
		//Use Global Key if provided
		if len(c.Env[BazookaEnvSCMKeyfile]) > 0 {
			orchestrationEnv["BZK_SCM_KEYFILE"] = c.Env[BazookaEnvSCMKeyfile]
		}
	} else {
		err = os.MkdirAll(buildFolderLocal, 0644)
		if err != nil {
			return nil, err
		}

		err = ioutil.WriteFile(fmt.Sprintf("%s/key", buildFolderLocal), []byte(projectSSHKey.Content), 0600)
		if err != nil {
			return nil, err
		}
		orchestrationEnv["BZK_SCM_KEYFILE"] = fmt.Sprintf("%s/key", buildFolder)
	}

	projectCryptoKey, err := c.Connector.GetProjectCryptoKey(project.ID)

	if err != nil {
		_, keyNotFound := err.(*mongo.NotFoundError)
		if !keyNotFound {
			return nil, err
		}
	} else {
		err = os.MkdirAll(buildFolderLocal, 0644)
		if err != nil {
			return nil, err
		}

		err = ioutil.WriteFile(fmt.Sprintf("%s/crypto-key", buildFolderLocal), []byte(projectCryptoKey.Content), 0600)
		if err != nil {
			return nil, err
		}
		orchestrationEnv["BZK_CRYPTO_KEYFILE"] = fmt.Sprintf("%s/crypto-key", buildFolder)
	}

	container, err := client.Run(&docker.RunOptions{
		Image: orchestrationImage,
		VolumeBinds: []string{
			fmt.Sprintf("%s:/bazooka", buildFolder),
			fmt.Sprintf("%s:/var/run/docker.sock", c.Env[BazookaEnvDockerSock]),
		},
		Env:    orchestrationEnv,
		Detach: true,
	})

	// remove the container at the end of its execution
	go func(container *docker.Container) {
		exitCode, err := container.Wait()
		if err != nil {
			log.Errorf("Error while listening container %s", container.ID, err)
		}

		if exitCode != 0 {
			log.Errorf("Error during execution of Orchestrator container. Check Docker container logs, id is %s\n", container.ID())
			return
		}

		err = container.Remove(&docker.RemoveOptions{
			Force:         true,
			RemoveVolumes: true,
		})
		if err != nil {
			log.Errorf("Cannot remove container %s", container.ID)
		}
	}(container)

	runningJob.OrchestrationID = container.ID()
	log.WithFields(log.Fields{
		"job_id":           runningJob.ID,
		"project_id":       runningJob.ProjectID,
		"orchestration_id": runningJob.OrchestrationID,
	}).Info("Starting job")

	err = c.Connector.SetJobOrchestrationId(runningJob.ID, container.ID())
	if err != nil {
		log.Error(err.Error())
		return nil, err
	}

	return accepted(runningJob, "/job/"+runningJob.ID)
}

func (c *context) getJob(params map[string]string, body bodyFunc) (*response, error) {

	job, err := c.Connector.GetJobByID(params["id"])
	if err != nil {
		if err.Error() != "not found" {
			return nil, err
		}
		return notFound("job not found")
	}

	return ok(&job)
}

func (c *context) getJobs(params map[string]string, body bodyFunc) (*response, error) {

	jobs, err := c.Connector.GetJobs(params["id"])
	if err != nil {
		return nil, err
	}

	return ok(&jobs)
}

func (c *context) getAllJobs(params map[string]string, body bodyFunc) (*response, error) {

	jobs, err := c.Connector.GetAllJobs()
	if err != nil {
		return nil, err
	}

	return ok(&jobs)
}

func (c *context) getJobLog(params map[string]string, body bodyFunc) (*response, error) {

	log, err := c.Connector.GetLog(&mongo.LogExample{
		JobID: params["id"],
	})
	if err != nil {
		if err.Error() != "not found" {
			return nil, err
		}
		return notFound("log not found")
	}

	return ok(&log)
}
<|endoftext|>


<commit_before>package disconnectedclients

import (
	"fmt"
	"testing"
	"time"

	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/nomad/e2e/e2eutil"
	"github.com/hashicorp/nomad/helper/uuid"
	"github.com/hashicorp/nomad/testutil"
	"github.com/stretchr/testify/require"
)

const ns = ""

// typical wait times for this test package
var wait30s = &e2eutil.WaitConfig{Interval: time.Second, Retries: 30}
var wait60s = &e2eutil.WaitConfig{Interval: time.Second, Retries: 60}

type expectedAllocStatus struct {
	disconnected string
	unchanged    string
	replacement  string
}

func TestDisconnectedClients(t *testing.T) {

	nomad := e2eutil.NomadClient(t)
	e2eutil.WaitForLeader(t, nomad)
	e2eutil.WaitForNodesReady(t, nomad, 2) // needs at least 2 to test replacement

	testCases := []struct {
		name                    string
		jobFile                 string
		disconnectFn            func(string, time.Duration) (string, error)
		expectedAfterDisconnect expectedAllocStatus
		expectedAfterReconnect  expectedAllocStatus
	}{
		{
			// test that allocations on clients that are netsplit and
			// marked disconnected are replaced
			name:         "netsplit client no max disconnect",
			jobFile:      "./input/lost_simple.nomad",
			disconnectFn: e2eutil.AgentDisconnect,
			expectedAfterDisconnect: expectedAllocStatus{
				disconnected: "lost",
				unchanged:    "running",
				replacement:  "running",
			},
			expectedAfterReconnect: expectedAllocStatus{
				disconnected: "complete",
				unchanged:    "running",
				replacement:  "running",
			},
		},

		{
			// test that allocations on clients that are netsplit and
			// marked disconnected are replaced but that the
			// replacements are rolled back after reconnection
			name:         "netsplit client with max disconnect",
			jobFile:      "./input/lost_max_disconnect.nomad",
			disconnectFn: e2eutil.AgentDisconnect,
			expectedAfterDisconnect: expectedAllocStatus{
				disconnected: "unknown",
				unchanged:    "running",
				replacement:  "running",
			},
			expectedAfterReconnect: expectedAllocStatus{
				disconnected: "running",
				unchanged:    "running",
				replacement:  "complete",
			},
		},

		{
			// test that allocations on clients that are shutdown and
			// marked disconnected are replaced
			name:         "shutdown client no max disconnect",
			jobFile:      "./input/lost_simple.nomad",
			disconnectFn: e2eutil.AgentDisconnect,
			expectedAfterDisconnect: expectedAllocStatus{
				disconnected: "lost",
				unchanged:    "running",
				replacement:  "running",
			},
			expectedAfterReconnect: expectedAllocStatus{
				disconnected: "complete",
				unchanged:    "running",
				replacement:  "running",
			},
		},

		{
			// test that allocations on clients that are shutdown and
			// marked disconnected are replaced
			name:         "shutdown client with max disconnect",
			jobFile:      "./input/lost_max_disconnect.nomad",
			disconnectFn: e2eutil.AgentDisconnect,
			expectedAfterDisconnect: expectedAllocStatus{
				disconnected: "unknown",
				unchanged:    "running",
				replacement:  "running",
			},
			expectedAfterReconnect: expectedAllocStatus{
				disconnected: "running",
				unchanged:    "running",
				replacement:  "complete",
			},
		},
	}

	for _, tc := range testCases {
		tc := tc
		t.Run(tc.name, func(t *testing.T) {

			jobIDs := []string{}
			t.Cleanup(disconnectedClientsCleanup(t))
			t.Cleanup(e2eutil.CleanupJobsAndGC(t, &jobIDs))

			jobID := "test-disconnected-clients-" + uuid.Short()

			err := e2eutil.Register(jobID, tc.jobFile)
			require.NoError(t, err)
			jobIDs = append(jobIDs, jobID)

			err = e2eutil.WaitForAllocStatusExpected(jobID, ns,
				[]string{"running", "running"})
			require.NoError(t, err, "job should be running")

			// pick one alloc to make our disconnected alloc (and its node)
			allocs, err := e2eutil.AllocsForJob(jobID, ns)
			require.NoError(t, err, "could not query allocs for job")
			require.Len(t, allocs, 2, "could not find 2 allocs for job")

			disconnectedAllocID := allocs[0]["ID"]
			disconnectedNodeID := allocs[0]["Node ID"]
			unchangedAllocID := allocs[1]["ID"]

			// disconnect the node and wait for the results

			restartJobID, err := tc.disconnectFn(disconnectedNodeID, 30*time.Second)
			require.NoError(t, err, "expected agent disconnect job to register")
			jobIDs = append(jobIDs, restartJobID)

			err = e2eutil.WaitForNodeStatus(disconnectedNodeID, "disconnected", wait60s)
			require.NoError(t, err, "expected node to go down")

			require.NoError(t, waitForAllocStatusMap(
				jobID, disconnectedAllocID, unchangedAllocID, tc.expectedAfterDisconnect, wait60s))

			allocs, err = e2eutil.AllocsForJob(jobID, ns)
			require.NoError(t, err, "could not query allocs for job")
			require.Len(t, allocs, 3, "could not find 3 allocs for job")

			// wait for the reconnect and wait for the results

			err = e2eutil.WaitForNodeStatus(disconnectedNodeID, "ready", wait30s)
			require.NoError(t, err, "expected node to come back up")
			require.NoError(t, waitForAllocStatusMap(
				jobID, disconnectedAllocID, unchangedAllocID, tc.expectedAfterReconnect, wait60s))
		})
	}

}

// disconnectedClientsCleanup sets up a cleanup function to make sure
// we've waited for all the nodes to come back up between tests
func disconnectedClientsCleanup(t *testing.T) func() {
	nodeIDs := []string{}
	nodeStatuses, err := e2eutil.NodeStatusList()
	require.NoError(t, err)
	for _, nodeStatus := range nodeStatuses {
		nodeIDs = append(nodeIDs, nodeStatus["ID"])
	}
	return func() {
		nomad := e2eutil.NomadClient(t)
		t.Logf("waiting for %d nodes to become ready again", len(nodeIDs))
		e2eutil.WaitForNodesReady(t, nomad, len(nodeIDs))
	}
}

func waitForAllocStatusMap(jobID, disconnectedAllocID, unchangedAllocID string, expected expectedAllocStatus, wc *e2eutil.WaitConfig) error {
	var err error
	interval, retries := wc.OrDefault()
	testutil.WaitForResultRetries(retries, func() (bool, error) {
		time.Sleep(interval)
		allocs, err := e2eutil.AllocsForJob(jobID, ns)
		if err != nil {
			return false, err
		}

		var merr *multierror.Error

		for _, alloc := range allocs {
			switch allocID, allocStatus := alloc["ID"], alloc["Status"]; allocID {
			case disconnectedAllocID:
				if allocStatus != expected.disconnected {
					merr = multierror.Append(merr, fmt.Errorf(
						"disconnected alloc should be %q, got %q",
						expected.disconnected, allocStatus))
				}
			case unchangedAllocID:
				if allocStatus != expected.unchanged {
					merr = multierror.Append(merr, fmt.Errorf(
						"unchanged alloc should be %q, got %q",
						expected.unchanged, allocStatus))
				}
			default:
				if allocStatus != expected.replacement {
					merr = multierror.Append(merr, fmt.Errorf(
						"replacement alloc should be %q, got %q",
						expected.replacement, allocStatus))
				}
			}
		}
		if merr != nil {
			fmt.Printf("test failed, printing allocation status of all %q allocs for analysis\n", jobID)
			fmt.Println("----------------")
			for _, alloc := range allocs {
				out, _ := e2eutil.Command("nomad", "alloc", "status", alloc["ID"])
				fmt.Println(out)
				fmt.Println("----------------")
			}
			return false, merr.ErrorOrNil()
		}

		return true, nil
	}, func(e error) {
		err = e
	})
	return err
}
<commit_msg>E2E: fix debug logging on disconnected clients test (#12621)<commit_after>package disconnectedclients

import (
	"fmt"
	"testing"
	"time"

	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/nomad/e2e/e2eutil"
	"github.com/hashicorp/nomad/helper/uuid"
	"github.com/hashicorp/nomad/testutil"
	"github.com/stretchr/testify/require"
)

const ns = ""

// typical wait times for this test package
var wait30s = &e2eutil.WaitConfig{Interval: time.Second, Retries: 30}
var wait60s = &e2eutil.WaitConfig{Interval: time.Second, Retries: 60}

type expectedAllocStatus struct {
	disconnected string
	unchanged    string
	replacement  string
}

func TestDisconnectedClients(t *testing.T) {

	nomad := e2eutil.NomadClient(t)
	e2eutil.WaitForLeader(t, nomad)
	e2eutil.WaitForNodesReady(t, nomad, 2) // needs at least 2 to test replacement

	testCases := []struct {
		name                    string
		jobFile                 string
		disconnectFn            func(string, time.Duration) (string, error)
		expectedAfterDisconnect expectedAllocStatus
		expectedAfterReconnect  expectedAllocStatus
	}{
		{
			// test that allocations on clients that are netsplit and
			// marked disconnected are replaced
			name:         "netsplit client no max disconnect",
			jobFile:      "./input/lost_simple.nomad",
			disconnectFn: e2eutil.AgentDisconnect,
			expectedAfterDisconnect: expectedAllocStatus{
				disconnected: "lost",
				unchanged:    "running",
				replacement:  "running",
			},
			expectedAfterReconnect: expectedAllocStatus{
				disconnected: "complete",
				unchanged:    "running",
				replacement:  "running",
			},
		},

		{
			// test that allocations on clients that are netsplit and
			// marked disconnected are replaced but that the
			// replacements are rolled back after reconnection
			name:         "netsplit client with max disconnect",
			jobFile:      "./input/lost_max_disconnect.nomad",
			disconnectFn: e2eutil.AgentDisconnect,
			expectedAfterDisconnect: expectedAllocStatus{
				disconnected: "unknown",
				unchanged:    "running",
				replacement:  "running",
			},
			expectedAfterReconnect: expectedAllocStatus{
				disconnected: "running",
				unchanged:    "running",
				replacement:  "complete",
			},
		},

		{
			// test that allocations on clients that are shutdown and
			// marked disconnected are replaced
			name:         "shutdown client no max disconnect",
			jobFile:      "./input/lost_simple.nomad",
			disconnectFn: e2eutil.AgentDisconnect,
			expectedAfterDisconnect: expectedAllocStatus{
				disconnected: "lost",
				unchanged:    "running",
				replacement:  "running",
			},
			expectedAfterReconnect: expectedAllocStatus{
				disconnected: "complete",
				unchanged:    "running",
				replacement:  "running",
			},
		},

		{
			// test that allocations on clients that are shutdown and
			// marked disconnected are replaced
			name:         "shutdown client with max disconnect",
			jobFile:      "./input/lost_max_disconnect.nomad",
			disconnectFn: e2eutil.AgentDisconnect,
			expectedAfterDisconnect: expectedAllocStatus{
				disconnected: "unknown",
				unchanged:    "running",
				replacement:  "running",
			},
			expectedAfterReconnect: expectedAllocStatus{
				disconnected: "running",
				unchanged:    "running",
				replacement:  "complete",
			},
		},
	}

	for _, tc := range testCases {
		tc := tc
		t.Run(tc.name, func(t *testing.T) {

			jobIDs := []string{}
			t.Cleanup(disconnectedClientsCleanup(t))
			t.Cleanup(e2eutil.CleanupJobsAndGC(t, &jobIDs))

			jobID := "test-disconnected-clients-" + uuid.Short()

			err := e2eutil.Register(jobID, tc.jobFile)
			require.NoError(t, err)
			jobIDs = append(jobIDs, jobID)

			err = e2eutil.WaitForAllocStatusExpected(jobID, ns,
				[]string{"running", "running"})
			require.NoError(t, err, "job should be running")

			// pick one alloc to make our disconnected alloc (and its node)
			allocs, err := e2eutil.AllocsForJob(jobID, ns)
			require.NoError(t, err, "could not query allocs for job")
			require.Len(t, allocs, 2, "could not find 2 allocs for job")

			disconnectedAllocID := allocs[0]["ID"]
			disconnectedNodeID := allocs[0]["Node ID"]
			unchangedAllocID := allocs[1]["ID"]

			// disconnect the node and wait for the results

			restartJobID, err := tc.disconnectFn(disconnectedNodeID, 30*time.Second)
			require.NoError(t, err, "expected agent disconnect job to register")
			jobIDs = append(jobIDs, restartJobID)

			err = e2eutil.WaitForNodeStatus(disconnectedNodeID, "disconnected", wait60s)
			require.NoError(t, err, "expected node to go down")

			require.NoError(t, waitForAllocStatusMap(
				jobID, disconnectedAllocID, unchangedAllocID, tc.expectedAfterDisconnect, wait60s))

			allocs, err = e2eutil.AllocsForJob(jobID, ns)
			require.NoError(t, err, "could not query allocs for job")
			require.Len(t, allocs, 3, "could not find 3 allocs for job")

			// wait for the reconnect and wait for the results

			err = e2eutil.WaitForNodeStatus(disconnectedNodeID, "ready", wait30s)
			require.NoError(t, err, "expected node to come back up")
			require.NoError(t, waitForAllocStatusMap(
				jobID, disconnectedAllocID, unchangedAllocID, tc.expectedAfterReconnect, wait60s))
		})
	}

}

// disconnectedClientsCleanup sets up a cleanup function to make sure
// we've waited for all the nodes to come back up between tests
func disconnectedClientsCleanup(t *testing.T) func() {
	nodeIDs := []string{}
	nodeStatuses, err := e2eutil.NodeStatusList()
	require.NoError(t, err)
	for _, nodeStatus := range nodeStatuses {
		nodeIDs = append(nodeIDs, nodeStatus["ID"])
	}
	return func() {
		nomad := e2eutil.NomadClient(t)
		t.Logf("waiting for %d nodes to become ready again", len(nodeIDs))
		e2eutil.WaitForNodesReady(t, nomad, len(nodeIDs))
	}
}

func waitForAllocStatusMap(jobID, disconnectedAllocID, unchangedAllocID string, expected expectedAllocStatus, wc *e2eutil.WaitConfig) error {
	var err error
	interval, retries := wc.OrDefault()
	testutil.WaitForResultRetries(retries, func() (bool, error) {
		time.Sleep(interval)
		allocs, err := e2eutil.AllocsForJob(jobID, ns)
		if err != nil {
			return false, err
		}

		var merr *multierror.Error

		for _, alloc := range allocs {
			switch allocID, allocStatus := alloc["ID"], alloc["Status"]; allocID {
			case disconnectedAllocID:
				if allocStatus != expected.disconnected {
					merr = multierror.Append(merr, fmt.Errorf(
						"disconnected alloc %q on node %q should be %q, got %q",
						allocID, alloc["Node ID"], expected.disconnected, allocStatus))
				}
			case unchangedAllocID:
				if allocStatus != expected.unchanged {
					merr = multierror.Append(merr, fmt.Errorf(
						"unchanged alloc %q on node %q should be %q, got %q",
						allocID, alloc["Node ID"], expected.unchanged, allocStatus))
				}
			default:
				if allocStatus != expected.replacement {
					merr = multierror.Append(merr, fmt.Errorf(
						"replacement alloc %q on node %q should be %q, got %q",
						allocID, alloc["Node ID"], expected.replacement, allocStatus))
				}
			}
		}
		if merr != nil {
			return false, merr.ErrorOrNil()
		}
		return true, nil
	}, func(e error) {
		err = e
	})

	// TODO(tgross): remove this block once this test has stabilized
	if err != nil {
		fmt.Printf("test failed, printing allocation status of all %q allocs for analysis\n", jobID)
		fmt.Println("----------------")
		allocs, _ := e2eutil.AllocsForJob(jobID, ns)
		for _, alloc := range allocs {
			out, _ := e2eutil.Command("nomad", "alloc", "status", alloc["ID"])
			fmt.Println(out)
			fmt.Println("----------------")
		}
	}

	return err
}
<|endoftext|>