<commit_before>// Orbit provides an implementation of the LMAX Disruptor.
//
// For more info on LMAX see Martin Fowler's blog post: http://martinfowler.com/articles/lmax.html
//
// Alternatively read the paper: http://disruptor.googlecode.com/files/Disruptor-1.0.pdf
package orbit

import (
	"errors"
	"sync"
	"sync/atomic"
	"unsafe"
)

// Handler is the Consumer handler function type.
// Defaults are provided, however it is expected most users will define their
// own functions. It will be called by the Loops when new messages are
// available for processing. It is not expected to be a long-running process.
//
// It should remember the index of the last item it has processed. It is the
// Handler's responsibility to update the corresponding index via the
// appropriate Loop methods once processing of a Message is complete.
//
// It is assumed that all objects between the index stored in the Loop
// and the next consumer (backwards). Once this index is set the consumer behind
// will be allowed to process up to an including the new value.
// The second parameter is an array of indexes for the handler to process.
//
// If the consumers current index is equal to the index of the consumer ahead of
// it the consumer should wait until this changes before processing any
// messages. This state should only really ever happen on startup or reset, but
// it is safe practice for the system in any case.
type Handler func(Processor, []uint64)

// ReceiverHandler is the same as Handler except it is designed to take arbitrary input
// and store it in the buffer instead of processing data that is already in the buffer.
type ReceiverHandler func(Processor, uint64, interface{})

// Loop maintains buffers and Consumers.
//
// Consumers are defined in the order they will be positined in the ring buffer.
// If the Loop is intended to receive data from a client the Executor handler
// should perform business logic. If the Loop is intended to send responses to
// clients then the Executor handler should write to a HTTP response.
type Loop struct {
	// Buffered interface channel for the input stream
	Input chan interface{}

	// Receiver handler. Similar to Handler but accepts interface
	receiver ReceiverHandler

	// The circular buffer
	buffer []unsafe.Pointer

	// Size of the buffer
	// Must be a power of 2
	bufferSize uint64

	// Indexes, handlers and channels for all consumers
	index   []uint64
	handler []Handler
	channel []chan struct{}

	// Flag to indicate whether Loop is running
	running       bool
	startStopLock *sync.Mutex
}

// Processor marshals/unmarshals Business Logic Consumer output received from and sent to clients.
//
// The typical use case is as follows:
//     Client ->
//     Receiver (Receiver -> Journaler -> Replicator -> Unmarshaller -> Executor) ->
//     Sender (Receiver -> Journaler -> Replicator -> Marshaller -> Publisher) ->
//     Client
// Strictly speaking the Sender is not required and is mostly useful for using Orbit
// behind a client endpoint where the client expects a response.
type Processor interface {
	// Set the receiver index to the given value
	SetReceiverIndex(uint64) error

	// Set the journaler index to the given value
	SetJournalerIndex(uint64) error

	// Set the replicator index to the given value
	SetReplicatorIndex(uint64) error

	// Set the unmarshaller index to the given value
	SetUnmarshallerIndex(uint64) error

	// Set index of Business Logic Consumer to the given value
	SetExecutorIndex(uint64) error

	// Get the message at the given address in the buffer.
	GetMessage(uint64) *Message

	// Set the message at the given address in the buffer.
	SetMessage(uint64, *Message)
}

// New initializes a new Loop
//
// All indexes are set to the beginning of the buffer and Handlers are assigned.
// Space for the buffer is allocated and is filled with empty Message objects.
func New(
	size uint64,
	receiver ReceiverHandler,
	journaler Handler,
	replicator Handler,
	unmarshaller Handler,
	executor Handler,
) *Loop {
	loop := &Loop{
		bufferSize: size,
		buffer:     make([]unsafe.Pointer, size),

		// Start in stopped state
		running:       false,
		startStopLock: &sync.Mutex{},

		// Create handler and index arrays
		handler: make([]Handler, 5),
		index:   make([]uint64, 5),
		channel: make([]chan struct{}, 5),

		// Create the input stream channel
		// TODO allow tweaking this size. It is likely to affect performance, especially in IO-heavy workloads
		Input: make(chan interface{}, 4096),
	}
	// Assign Handlers
	loop.receiver = receiver
	loop.handler[JOURNALER] = journaler
	loop.handler[REPLICATOR] = replicator
	loop.handler[UNMARSHALLER] = unmarshaller
	loop.handler[EXECUTOR] = executor

	if loop.receiver == nil {
		loop.receiver = defaultReceiverFunction
	}
	for k, v := range loop.handler {
		if k != RECEIVER && v == nil {
			loop.handler[k] = DEFAULT_HANDLER[k]
		}
	}

	// Start at index 4 so we don't have index underflow
	loop.Reset(4)

	// Create 'size' new Message objects and store them in the buffer.
	// This avoids costly object creation and GC while streaming data.
	var i uint64
	for i = 0; i < size; i++ {
		msg := &Message{}
		msg.Init()
		// Just needs to be anything, so that the atomic.StorePointer doesn't panic on nil deref
		loop.buffer[i] = unsafe.Pointer(&struct{}{})
		atomic.StorePointer(&loop.buffer[i], unsafe.Pointer(msg))
	}

	return loop
}

// Reset sets all indexes to a given value.
// This is useful for rebuilding Loop state from an input file
// (eg: journaled output) instead of manually looping through the buffer until
// the desired index is reached.
//
// Returns an error if called while Lopp is running. Stop Loop with
// Loop.Stop() before resetting.
func (l *Loop) Reset(i uint64) error {
	if l.running {
		return errors.New("Cannot reset a running Loop")
	}

	// Bypass the setters otherwise their sanity checks will error
	var j uint64
	for j = 0; j <= EXECUTOR; j++ {
		// The first item in index should be first in the buffer
		l.index[j] = i - j
	}

	return nil
}

// Start starts the Loop processing.
// It launches a number of goroutines (one for each Handler + one manager).
//
// These goroutines handle the index checking logic and call the provided
// Handler function when there is data in the buffer available for it to
// process.
func (l *Loop) Start() {
	// Allocate channels
	for i := range l.channel {
		l.channel[i] = make(chan struct{}, 1)
	}

	go l.run()
}

// Stop stops the Loop processing.
// This stops processing gracefully. All messages sent to l.Input before calling Stop
// will be processed.
func (l *Loop) Stop() {
	l.startStopLock.Lock()
	defer l.startStopLock.Unlock()
	l.running = false
	close(l.channel[RECEIVER])
}

// GetIndex returns the Loop's current index for the provided Consumer.
// This index may be larger than the buffer size, as the modulus is used to get
// a valid array index.
//
// h is the handler to fetch the index for.
func (l *Loop) GetIndex(h int) uint64 {
	return l.index[h]
}

// GetMessage returns the message at the given address in the buffer.
//
// If the provided index is larger than the buffer size then the modulus is
// used to generate an index that is in range.
func (l *Loop) GetMessage(i uint64) *Message {
	// Bounds check
	if i >= l.bufferSize {
		i = i % l.bufferSize
	}

	return (*Message)(atomic.LoadPointer(&l.buffer[i]))
}

// SetMessage sets the message at the given address in the buffer.
//
// If the provided index is larger than the buffer size then the modulus is
// used to generate an index that is in range.
//
// The message is not copied, do not hold a reference to it after setting it with this method.
func (l *Loop) SetMessage(i uint64, m *Message) {
	// Bounds check
	if i >= l.bufferSize {
		i = i % l.bufferSize
	}

	atomic.StorePointer(&l.buffer[i], unsafe.Pointer(m))
}

// GetBufferSize returns the size of the Loop's Message buffer array.
func (l *Loop) GetBufferSize() uint64 {
	return l.bufferSize
}

// run starts all the Handler management goroutines.
func (l *Loop) run() {
	l.startStopLock.Lock()
	defer l.startStopLock.Unlock()
	l.running = true

	go l.runReceiver(l.receiver)
	go l.runHandler(l.handler[JOURNALER], JOURNALER)
	go l.runHandler(l.handler[REPLICATOR], REPLICATOR)
	go l.runHandler(l.handler[UNMARSHALLER], UNMARSHALLER)
	go l.runHandler(l.handler[EXECUTOR], EXECUTOR)
}

// runReceiver processes messages sent to it until the channel is closed.
func (l *Loop) runReceiver(h ReceiverHandler) {
	var i uint64
	journalChannel := l.channel[RECEIVER+1]
	arr := []uint64{0}
	for msg := range l.Input {
		i = l.GetIndex(RECEIVER)

		// Run handler
		if h != nil {
			arr[0] = i
			h(l, i, msg)
		}

		// Let the next handler know it can proceed without blocking this one
		select {
		case journalChannel <- struct{}{}:
			// journal handler was notified
		default:
			// journal handler already knows, but hasn't processed new messages yet
		}
	}

	// Close the other handler channels so they stop gracefully
	l.startStopLock.Lock()
	defer l.startStopLock.Unlock()
	for k, v := range l.channel {
		if k != RECEIVER {
			close(v)
		}
	}
}

// runHandler loops, calling the Handler when Messages are available to process.
//
// Gracefully handles Loop.Stop(). runReceiver stops first and the rest of
// handlers finish processing anything available to them before stopping.
func (l *Loop) runHandler(h Handler, t int) {
	var this, last, i, j uint64
	nextChannel := l.channel[(t+1)%(EXECUTOR+1)]
	for _ = range l.channel[t] {
		// Get the current indexes.
		// this - current index of this Handler
		// last - highest index that this Handler can process
		this = l.GetIndex(t)
		last = l.GetIndex(t-1) - 1

		// Check if we can process anything
		if this < last {
			// Build list of indexes to process
			indexes := make([]uint64, last-this)
			for i, j = this+1, 0; i <= last; i, j = i+1, j+1 {
				indexes[j] = i
			}

			// Call the Handler
			if h != nil {
				h(l, indexes)
			}

			// Let the next handler know it can proceed without blocking this one
			if t != EXECUTOR {
				select {
				case nextChannel <- struct{}{}:
					// Notified next handler that Messages are available
				default:
					// Handler already knows, but hasn't processed new messages yet
				}
			}
		}
	}
}
<commit_msg>revise comment<commit_after>// Orbit provides an implementation of the LMAX Disruptor.
//
// For more info on LMAX see Martin Fowler's blog post: http://martinfowler.com/articles/lmax.html
//
// Alternatively read the paper: http://disruptor.googlecode.com/files/Disruptor-1.0.pdf
package orbit

import (
	"errors"
	"sync"
	"sync/atomic"
	"unsafe"
)

// Handler is the Consumer handler function type.
// Defaults are provided, however it is expected most users will define their
// own functions. It will be called by the Loops when new messages are
// available for processing. It is not expected to be a long-running process.
//
// It should remember the index of the last item it has processed. It is the
// Handler's responsibility to update the corresponding index via the
// appropriate Loop methods once processing of a Message is complete.
//
// It is assumed that all objects between the index stored in the Loop
// and the next consumer (backwards). Once this index is set the consumer behind
// will be allowed to process up to an including the new value.
// The second parameter is an array of indexes for the handler to process.
//
// If the consumers current index is equal to the index of the consumer ahead of
// it the consumer should wait until this changes before processing any
// messages. This state should only really ever happen on startup or reset, but
// it is safe practice for the system in any case.
type Handler func(Processor, []uint64)

// ReceiverHandler is the same as Handler except it takes arbitrary input
// and stores it in the buffer instead of processing data that is already in the buffer.
// TODO merge this with Handler by passing in a struct with a bool to store, and a start/end
// index instead of a slice?
type ReceiverHandler func(Processor, uint64, interface{})

// Loop maintains buffers and Consumers.
//
// Consumers are defined in the order they will be positined in the ring buffer.
// If the Loop is intended to receive data from a client the Executor handler
// should perform business logic. If the Loop is intended to send responses to
// clients then the Executor handler should write to a HTTP response.
type Loop struct {
	// Buffered interface channel for the input stream
	Input chan interface{}

	// Receiver handler. Similar to Handler but accepts interface
	receiver ReceiverHandler

	// The circular buffer
	buffer []unsafe.Pointer

	// Size of the buffer
	// Must be a power of 2
	bufferSize uint64

	// Indexes, handlers and channels for all consumers
	index   []uint64
	handler []Handler
	channel []chan struct{}

	// Flag to indicate whether Loop is running
	running       bool
	startStopLock *sync.Mutex
}

// Processor marshals/unmarshals Business Logic Consumer output received from and sent to clients.
//
// The typical use case is as follows:
//     Client ->
//     Receiver (Receiver -> Journaler -> Replicator -> Unmarshaller -> Executor) ->
//     Sender (Receiver -> Journaler -> Replicator -> Marshaller -> Publisher) ->
//     Client
// Strictly speaking the Sender is not required and is mostly useful for using Orbit
// behind a client endpoint where the client expects a response.
type Processor interface {
	// Set the receiver index to the given value
	SetReceiverIndex(uint64) error

	// Set the journaler index to the given value
	SetJournalerIndex(uint64) error

	// Set the replicator index to the given value
	SetReplicatorIndex(uint64) error

	// Set the unmarshaller index to the given value
	SetUnmarshallerIndex(uint64) error

	// Set index of Business Logic Consumer to the given value
	SetExecutorIndex(uint64) error

	// Get the message at the given address in the buffer.
	GetMessage(uint64) *Message

	// Set the message at the given address in the buffer.
	SetMessage(uint64, *Message)
}

// New initializes a new Loop
//
// All indexes are set to the beginning of the buffer and Handlers are assigned.
// Space for the buffer is allocated and is filled with empty Message objects.
func New(
	size uint64,
	receiver ReceiverHandler,
	journaler Handler,
	replicator Handler,
	unmarshaller Handler,
	executor Handler,
) *Loop {
	loop := &Loop{
		bufferSize: size,
		buffer:     make([]unsafe.Pointer, size),

		// Start in stopped state
		running:       false,
		startStopLock: &sync.Mutex{},

		// Create handler and index arrays
		handler: make([]Handler, 5),
		index:   make([]uint64, 5),
		channel: make([]chan struct{}, 5),

		// Create the input stream channel
		// TODO allow tweaking this size. It is likely to affect performance, especially in IO-heavy workloads
		Input: make(chan interface{}, 4096),
	}
	// Assign Handlers
	loop.receiver = receiver
	loop.handler[JOURNALER] = journaler
	loop.handler[REPLICATOR] = replicator
	loop.handler[UNMARSHALLER] = unmarshaller
	loop.handler[EXECUTOR] = executor

	if loop.receiver == nil {
		loop.receiver = defaultReceiverFunction
	}
	for k, v := range loop.handler {
		if k != RECEIVER && v == nil {
			loop.handler[k] = DEFAULT_HANDLER[k]
		}
	}

	// Start at index 4 so we don't have index underflow
	loop.Reset(4)

	// Create 'size' new Message objects and store them in the buffer.
	// This avoids costly object creation and GC while streaming data.
	var i uint64
	for i = 0; i < size; i++ {
		msg := &Message{}
		msg.Init()
		// Just needs to be anything, so that the atomic.StorePointer doesn't panic on nil deref
		loop.buffer[i] = unsafe.Pointer(&struct{}{})
		atomic.StorePointer(&loop.buffer[i], unsafe.Pointer(msg))
	}

	return loop
}

// Reset sets all indexes to a given value.
// This is useful for rebuilding Loop state from an input file
// (eg: journaled output) instead of manually looping through the buffer until
// the desired index is reached.
//
// Returns an error if called while Lopp is running. Stop Loop with
// Loop.Stop() before resetting.
func (l *Loop) Reset(i uint64) error {
	if l.running {
		return errors.New("Cannot reset a running Loop")
	}

	// Bypass the setters otherwise their sanity checks will error
	var j uint64
	for j = 0; j <= EXECUTOR; j++ {
		// The first item in index should be first in the buffer
		l.index[j] = i - j
	}

	return nil
}

// Start starts the Loop processing.
// It launches a number of goroutines (one for each Handler + one manager).
//
// These goroutines handle the index checking logic and call the provided
// Handler function when there is data in the buffer available for it to
// process.
func (l *Loop) Start() {
	// Allocate channels
	for i := range l.channel {
		l.channel[i] = make(chan struct{}, 1)
	}

	go l.run()
}

// Stop stops the Loop processing.
// This stops processing gracefully. All messages sent to l.Input before calling Stop
// will be processed.
func (l *Loop) Stop() {
	l.startStopLock.Lock()
	defer l.startStopLock.Unlock()
	l.running = false
	close(l.channel[RECEIVER])
}

// GetIndex returns the Loop's current index for the provided Consumer.
// This index may be larger than the buffer size, as the modulus is used to get
// a valid array index.
//
// h is the handler to fetch the index for.
func (l *Loop) GetIndex(h int) uint64 {
	return l.index[h]
}

// GetMessage returns the message at the given address in the buffer.
//
// If the provided index is larger than the buffer size then the modulus is
// used to generate an index that is in range.
func (l *Loop) GetMessage(i uint64) *Message {
	// Bounds check
	if i >= l.bufferSize {
		i = i % l.bufferSize
	}

	return (*Message)(atomic.LoadPointer(&l.buffer[i]))
}

// SetMessage sets the message at the given address in the buffer.
//
// If the provided index is larger than the buffer size then the modulus is
// used to generate an index that is in range.
//
// The message is not copied, do not hold a reference to it after setting it with this method.
func (l *Loop) SetMessage(i uint64, m *Message) {
	// Bounds check
	if i >= l.bufferSize {
		i = i % l.bufferSize
	}

	atomic.StorePointer(&l.buffer[i], unsafe.Pointer(m))
}

// GetBufferSize returns the size of the Loop's Message buffer array.
func (l *Loop) GetBufferSize() uint64 {
	return l.bufferSize
}

// run starts all the Handler management goroutines.
func (l *Loop) run() {
	l.startStopLock.Lock()
	defer l.startStopLock.Unlock()
	l.running = true

	go l.runReceiver(l.receiver)
	go l.runHandler(l.handler[JOURNALER], JOURNALER)
	go l.runHandler(l.handler[REPLICATOR], REPLICATOR)
	go l.runHandler(l.handler[UNMARSHALLER], UNMARSHALLER)
	go l.runHandler(l.handler[EXECUTOR], EXECUTOR)
}

// runReceiver processes messages sent to it until the channel is closed.
func (l *Loop) runReceiver(h ReceiverHandler) {
	var i uint64
	journalChannel := l.channel[RECEIVER+1]
	arr := []uint64{0}
	for msg := range l.Input {
		i = l.GetIndex(RECEIVER)

		// Run handler
		if h != nil {
			arr[0] = i
			h(l, i, msg)
		}

		// Let the next handler know it can proceed without blocking this one
		select {
		case journalChannel <- struct{}{}:
			// journal handler was notified
		default:
			// journal handler already knows, but hasn't processed new messages yet
		}
	}

	// Close the other handler channels so they stop gracefully
	l.startStopLock.Lock()
	defer l.startStopLock.Unlock()
	for k, v := range l.channel {
		if k != RECEIVER {
			close(v)
		}
	}
}

// runHandler loops, calling the Handler when Messages are available to process.
//
// Gracefully handles Loop.Stop(). runReceiver stops first and the rest of
// handlers finish processing anything available to them before stopping.
func (l *Loop) runHandler(h Handler, t int) {
	var this, last, i, j uint64
	nextChannel := l.channel[(t+1)%(EXECUTOR+1)]
	for _ = range l.channel[t] {
		// Get the current indexes.
		// this - current index of this Handler
		// last - highest index that this Handler can process
		this = l.GetIndex(t)
		last = l.GetIndex(t-1) - 1

		// Check if we can process anything
		if this < last {
			// Build list of indexes to process
			indexes := make([]uint64, last-this)
			for i, j = this+1, 0; i <= last; i, j = i+1, j+1 {
				indexes[j] = i
			}

			// Call the Handler
			if h != nil {
				h(l, indexes)
			}

			// Let the next handler know it can proceed without blocking this one
			if t != EXECUTOR {
				select {
				case nextChannel <- struct{}{}:
					// Notified next handler that Messages are available
				default:
					// Handler already knows, but hasn't processed new messages yet
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package pizzapi

import (
	"errors"
)

type Order struct {
	Id     int    `json:"id"`
	Item   *Pizza `json:"pizza"`
	Status string `json:"status"`
}

var loadedOrders []*Order

func CreateOrder(item *Pizza) *Order {
	id := len(loadedOrders) + 1
	order := &Order{Id: id, Item: item, Status: "new"}
	loadedOrders = append(loadedOrders, order)

	return order
}

func FindOrder(id int) (*Order, error) {
	for _, v := range loadedOrders {
		if v.Id == id {
			return v, nil
		}
	}
	return nil, errors.New("Cannot find order " + string(id))
}

func AllOrders() ([]*Order, error) {
	return loadedOrders, nil
}
<commit_msg>return an empty array if there are no orders<commit_after>package pizzapi

import (
	"errors"
)

type Order struct {
	Id     int    `json:"id"`
	Item   *Pizza `json:"pizza"`
	Status string `json:"status"`
}

var loadedOrders []*Order

func CreateOrder(item *Pizza) *Order {
	id := len(loadedOrders) + 1
	order := &Order{Id: id, Item: item, Status: "new"}
	loadedOrders = append(loadedOrders, order)

	return order
}

func FindOrder(id int) (*Order, error) {
	for _, v := range loadedOrders {
		if v.Id == id {
			return v, nil
		}
	}
	return nil, errors.New("Cannot find order " + string(id))
}

func AllOrders() ([]*Order, error) {
	if loadedOrders == nil {
		return []*Order{}, nil
	}
	return loadedOrders, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd.

// Package talksapp implements the go-talks.appspot.com server.
package talksapp

import (
	"bytes"
	"errors"
	"fmt"
	"html/template"
	"io"
	"net/http"
	"os"
	"path"
	"time"

	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/memcache"
	"google.golang.org/appengine/urlfetch"

	"github.com/golang/gddo/gosrc"
	"github.com/golang/gddo/httputil"

	"golang.org/x/net/context"
	"golang.org/x/tools/present"
)

var (
	presentTemplates = map[string]*template.Template{
		".article": parsePresentTemplate("article.tmpl"),
		".slide":   parsePresentTemplate("slides.tmpl"),
	}
	homeArticle  = loadHomeArticle()
	contactEmail = "golang-dev@googlegroups.com"

	// used for mocking in tests
	getPresentation = gosrc.GetPresentation
	playCompileURL  = "https://play.golang.org/compile"
)

func init() {
	http.Handle("/", handlerFunc(serveRoot))
	http.Handle("/compile", handlerFunc(serveCompile))
	http.Handle("/bot.html", handlerFunc(serveBot))
	present.PlayEnabled = true
	if s := os.Getenv("CONTACT_EMAIL"); s != "" {
		contactEmail = s
	}

	if appengine.IsDevAppServer() {
		return
	}
	github := httputil.NewAuthTransportFromEnvironment(nil)
	if github.Token == "" || github.ClientID == "" || github.ClientSecret == "" {
		panic("missing GitHub metadata, follow the instructions on README.md")
	}
}

func playable(c present.Code) bool {
	return present.PlayEnabled && c.Play && c.Ext == ".go"
}

func parsePresentTemplate(name string) *template.Template {
	t := present.Template()
	t = t.Funcs(template.FuncMap{"playable": playable})
	if _, err := t.ParseFiles("present/templates/"+name, "present/templates/action.tmpl"); err != nil {
		panic(err)
	}
	t = t.Lookup("root")
	if t == nil {
		panic("root template not found for " + name)
	}
	return t
}

func loadHomeArticle() []byte {
	const fname = "assets/home.article"
	f, err := os.Open(fname)
	if err != nil {
		panic(err)
	}
	defer f.Close()
	doc, err := present.Parse(f, fname, 0)
	if err != nil {
		panic(err)
	}
	var buf bytes.Buffer
	if err := renderPresentation(&buf, fname, doc); err != nil {
		panic(err)
	}
	return buf.Bytes()
}

func renderPresentation(w io.Writer, fname string, doc *present.Doc) error {
	t := presentTemplates[path.Ext(fname)]
	if t == nil {
		return errors.New("unknown template extension")
	}
	data := struct {
		*present.Doc
		Template     *template.Template
		PlayEnabled  bool
		NotesEnabled bool
	}{doc, t, true, true}
	return t.Execute(w, &data)
}

type presFileNotFoundError string

func (s presFileNotFoundError) Error() string { return fmt.Sprintf("File %s not found.", string(s)) }

func writeHTMLHeader(w http.ResponseWriter, status int) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	w.WriteHeader(status)
}

func writeTextHeader(w http.ResponseWriter, status int) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	w.WriteHeader(status)
}

func httpClient(r *http.Request) *http.Client {
	ctx, _ := context.WithTimeout(appengine.NewContext(r), 10*time.Second)
	github := httputil.NewAuthTransportFromEnvironment(nil)

	return &http.Client{
		Transport: &httputil.AuthTransport{
			Token:        github.Token,
			ClientID:     github.ClientID,
			ClientSecret: github.ClientSecret,
			Base:         &urlfetch.Transport{Context: ctx},
			UserAgent:    fmt.Sprintf("%s (+http://%s/-/bot)", appengine.AppID(ctx), r.Host),
		},
	}
}

type handlerFunc func(http.ResponseWriter, *http.Request) error

func (f handlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)

	err := f(w, r)
	if err == nil {
		return
	} else if gosrc.IsNotFound(err) {
		writeTextHeader(w, 400)
		io.WriteString(w, "Not Found.")
	} else if e, ok := err.(*gosrc.RemoteError); ok {
		writeTextHeader(w, 500)
		fmt.Fprintf(w, "Error accessing %s.\n%v", e.Host, e)
		log.Infof(ctx, "Remote error %s: %v", e.Host, e)
	} else if e, ok := err.(presFileNotFoundError); ok {
		writeTextHeader(w, 200)
		io.WriteString(w, e.Error())
	} else if err != nil {
		writeTextHeader(w, 500)
		io.WriteString(w, "Internal server error.")
		log.Errorf(ctx, "Internal error %v", err)
	}
}

func serveRoot(w http.ResponseWriter, r *http.Request) error {
	switch {
	case r.Method != "GET" && r.Method != "HEAD":
		writeTextHeader(w, 405)
		_, err := io.WriteString(w, "Method not supported.")
		return err
	case r.URL.Path == "/":
		writeHTMLHeader(w, 200)
		_, err := w.Write(homeArticle)
		return err
	default:
		return servePresentation(w, r)
	}
}

func servePresentation(w http.ResponseWriter, r *http.Request) error {
	ctx := appengine.NewContext(r)
	importPath := r.URL.Path[1:]

	item, err := memcache.Get(ctx, importPath)
	if err == nil {
		writeHTMLHeader(w, 200)
		w.Write(item.Value)
		return nil
	} else if err != memcache.ErrCacheMiss {
		return err
	}

	log.Infof(ctx, "Fetching presentation %s.", importPath)
	pres, err := getPresentation(httpClient(r), importPath)
	if err != nil {
		return fmt.Errorf("Could not get presentation: %v", err)
	}
	parser := &present.Context{
		ReadFile: func(name string) ([]byte, error) {
			if p, ok := pres.Files[name]; ok {
				return p, nil
			}
			return nil, presFileNotFoundError(name)
		},
	}
	doc, err := parser.Parse(bytes.NewReader(pres.Files[pres.Filename]), pres.Filename, 0)
	if err != nil {
		return fmt.Errorf("Could not parse presentation: %v", err)
	}

	var buf bytes.Buffer
	if err := renderPresentation(&buf, importPath, doc); err != nil {
		return fmt.Errorf("Could not render presentation: %v", err)
	}

	if err := memcache.Add(ctx, &memcache.Item{
		Key:        importPath,
		Value:      buf.Bytes(),
		Expiration: time.Hour,
	}); err != nil {
		log.Errorf(ctx, "Could not cache presentation %s: %v", importPath, err)
		return nil
	}

	writeHTMLHeader(w, 200)
	_, err = w.Write(buf.Bytes())
	return err
}

func serveCompile(w http.ResponseWriter, r *http.Request) error {
	client := urlfetch.Client(appengine.NewContext(r))
	if err := r.ParseForm(); err != nil {
		return err
	}
	resp, err := client.PostForm(playCompileURL, r.Form)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
	_, err = io.Copy(w, resp.Body)
	return err
}

func serveBot(w http.ResponseWriter, r *http.Request) error {
	ctx := appengine.NewContext(r)
	writeTextHeader(w, 200)
	_, err := fmt.Fprintf(w, "Contact %s for help with the %s bot.", contactEmail, appengine.AppID(ctx))
	return err
}
<commit_msg>talksapp: remove decoration to errors at top level<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd.

// Package talksapp implements the go-talks.appspot.com server.
package talksapp

import (
	"bytes"
	"errors"
	"fmt"
	"html/template"
	"io"
	"net/http"
	"os"
	"path"
	"time"

	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/memcache"
	"google.golang.org/appengine/urlfetch"

	"github.com/golang/gddo/gosrc"
	"github.com/golang/gddo/httputil"

	"golang.org/x/net/context"
	"golang.org/x/tools/present"
)

var (
	presentTemplates = map[string]*template.Template{
		".article": parsePresentTemplate("article.tmpl"),
		".slide":   parsePresentTemplate("slides.tmpl"),
	}
	homeArticle  = loadHomeArticle()
	contactEmail = "golang-dev@googlegroups.com"

	// used for mocking in tests
	getPresentation = gosrc.GetPresentation
	playCompileURL  = "https://play.golang.org/compile"
)

func init() {
	http.Handle("/", handlerFunc(serveRoot))
	http.Handle("/compile", handlerFunc(serveCompile))
	http.Handle("/bot.html", handlerFunc(serveBot))
	present.PlayEnabled = true
	if s := os.Getenv("CONTACT_EMAIL"); s != "" {
		contactEmail = s
	}

	if appengine.IsDevAppServer() {
		return
	}
	github := httputil.NewAuthTransportFromEnvironment(nil)
	if github.Token == "" || github.ClientID == "" || github.ClientSecret == "" {
		panic("missing GitHub metadata, follow the instructions on README.md")
	}
}

func playable(c present.Code) bool {
	return present.PlayEnabled && c.Play && c.Ext == ".go"
}

func parsePresentTemplate(name string) *template.Template {
	t := present.Template()
	t = t.Funcs(template.FuncMap{"playable": playable})
	if _, err := t.ParseFiles("present/templates/"+name, "present/templates/action.tmpl"); err != nil {
		panic(err)
	}
	t = t.Lookup("root")
	if t == nil {
		panic("root template not found for " + name)
	}
	return t
}

func loadHomeArticle() []byte {
	const fname = "assets/home.article"
	f, err := os.Open(fname)
	if err != nil {
		panic(err)
	}
	defer f.Close()
	doc, err := present.Parse(f, fname, 0)
	if err != nil {
		panic(err)
	}
	var buf bytes.Buffer
	if err := renderPresentation(&buf, fname, doc); err != nil {
		panic(err)
	}
	return buf.Bytes()
}

func renderPresentation(w io.Writer, fname string, doc *present.Doc) error {
	t := presentTemplates[path.Ext(fname)]
	if t == nil {
		return errors.New("unknown template extension")
	}
	data := struct {
		*present.Doc
		Template     *template.Template
		PlayEnabled  bool
		NotesEnabled bool
	}{doc, t, true, true}
	return t.Execute(w, &data)
}

type presFileNotFoundError string

func (s presFileNotFoundError) Error() string { return fmt.Sprintf("File %s not found.", string(s)) }

func writeHTMLHeader(w http.ResponseWriter, status int) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	w.WriteHeader(status)
}

func writeTextHeader(w http.ResponseWriter, status int) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	w.WriteHeader(status)
}

func httpClient(r *http.Request) *http.Client {
	ctx, _ := context.WithTimeout(appengine.NewContext(r), 10*time.Second)
	github := httputil.NewAuthTransportFromEnvironment(nil)

	return &http.Client{
		Transport: &httputil.AuthTransport{
			Token:        github.Token,
			ClientID:     github.ClientID,
			ClientSecret: github.ClientSecret,
			Base:         &urlfetch.Transport{Context: ctx},
			UserAgent:    fmt.Sprintf("%s (+http://%s/-/bot)", appengine.AppID(ctx), r.Host),
		},
	}
}

type handlerFunc func(http.ResponseWriter, *http.Request) error

func (f handlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)

	err := f(w, r)
	if err == nil {
		return
	} else if gosrc.IsNotFound(err) {
		writeTextHeader(w, 400)
		io.WriteString(w, "Not Found.")
	} else if e, ok := err.(*gosrc.RemoteError); ok {
		writeTextHeader(w, 500)
		fmt.Fprintf(w, "Error accessing %s.\n%v", e.Host, e)
		log.Infof(ctx, "Remote error %s: %v", e.Host, e)
	} else if e, ok := err.(presFileNotFoundError); ok {
		writeTextHeader(w, 200)
		io.WriteString(w, e.Error())
	} else if err != nil {
		writeTextHeader(w, 500)
		io.WriteString(w, "Internal server error.")
		log.Errorf(ctx, "Internal error %v", err)
	}
}

func serveRoot(w http.ResponseWriter, r *http.Request) error {
	switch {
	case r.Method != "GET" && r.Method != "HEAD":
		writeTextHeader(w, 405)
		_, err := io.WriteString(w, "Method not supported.")
		return err
	case r.URL.Path == "/":
		writeHTMLHeader(w, 200)
		_, err := w.Write(homeArticle)
		return err
	default:
		return servePresentation(w, r)
	}
}

func servePresentation(w http.ResponseWriter, r *http.Request) error {
	ctx := appengine.NewContext(r)
	importPath := r.URL.Path[1:]

	item, err := memcache.Get(ctx, importPath)
	if err == nil {
		writeHTMLHeader(w, 200)
		w.Write(item.Value)
		return nil
	} else if err != memcache.ErrCacheMiss {
		return err
	}

	log.Infof(ctx, "Fetching presentation %s.", importPath)
	pres, err := getPresentation(httpClient(r), importPath)
	if err != nil {
		return err
	}
	parser := &present.Context{
		ReadFile: func(name string) ([]byte, error) {
			if p, ok := pres.Files[name]; ok {
				return p, nil
			}
			return nil, presFileNotFoundError(name)
		},
	}
	doc, err := parser.Parse(bytes.NewReader(pres.Files[pres.Filename]), pres.Filename, 0)
	if err != nil {
		return err
	}

	var buf bytes.Buffer
	if err := renderPresentation(&buf, importPath, doc); err != nil {
		return err
	}

	if err := memcache.Add(ctx, &memcache.Item{
		Key:        importPath,
		Value:      buf.Bytes(),
		Expiration: time.Hour,
	}); err != nil {
		log.Errorf(ctx, "Could not cache presentation %s: %v", importPath, err)
		return nil
	}

	writeHTMLHeader(w, 200)
	_, err = w.Write(buf.Bytes())
	return err
}

func serveCompile(w http.ResponseWriter, r *http.Request) error {
	client := urlfetch.Client(appengine.NewContext(r))
	if err := r.ParseForm(); err != nil {
		return err
	}
	resp, err := client.PostForm(playCompileURL, r.Form)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
	_, err = io.Copy(w, resp.Body)
	return err
}

func serveBot(w http.ResponseWriter, r *http.Request) error {
	ctx := appengine.NewContext(r)
	writeTextHeader(w, 200)
	_, err := fmt.Fprintf(w, "Contact %s for help with the %s bot.", contactEmail, appengine.AppID(ctx))
	return err
}
<|endoftext|>


<commit_before>package source

import (
	"os"
	"path/filepath"
)

// Info is detected platform information from a source directory
type Info struct {
	Platform string
	Version  string
}

// DetectorFunc is a function that returns source Info from a given directory.
// It returns true if it was able to detect the code in the given directory.
type DetectorFunc func(dir string) (*Info, bool)

// Detectors is a set of DetectorFunc that is used to detect the
// language/platform for a given source directory
type Detectors []DetectorFunc

// DefafultDetectors is a default set of Detector functions
var DefaultDetectors = Detectors{
	DetectRuby,
	DetectJava,
	DetectNodeJS,
	DetectPHP,
	DetectPython,
	DetectPerl,
}

type sourceDetector struct {
	detectors []DetectorFunc
}

// DetectSource returns source information from a given directory using
// a set of Detectors
func (s Detectors) DetectSource(dir string) (*Info, bool) {
	for _, d := range s {
		if info, found := d(dir); found {
			return info, true
		}
	}
	return nil, false
}

// DetectRuby detects Ruby source
func DetectRuby(dir string) (*Info, bool) {
	return detect("ruby", dir, "Gemfile", "Rakefile", "config.ru")
}

// DetectJava detects Java source
func DetectJava(dir string) (*Info, bool) {
	return detect("jee", dir, "pom.xml")
}

// DetectNodeJS detects NodeJS source
func DetectNodeJS(dir string) (*Info, bool) {
	return detect("nodejs", dir, "app.json", "package.json")
}

// DetectPHP detects PHP source
func DetectPHP(dir string) (*Info, bool) {
	return detect("php", dir, "index.php", "composer.json")
}

// DetectPython detects Python source
func DetectPython(dir string) (*Info, bool) {
	return detect("python", dir, "requirements.txt", "config.py")
}

// DetectPerl detects Perl source
func DetectPerl(dir string) (*Info, bool) {
	return detect("perl", dir, "index.pl", "cpanfile")
}

// detect returns an Info object with the given platform if the source at dir contains any of the argument files
func detect(platform string, dir string, files ...string) (*Info, bool) {
	if filesPresent(dir, files) {
		return &Info{
			Platform: platform,
		}, true
	}
	return nil, false
}

func filesPresent(dir string, files []string) bool {
	for _, f := range files {
		_, err := os.Stat(filepath.Join(dir, f))
		if err == nil {
			return true
		}
	}
	return false
}
<commit_msg>Add scala source detector<commit_after>package source

import (
	"os"
	"path/filepath"
)

// Info is detected platform information from a source directory
type Info struct {
	Platform string
	Version  string
}

// DetectorFunc is a function that returns source Info from a given directory.
// It returns true if it was able to detect the code in the given directory.
type DetectorFunc func(dir string) (*Info, bool)

// Detectors is a set of DetectorFunc that is used to detect the
// language/platform for a given source directory
type Detectors []DetectorFunc

// DefafultDetectors is a default set of Detector functions
var DefaultDetectors = Detectors{
	DetectRuby,
	DetectJava,
	DetectNodeJS,
	DetectPHP,
	DetectPython,
	DetectPerl,
	DetectScala,
}

type sourceDetector struct {
	detectors []DetectorFunc
}

// DetectSource returns source information from a given directory using
// a set of Detectors
func (s Detectors) DetectSource(dir string) (*Info, bool) {
	for _, d := range s {
		if info, found := d(dir); found {
			return info, true
		}
	}
	return nil, false
}

// DetectRuby detects Ruby source
func DetectRuby(dir string) (*Info, bool) {
	return detect("ruby", dir, "Gemfile", "Rakefile", "config.ru")
}

// DetectJava detects Java source
func DetectJava(dir string) (*Info, bool) {
	return detect("jee", dir, "pom.xml")
}

// DetectNodeJS detects NodeJS source
func DetectNodeJS(dir string) (*Info, bool) {
	return detect("nodejs", dir, "app.json", "package.json")
}

// DetectPHP detects PHP source
func DetectPHP(dir string) (*Info, bool) {
	return detect("php", dir, "index.php", "composer.json")
}

// DetectPython detects Python source
func DetectPython(dir string) (*Info, bool) {
	return detect("python", dir, "requirements.txt", "config.py")
}

// DetectPerl detects Perl source
func DetectPerl(dir string) (*Info, bool) {
	return detect("perl", dir, "index.pl", "cpanfile")
}

// DetectScala  detects Scala source
func DetectScala(dir string) (*Info, bool) {
	return detect("scala", dir, "build.sbt")
}

// detect returns an Info object with the given platform if the source at dir contains any of the argument files
func detect(platform string, dir string, files ...string) (*Info, bool) {
	if filesPresent(dir, files) {
		return &Info{
			Platform: platform,
		}, true
	}
	return nil, false
}

func filesPresent(dir string, files []string) bool {
	for _, f := range files {
		_, err := os.Stat(filepath.Join(dir, f))
		if err == nil {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package local

import (
	"context"
	"crypto/sha1"
	"encoding/hex"
	"io"
	"os/exec"
	"regexp"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v1alpha3"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

func (b *Builder) buildJibMaven(ctx context.Context, out io.Writer, workspace string, a *v1alpha3.JibMavenArtifact) (string, error) {
	skaffoldImage := generateJibImageRef(workspace, a.Module)

	maven, err := findBuilder("mvn", "mvnw", workspace)
	if err != nil {
		return "", err
	}
	mavenCommand, err := generateMavenCommand(workspace, skaffoldImage, a)
	if err != nil {
		return "", err
	}
	commandLine := append(maven, mavenCommand...)

	err = executeBuildCommand(ctx, out, workspace, commandLine)
	if err != nil {
		return "", errors.Wrap(err, "maven build failed")
	}
	return skaffoldImage, nil
}

// generateMavenCommand generates the command-line to pass to maven for building a
// project found in `workspace`.  The resulting image is added to the local docker daemon
// and called `skaffoldImage`.
func generateMavenCommand(workspace string, skaffoldImage string, a *v1alpha3.JibMavenArtifact) ([]string, error) {
	if a.Module != "" {
		// TODO: multi-module
		return nil, errors.New("Maven multi-modules not supported yet")
	}
	// use mostly-qualified plugin ID in case jib is not a configured plugin
	commandLine := []string{"prepare-package", "com.google.cloud.tools:jib-maven-plugin::dockerBuild", "-Dimage="+skaffoldImage}
	if a.Profile != "" {
		commandLine = append(commandLine, "-P"+a.Profile)
	}
	return commandLine, nil
}

func (b *Builder) buildJibGradle(ctx context.Context, out io.Writer, workspace string, a *v1alpha3.JibGradleArtifact) (string, error) {
	skaffoldImage := generateJibImageRef(workspace, a.Project)
	gradle, err := findBuilder("gradle", "gradlew", workspace)
	if err != nil {
		return "", err
	}
	gradleCommand, err := generateGradleCommand(workspace, skaffoldImage, a)
	if err != nil {
		return "", err
	}
	commandLine := append(gradle, gradleCommand...)

	err = executeBuildCommand(ctx, out, workspace, commandLine)
	if err != nil {
		return "", errors.Wrap(err, "gradle build failed")
	}
	return skaffoldImage, nil
}

// generateGradleCommand generates the command-line to pass to gradle for building an
// project in `workspace`.  The resulting image is added to the local docker daemon
// and called `skaffoldImage`.
func generateGradleCommand(workspace string, skaffoldImage string, a *v1alpha3.JibGradleArtifact) ([]string, error) {
	command := []string{}
	if a.Project == "" {
		command = append(command, ":jibDockerBuild")
	} else {
		// multi-module
		command = append(command, ":" + a.Project + ":jibDockerBuild")
	}
	command = append(command, "--image="+skaffoldImage)
	return command, nil
}

// executeBuildCommand executes the command-line with the working directory set to `workspace`.
func executeBuildCommand(ctx context.Context, out io.Writer, workspace string, commandLine []string) error {
	logrus.Infof("Building %s: %s", workspace, commandLine)
	cmd := exec.CommandContext(ctx, commandLine[0], commandLine[1:]...)
	cmd.Dir = workspace
	cmd.Stdout = out
	cmd.Stderr = out
	return util.RunCmd(cmd)
}

// jibBuildImageRef generates a valid image name for the workspace and project.
// The image name is always prefixed with `jib`.
func generateJibImageRef(workspace string, project string) string {
	imageName := "jib" + workspace
	if project != "" {
		imageName += "_" + project
	}
	// if the workspace + project is a valid image name then use it
	match, _ := regexp.MatchString(constants.RepositoryComponentRegex, imageName);
	if match {
		return imageName
	}
	// otherwise use a hash for a deterministic name
	hasher := sha1.New()
	io.WriteString(hasher, imageName)
    return "jib__" + hex.EncodeToString(hasher.Sum(nil))
}
<commit_msg>regexp.MatchString -> regexp.MustParse<commit_after>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package local

import (
	"context"
	"crypto/sha1"
	"encoding/hex"
	"io"
	"os/exec"
	"regexp"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v1alpha3"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

func (b *Builder) buildJibMaven(ctx context.Context, out io.Writer, workspace string, a *v1alpha3.JibMavenArtifact) (string, error) {
	skaffoldImage := generateJibImageRef(workspace, a.Module)

	maven, err := findBuilder("mvn", "mvnw", workspace)
	if err != nil {
		return "", err
	}
	mavenCommand, err := generateMavenCommand(workspace, skaffoldImage, a)
	if err != nil {
		return "", err
	}
	commandLine := append(maven, mavenCommand...)

	err = executeBuildCommand(ctx, out, workspace, commandLine)
	if err != nil {
		return "", errors.Wrap(err, "maven build failed")
	}
	return skaffoldImage, nil
}

// generateMavenCommand generates the command-line to pass to maven for building a
// project found in `workspace`.  The resulting image is added to the local docker daemon
// and called `skaffoldImage`.
func generateMavenCommand(workspace string, skaffoldImage string, a *v1alpha3.JibMavenArtifact) ([]string, error) {
	if a.Module != "" {
		// TODO: multi-module
		return nil, errors.New("Maven multi-modules not supported yet")
	}
	// use mostly-qualified plugin ID in case jib is not a configured plugin
	commandLine := []string{"prepare-package", "com.google.cloud.tools:jib-maven-plugin::dockerBuild", "-Dimage="+skaffoldImage}
	if a.Profile != "" {
		commandLine = append(commandLine, "-P"+a.Profile)
	}
	return commandLine, nil
}

func (b *Builder) buildJibGradle(ctx context.Context, out io.Writer, workspace string, a *v1alpha3.JibGradleArtifact) (string, error) {
	skaffoldImage := generateJibImageRef(workspace, a.Project)
	gradle, err := findBuilder("gradle", "gradlew", workspace)
	if err != nil {
		return "", err
	}
	gradleCommand, err := generateGradleCommand(workspace, skaffoldImage, a)
	if err != nil {
		return "", err
	}
	commandLine := append(gradle, gradleCommand...)

	err = executeBuildCommand(ctx, out, workspace, commandLine)
	if err != nil {
		return "", errors.Wrap(err, "gradle build failed")
	}
	return skaffoldImage, nil
}

// generateGradleCommand generates the command-line to pass to gradle for building an
// project in `workspace`.  The resulting image is added to the local docker daemon
// and called `skaffoldImage`.
func generateGradleCommand(workspace string, skaffoldImage string, a *v1alpha3.JibGradleArtifact) ([]string, error) {
	command := []string{}
	if a.Project == "" {
		command = append(command, ":jibDockerBuild")
	} else {
		// multi-module
		command = append(command, ":" + a.Project + ":jibDockerBuild")
	}
	command = append(command, "--image="+skaffoldImage)
	return command, nil
}

// executeBuildCommand executes the command-line with the working directory set to `workspace`.
func executeBuildCommand(ctx context.Context, out io.Writer, workspace string, commandLine []string) error {
	logrus.Infof("Building %s: %s", workspace, commandLine)
	cmd := exec.CommandContext(ctx, commandLine[0], commandLine[1:]...)
	cmd.Dir = workspace
	cmd.Stdout = out
	cmd.Stderr = out
	return util.RunCmd(cmd)
}

// jibBuildImageRef generates a valid image name for the workspace and project.
// The image name is always prefixed with `jib`.
func generateJibImageRef(workspace string, project string) string {
	imageName := "jib" + workspace
	if project != "" {
		imageName += "_" + project
	}
	// if the workspace + project is a valid image name then use it
	match, _ := regexp.MustParse(constants.RepositoryComponentRegex, imageName);
	if match {
		return imageName
	}
	// otherwise use a hash for a deterministic name
	hasher := sha1.New()
	io.WriteString(hasher, imageName)
    return "jib__" + hex.EncodeToString(hasher.Sum(nil))
}
<|endoftext|>


<commit_before>package pangu

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"regexp"
	"text/template"
)

const VERSION = "1.1.0"

// CJK is short for Chinese, Japanese and Korean.
//
// The constant cjk contains following Unicode blocks:
// 	\u2e80-\u2eff CJK Radicals Supplement
// 	\u2f00-\u2fdf Kangxi Radicals
// 	\u3040-\u309f Hiragana
// 	\u30a0-\u30ff Katakana
// 	\u3100-\u312f Bopomofo
// 	\u3200-\u32ff Enclosed CJK Letters and Months
// 	\u3400-\u4dbf CJK Unified Ideographs Extension A
// 	\u4e00-\u9fff CJK Unified Ideographs
// 	\uf900-\ufaff CJK Compatibility Ideographs
//
// For more information about Unicode blocks, see
// 	http://unicode-table.com/en/
const cjk = "" +
	"\u2e80-\u2eff" +
	"\u2f00-\u2fdf" +
	"\u3040-\u309f" +
	"\u30a0-\u30ff" +
	"\u3100-\u312f" +
	"\u3200-\u32ff" +
	"\u3400-\u4dbf" +
	"\u4e00-\u9fff" +
	"\uf900-\ufaff"

// ANS is short for Alphabets, Numbers
// and Symbols (`~!@#$%^&*()-_=+[]{}\|;:'",<.>/?).
//
// The constant ans doesn't contain all symbols above.
const ans = "A-Za-z0-9`\\$%\\^&\\*\\-=\\+\\\\|/\u00a1-\u00ff\u2022\u2027\u2150-\u218f"

var cjk_quote = regexp.MustCompile(re("([{{ .CJK }}])" + "([\"'])"))
var quote_cjk = regexp.MustCompile(re("([\"'])" + "([{{ .CJK }}])"))
var fix_quote = regexp.MustCompile(re("([\"'\\(\\[\\{<\u201c])" + "(\\s*)" + "(.+?)" + "(\\s*)" + "([\"'\\)\\]\\}>\u201d])"))
var fix_single_quote = regexp.MustCompile(re("([{{ .CJK }}])" + "( )" + "(')" + "([A-Za-z])"))

var cjk_hash = regexp.MustCompile(re("([{{ .CJK }}])" + "(#(\\S+))"))
var hash_cjk = regexp.MustCompile(re("((\\S+)#)" + "([{{ .CJK }}])"))

var cjk_operator_ans = regexp.MustCompile(re("([{{ .CJK }}])" + "([\\+\\-\\*/=&\\|<>])" + "([A-Za-z0-9])"))
var ans_operator_cjk = regexp.MustCompile(re("([A-Za-z0-9])" + "([\\+\\-\\*/=&\\|<>])" + "([{{ .CJK }}])"))

var cjk_bracket_cjk = regexp.MustCompile(re("([{{ .CJK }}])" + "([\\(\\[\\{<\u201c]+(.*?)[\\)\\]\\}>\u201d]+)" + "([{{ .CJK }}])"))
var cjk_bracket = regexp.MustCompile(re("([{{ .CJK }}])" + "([\\(\\[\\{<\u201c>])"))
var bracket_cjk = regexp.MustCompile(re("([\\)\\]\\}>\u201d<])" + "([{{ .CJK }}])"))
var fix_bracket = regexp.MustCompile(re("([\\(\\[\\{<\u201c]+)" + "(\\s*)" + "(.+?)" + "(\\s*)" + "([\\)\\]\\}>\u201d]+)"))

var fix_symbol = regexp.MustCompile(re("([{{ .CJK }}])" + "([~!;:,\\.\\?\u2026])" + "([A-Za-z0-9])"))

var cjk_ans = regexp.MustCompile(re("([{{ .CJK }}])([{{ .ANS }}@])"))
var ans_cjk = regexp.MustCompile(re("([{{ .ANS }}~!;:,\\.\\?\u2026])([{{ .CJK }}])"))

type pattern struct {
	CJK string
	ANS string
}

func re(exp string) string {
	var buf bytes.Buffer

	var tmpl = template.New("pangu")
	tmpl, _ = tmpl.Parse(exp)
	pat := pattern{
		CJK: cjk,
		ANS: ans,
	}
	tmpl.Execute(&buf, pat)
	expr := buf.String()

	return expr
}

// TextSpacing performs paranoid text spacing on text.
// It returns the processed text, with love.
func TextSpacing(text string) string {
	if len(text) < 2 {
		return text
	}

	text = cjk_quote.ReplaceAllString(text, "$1 $2")
	text = quote_cjk.ReplaceAllString(text, "$1 $2")
	text = fix_quote.ReplaceAllString(text, "$1$3$5")
	text = fix_single_quote.ReplaceAllString(text, "$1$3$4")

	text = cjk_hash.ReplaceAllString(text, "$1 $2")
	text = hash_cjk.ReplaceAllString(text, "$1 $3")

	text = cjk_operator_ans.ReplaceAllString(text, "$1 $2 $3")
	text = ans_operator_cjk.ReplaceAllString(text, "$1 $2 $3")

	oldText := text
	newText := cjk_bracket_cjk.ReplaceAllString(oldText, "$1 $2 $4")
	text = newText
	if oldText == newText {
		text = cjk_bracket.ReplaceAllString(text, "$1 $2")
		text = bracket_cjk.ReplaceAllString(text, "$1 $2")
	}
	text = fix_bracket.ReplaceAllString(text, "$1$3$5")

	text = fix_symbol.ReplaceAllString(text, "$1$2 $3")

	text = cjk_ans.ReplaceAllString(text, "$1 $2")
	text = ans_cjk.ReplaceAllString(text, "$1 $2")

	return text
}

// FileSpacing reads the file named by filename, performs paranoid text
// spacing on its contents and writes the processed content to w.
// A successful call returns err == nil.
func FileSpacing(filename string, w io.Writer) (err error) {
	// filename, err := filepath.Abs(filename)
	// if err != nil {
	// 	return err
	// }

	fr, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer fr.Close()

	br := bufio.NewReader(fr)
	bw := bufio.NewWriter(w)

	for {
		line, err := br.ReadString('\n')
		if err == nil {
			fmt.Fprint(bw, TextSpacing(line))
		} else {
			if err == io.EOF {
				fmt.Fprint(bw, TextSpacing(line))
				break
			}
			return err
		}
	}
	defer bw.Flush()

	return nil
}
<commit_msg>use map instead of struct for template context<commit_after>package pangu

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"regexp"
	"text/template"
)

const VERSION = "1.1.0"

// CJK is short for Chinese, Japanese and Korean.
//
// The constant cjk contains following Unicode blocks:
// 	\u2e80-\u2eff CJK Radicals Supplement
// 	\u2f00-\u2fdf Kangxi Radicals
// 	\u3040-\u309f Hiragana
// 	\u30a0-\u30ff Katakana
// 	\u3100-\u312f Bopomofo
// 	\u3200-\u32ff Enclosed CJK Letters and Months
// 	\u3400-\u4dbf CJK Unified Ideographs Extension A
// 	\u4e00-\u9fff CJK Unified Ideographs
// 	\uf900-\ufaff CJK Compatibility Ideographs
//
// For more information about Unicode blocks, see
// 	http://unicode-table.com/en/
const cjk = "" +
	"\u2e80-\u2eff" +
	"\u2f00-\u2fdf" +
	"\u3040-\u309f" +
	"\u30a0-\u30ff" +
	"\u3100-\u312f" +
	"\u3200-\u32ff" +
	"\u3400-\u4dbf" +
	"\u4e00-\u9fff" +
	"\uf900-\ufaff"

// ANS is short for Alphabets, Numbers
// and Symbols (`~!@#$%^&*()-_=+[]{}\|;:'",<.>/?).
//
// The constant ans doesn't contain all symbols above.
const ans = "A-Za-z0-9`\\$%\\^&\\*\\-=\\+\\\\|/\u00a1-\u00ff\u2022\u2027\u2150-\u218f"

var cjk_quote = regexp.MustCompile(re("([{{ .CJK }}])" + "([\"'])"))
var quote_cjk = regexp.MustCompile(re("([\"'])" + "([{{ .CJK }}])"))
var fix_quote = regexp.MustCompile(re("([\"'\\(\\[\\{<\u201c])" + "(\\s*)" + "(.+?)" + "(\\s*)" + "([\"'\\)\\]\\}>\u201d])"))
var fix_single_quote = regexp.MustCompile(re("([{{ .CJK }}])" + "( )" + "(')" + "([A-Za-z])"))

var cjk_hash = regexp.MustCompile(re("([{{ .CJK }}])" + "(#(\\S+))"))
var hash_cjk = regexp.MustCompile(re("((\\S+)#)" + "([{{ .CJK }}])"))

var cjk_operator_ans = regexp.MustCompile(re("([{{ .CJK }}])" + "([\\+\\-\\*/=&\\|<>])" + "([A-Za-z0-9])"))
var ans_operator_cjk = regexp.MustCompile(re("([A-Za-z0-9])" + "([\\+\\-\\*/=&\\|<>])" + "([{{ .CJK }}])"))

var cjk_bracket_cjk = regexp.MustCompile(re("([{{ .CJK }}])" + "([\\(\\[\\{<\u201c]+(.*?)[\\)\\]\\}>\u201d]+)" + "([{{ .CJK }}])"))
var cjk_bracket = regexp.MustCompile(re("([{{ .CJK }}])" + "([\\(\\[\\{<\u201c>])"))
var bracket_cjk = regexp.MustCompile(re("([\\)\\]\\}>\u201d<])" + "([{{ .CJK }}])"))
var fix_bracket = regexp.MustCompile(re("([\\(\\[\\{<\u201c]+)" + "(\\s*)" + "(.+?)" + "(\\s*)" + "([\\)\\]\\}>\u201d]+)"))

var fix_symbol = regexp.MustCompile(re("([{{ .CJK }}])" + "([~!;:,\\.\\?\u2026])" + "([A-Za-z0-9])"))

var cjk_ans = regexp.MustCompile(re("([{{ .CJK }}])([{{ .ANS }}@])"))
var ans_cjk = regexp.MustCompile(re("([{{ .ANS }}~!;:,\\.\\?\u2026])([{{ .CJK }}])"))

var context = map[string]string{
	"CJK": cjk,
	"ANS": ans,
}

func re(exp string) string {
	var buf bytes.Buffer

	var tmpl = template.New("pangu")
	tmpl, _ = tmpl.Parse(exp)
	tmpl.Execute(&buf, context)
	expr := buf.String()

	return expr
}

// TextSpacing performs paranoid text spacing on text.
// It returns the processed text, with love.
func TextSpacing(text string) string {
	if len(text) < 2 {
		return text
	}

	text = cjk_quote.ReplaceAllString(text, "$1 $2")
	text = quote_cjk.ReplaceAllString(text, "$1 $2")
	text = fix_quote.ReplaceAllString(text, "$1$3$5")
	text = fix_single_quote.ReplaceAllString(text, "$1$3$4")

	text = cjk_hash.ReplaceAllString(text, "$1 $2")
	text = hash_cjk.ReplaceAllString(text, "$1 $3")

	text = cjk_operator_ans.ReplaceAllString(text, "$1 $2 $3")
	text = ans_operator_cjk.ReplaceAllString(text, "$1 $2 $3")

	oldText := text
	newText := cjk_bracket_cjk.ReplaceAllString(oldText, "$1 $2 $4")
	text = newText
	if oldText == newText {
		text = cjk_bracket.ReplaceAllString(text, "$1 $2")
		text = bracket_cjk.ReplaceAllString(text, "$1 $2")
	}
	text = fix_bracket.ReplaceAllString(text, "$1$3$5")

	text = fix_symbol.ReplaceAllString(text, "$1$2 $3")

	text = cjk_ans.ReplaceAllString(text, "$1 $2")
	text = ans_cjk.ReplaceAllString(text, "$1 $2")

	return text
}

// FileSpacing reads the file named by filename, performs paranoid text
// spacing on its contents and writes the processed content to w.
// A successful call returns err == nil.
func FileSpacing(filename string, w io.Writer) (err error) {
	fr, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer fr.Close()

	br := bufio.NewReader(fr)
	bw := bufio.NewWriter(w)

	for {
		line, err := br.ReadString('\n')
		if err == nil {
			fmt.Fprint(bw, TextSpacing(line))
		} else {
			if err == io.EOF {
				fmt.Fprint(bw, TextSpacing(line))
				break
			}
			return err
		}
	}
	defer bw.Flush()

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	//"io"
	"os"
	"strings"
)

const dbg_parse_line = false

func fmt_line(data []string) string {
	s := bytes.NewBufferString("[")
	for i, v := range data {
		if i == 0 {
			fmt.Fprintf(s, "%q", v)
		} else {
			fmt.Fprintf(s, ", %q", v)
		}
	}
	fmt.Fprintf(s, "]")
	return string(s.Bytes())
}

// dropCR drops a terminal \r from the data.
func dropCR(data []byte) []byte {
	if len(data) > 0 && data[len(data)-1] == '\r' {
		return data[0 : len(data)-1]
	}
	return data
}

func scan_line(data []byte, atEOF bool) (advance int, token []byte, err error) {
	advance, token, err = bufio.ScanLines(data, atEOF)
	return
	// sz := len(token)
	// if sz > 0 && token[sz-1] == '\\' {
	// 	return
	// }
}

type Parser struct {
	req       *ReqFile
	isPrivate bool

	table   map[string]ParseFunc
	f       *os.File
	scanner *bufio.Scanner
	ctx     []string
	tokens  []string
}

func (p *Parser) Close() error {
	if p.f == nil {
		return nil
	}
	err := p.f.Close()
	p.f = nil
	return err
}

func NewParser(fname string) (*Parser, error) {

	var err error
	f, err := os.Open(fname)
	if err != nil {
		return nil, err
	}

	scanner := bufio.NewScanner(bufio.NewReader(f))
	if scanner == nil {
		return nil, fmt.Errorf("cmt2yml: nil bufio.Scanner")
	}

	p := &Parser{
		table:   g_dispatch,
		f:       f,
		scanner: scanner,
		req:     &ReqFile{Filename: fname},
		tokens:  nil,
		ctx:     []string{tok_BEG_PUBLIC},
	}
	return p, nil
}

func (p *Parser) run() error {
	var err error
	bline := []byte{}
	for p.scanner.Scan() {
		data := p.scanner.Bytes()
		data = bytes.TrimSpace(data)
		data = bytes.Trim(data, " \t\r\n")

		if len(data) == 0 {
			continue
		}

		if data[0] == '#' {
			continue
		}

		idx := len(data) - 1
		if data[idx] == '\\' {
			bline = append(bline, ' ')
			bline = append(bline, data[:idx-1]...)
			continue
		} else {
			bline = append(bline, ' ')
			bline = append(bline, data...)
		}

		var tokens []string
		tokens, err = parse_line(bline)
		if err != nil {
			return err
		}
		p.tokens = tokens

		fct, ok := p.table[p.tokens[0]]
		if !ok {
			return fmt.Errorf("cmt2yml: unknown token [%v]", tokens[0])
		}
		err = fct(p)
		if err != nil {
			return err
		}
		bline = nil
	}

	return err
}

func parse_file(fname string) (*ReqFile, error) {
	fmt.Printf("req=%q\n", fname)
	p, err := NewParser(fname)
	if err != nil {
		return nil, err
	}
	defer p.Close()

	err = p.run()
	fmt.Printf("req=%q [done]\n", fname)
	return p.req, err
}

func parse_line(data []byte) ([]string, error) {
	var err error
	line := []string{}

	worder := bufio.NewScanner(bytes.NewBuffer(data))
	worder.Split(bufio.ScanWords)
	tokens := []string{}
	for worder.Scan() {
		tok := worder.Text()
		if tok != "" {
			tokens = append(tokens, worder.Text())
		}
	}

	my_printf := func(format string, args ...interface{}) (int, error) {
		return 0, nil
	}
	if dbg_parse_line {
		my_printf = func(format string, args ...interface{}) (int, error) {
			return fmt.Printf(format, args...)
		}
	}

	my_printf("===============\n")
	my_printf("tokens: %v\n", fmt_line(tokens))

	in_dquote := false
	in_squote := false
	for i := 0; i < len(tokens); i++ {
		tok := strings.Trim(tokens[i], " \t")
		my_printf("tok[%d]=%q    (q=%v)\n", i, tok, in_squote || in_dquote)
		if in_squote || in_dquote {
			if len(line) > 0 {
				ttok := tok
				if strings.HasPrefix(ttok, `"`) || strings.HasPrefix(ttok, "'") {
					ttok = ttok[1:]
				}
				if strings.HasSuffix(ttok, `"`) || strings.HasSuffix(ttok, "'") {
					if !strings.HasSuffix(ttok, `\"`) {
						ttok = ttok[:len(ttok)-1]
					}
				}
				ttok = strings.Trim(ttok, " \t")
				if len(ttok) > 0 {
					line_val := line[len(line)-1]
					line_sep := ""
					if len(line_val) > 0 {
						line_sep = " "
					}
					ttok = strings.Replace(ttok, `\"`, `"`, -1)
					line[len(line)-1] += line_sep + ttok
				}
			} else {
				panic("logic error")
			}
		} else {
			ttok := tok
			if strings.HasPrefix(ttok, `"`) || strings.HasPrefix(ttok, "'") {
				ttok = ttok[1:]
			}
			if strings.HasSuffix(ttok, `"`) || strings.HasSuffix(ttok, "'") {
				if !strings.HasSuffix(ttok, `\"`) {
					ttok = ttok[:len(ttok)-1]
				}
			}
			ttok = strings.Replace(ttok, `\"`, `"`, -1)
			line = append(line, strings.Trim(ttok, " \t"))
		}
		if len(tok) == 1 && strings.HasPrefix(tok, "\"") {
			in_dquote = !in_dquote
			continue
		}
		if len(tok) == 1 && strings.HasPrefix(tok, "'") {
			in_squote = !in_squote
			continue
		}
		if strings.HasPrefix(tok, "\"") && !strings.HasSuffix(tok, "\"") {
			in_dquote = !in_dquote
			my_printf("--> dquote: %v -> %v\n", !in_dquote, in_dquote)
		}
		if strings.HasPrefix(tok, "'") && !strings.HasSuffix(tok, "'") {
			in_squote = !in_squote
			my_printf("--> squote: %v -> %v\n", !in_squote, in_squote)
		}
		if in_dquote && strings.HasSuffix(tok, "\"") && !strings.HasSuffix(tok, `\""`) {
			in_dquote = !in_dquote
			my_printf("<-- dquote: %v -> %v\n", !in_dquote, in_dquote)
		}
		if in_squote && strings.HasSuffix(tok, "'") {
			in_squote = !in_squote
			my_printf("<-- squote: %v -> %v\n", !in_squote, in_squote)
		}
	}

	return line, err
}

// EOF
<commit_msg>parse: dont display DONE when an error occured<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	//"io"
	"os"
	"strings"
)

const dbg_parse_line = false

func fmt_line(data []string) string {
	s := bytes.NewBufferString("[")
	for i, v := range data {
		if i == 0 {
			fmt.Fprintf(s, "%q", v)
		} else {
			fmt.Fprintf(s, ", %q", v)
		}
	}
	fmt.Fprintf(s, "]")
	return string(s.Bytes())
}

// dropCR drops a terminal \r from the data.
func dropCR(data []byte) []byte {
	if len(data) > 0 && data[len(data)-1] == '\r' {
		return data[0 : len(data)-1]
	}
	return data
}

func scan_line(data []byte, atEOF bool) (advance int, token []byte, err error) {
	advance, token, err = bufio.ScanLines(data, atEOF)
	return
	// sz := len(token)
	// if sz > 0 && token[sz-1] == '\\' {
	// 	return
	// }
}

type Parser struct {
	req       *ReqFile
	isPrivate bool

	table   map[string]ParseFunc
	f       *os.File
	scanner *bufio.Scanner
	ctx     []string
	tokens  []string
}

func (p *Parser) Close() error {
	if p.f == nil {
		return nil
	}
	err := p.f.Close()
	p.f = nil
	return err
}

func NewParser(fname string) (*Parser, error) {

	var err error
	f, err := os.Open(fname)
	if err != nil {
		return nil, err
	}

	scanner := bufio.NewScanner(bufio.NewReader(f))
	if scanner == nil {
		return nil, fmt.Errorf("cmt2yml: nil bufio.Scanner")
	}

	p := &Parser{
		table:   g_dispatch,
		f:       f,
		scanner: scanner,
		req:     &ReqFile{Filename: fname},
		tokens:  nil,
		ctx:     []string{tok_BEG_PUBLIC},
	}
	return p, nil
}

func (p *Parser) run() error {
	var err error
	bline := []byte{}
	for p.scanner.Scan() {
		data := p.scanner.Bytes()
		data = bytes.TrimSpace(data)
		data = bytes.Trim(data, " \t\r\n")

		if len(data) == 0 {
			continue
		}

		if data[0] == '#' {
			continue
		}

		idx := len(data) - 1
		if data[idx] == '\\' {
			bline = append(bline, ' ')
			bline = append(bline, data[:idx-1]...)
			continue
		} else {
			bline = append(bline, ' ')
			bline = append(bline, data...)
		}

		var tokens []string
		tokens, err = parse_line(bline)
		if err != nil {
			return err
		}
		p.tokens = tokens

		fct, ok := p.table[p.tokens[0]]
		if !ok {
			return fmt.Errorf("cmt2yml: unknown token [%v]", tokens[0])
		}
		err = fct(p)
		if err != nil {
			return err
		}
		bline = nil
	}

	return err
}

func parse_file(fname string) (*ReqFile, error) {
	fmt.Printf("req=%q\n", fname)
	p, err := NewParser(fname)
	if err != nil {
		return nil, err
	}
	defer p.Close()

	err = p.run()
	if err != nil {
		fmt.Printf("req=%q [ERR]\n", fname)
		return nil, err
	}
	fmt.Printf("req=%q [done]\n", fname)
	return p.req, err
}

func parse_line(data []byte) ([]string, error) {
	var err error
	line := []string{}

	worder := bufio.NewScanner(bytes.NewBuffer(data))
	worder.Split(bufio.ScanWords)
	tokens := []string{}
	for worder.Scan() {
		tok := worder.Text()
		if tok != "" {
			tokens = append(tokens, worder.Text())
		}
	}

	my_printf := func(format string, args ...interface{}) (int, error) {
		return 0, nil
	}
	if dbg_parse_line {
		my_printf = func(format string, args ...interface{}) (int, error) {
			return fmt.Printf(format, args...)
		}
	}

	my_printf("===============\n")
	my_printf("tokens: %v\n", fmt_line(tokens))

	in_dquote := false
	in_squote := false
	for i := 0; i < len(tokens); i++ {
		tok := strings.Trim(tokens[i], " \t")
		my_printf("tok[%d]=%q    (q=%v)\n", i, tok, in_squote || in_dquote)
		if in_squote || in_dquote {
			if len(line) > 0 {
				ttok := tok
				if strings.HasPrefix(ttok, `"`) || strings.HasPrefix(ttok, "'") {
					ttok = ttok[1:]
				}
				if strings.HasSuffix(ttok, `"`) || strings.HasSuffix(ttok, "'") {
					if !strings.HasSuffix(ttok, `\"`) {
						ttok = ttok[:len(ttok)-1]
					}
				}
				ttok = strings.Trim(ttok, " \t")
				if len(ttok) > 0 {
					line_val := line[len(line)-1]
					line_sep := ""
					if len(line_val) > 0 {
						line_sep = " "
					}
					ttok = strings.Replace(ttok, `\"`, `"`, -1)
					line[len(line)-1] += line_sep + ttok
				}
			} else {
				panic("logic error")
			}
		} else {
			ttok := tok
			if strings.HasPrefix(ttok, `"`) || strings.HasPrefix(ttok, "'") {
				ttok = ttok[1:]
			}
			if strings.HasSuffix(ttok, `"`) || strings.HasSuffix(ttok, "'") {
				if !strings.HasSuffix(ttok, `\"`) {
					ttok = ttok[:len(ttok)-1]
				}
			}
			ttok = strings.Replace(ttok, `\"`, `"`, -1)
			line = append(line, strings.Trim(ttok, " \t"))
		}
		if len(tok) == 1 && strings.HasPrefix(tok, "\"") {
			in_dquote = !in_dquote
			continue
		}
		if len(tok) == 1 && strings.HasPrefix(tok, "'") {
			in_squote = !in_squote
			continue
		}
		if strings.HasPrefix(tok, "\"") && !strings.HasSuffix(tok, "\"") {
			in_dquote = !in_dquote
			my_printf("--> dquote: %v -> %v\n", !in_dquote, in_dquote)
		}
		if strings.HasPrefix(tok, "'") && !strings.HasSuffix(tok, "'") {
			in_squote = !in_squote
			my_printf("--> squote: %v -> %v\n", !in_squote, in_squote)
		}
		if in_dquote && strings.HasSuffix(tok, "\"") && !strings.HasSuffix(tok, `\""`) {
			in_dquote = !in_dquote
			my_printf("<-- dquote: %v -> %v\n", !in_dquote, in_dquote)
		}
		if in_squote && strings.HasSuffix(tok, "'") {
			in_squote = !in_squote
			my_printf("<-- squote: %v -> %v\n", !in_squote, in_squote)
		}
	}

	return line, err
}

// EOF
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"net/url"
	"path"
	"strconv"
	"sync"
	//	"github.com/davecgh/go-spew/spew"
)

var (
	derpiURL = url.URL{
		Scheme: "https",
		Host:   "trixiebooru.org",
	}
	derpiquery = make(url.Values)
)

//RawImage contains data we got from API that needs to be modified before further usage
type RawImage struct {
	Imgid          string `json:"id"`
	URL            string `json:"image"`
	Score          int    `json:"score"`
	OriginalFormat string `json:"original_format"`
	Faves          int    `json:"faves"`
}

//Image contains data needed to filter fetch and save image
type Image struct {
	Imgid    string
	URL      *url.URL
	Filename string
	Score    int
	Faves    int
}

//Search returns to us array of searched images...
type Search struct {
	Images []RawImage `json:"search"`
}

//ImageCh is a channel of image data. You can put images into channel by parsing
//Derpibooru API by id(s) or  by tags and you can download images that are already
//in channel
type ImageCh chan Image

//Push gets unmarchalled JSON info, massages it and plugs it into channel so it
//would be processed in other places
func trim(dat RawImage) Image {

	fn := dat.Imgid + "." + dat.OriginalFormat
	tu, _ := url.Parse(dat.URL)
	tu.Scheme = derpiURL.Scheme
	tu.Path = path.Dir(tu.Path) + "/" + fn

	return Image{
		Imgid:    dat.Imgid,
		Filename: fn,
		URL:      tu,
		Score:    dat.Score,
		Faves:    dat.Faves,
	}
}

//ParseImg gets image IDs, fetches information about those images from Derpibooru and pushes them into the channel.
func (imgchan ImageCh) ParseImg(ids []int, key string) {

	for _, imgid := range ids {

		if isInterrupted() {
			break
		}

		derpiURL.Path = strconv.Itoa(imgid) + ".json"
		derpiURL.RawQuery = ""

		lInfo("Getting image info at:", derpiURL.String())
		if key != "" {
			derpiquery.Add("key", key)
		}
		derpiURL.RawQuery = derpiquery.Encode()
		body, err := getJSON(derpiURL.String())
		if err != nil {
			lErr(err)
			break
		}
		var dat RawImage
		if err := json.Unmarshal(body, &dat); //transforming json into native map

		err != nil {
			lErr(err)
			continue
		}

		imgchan <- trim(dat)
	}

	close(imgchan) //closing channel, we are done here

	return
}

//DlImg reads image data from channel and downloads specified images to disc
func (imgchan ImageCh) downloadImages(opts *Config) {

	lInfo("Worker started; reading channel") //nice notification that we are not forgotten
	var n int
	var size int64
	var l sync.Mutex
	var wg sync.WaitGroup
	for k := 0; k < 4; k++ {
		wg.Add(1)
		go func() {
			for imgdata := range imgchan {

				lInfo("Saving as", imgdata.Filename)

				tsize, ok := imgdata.saveImage(opts)
				l.Lock()
				size += tsize
				if ok {
					n++
				}
				l.Unlock()
			}
			wg.Done()
		}()
	}
	wg.Wait()
	lInfof("Downloaded %d images, for a total of %s", n, fmtbytes(float64(size)))
}

//ParseTag gets image tags, fetches information about all images it could from Derpibooru and pushes them into the channel.
func (imgchan ImageCh) ParseTag(opts *TagOpts, key string) {

	//Unlike main, I don't see how I could separate bits out to decrease complexity
	derpiURL.Path = "search.json"
	derpiquery.Add("sbq", opts.Tag)
	derpiURL.RawQuery = derpiquery.Encode()
	lInfo("Searching as", derpiURL.String())

	if key != "" {
		derpiquery.Add("key", key)
	}

	for page := opts.StartPage; opts.StopPage == 0 || page <= opts.StopPage; page++ {

		if isInterrupted() {
			break
		}

		lInfo("Searching page", page)
		derpiquery.Set("page", strconv.Itoa(page))
		derpiURL.RawQuery = derpiquery.Encode()

		body, err := getJSON(derpiURL.String())
		if err != nil {
			lErr("Error while getting json from page ", page)
			lErr(err)
			break
		}

		var dats Search
		err = json.Unmarshal(body, &dats)

		if err != nil {
			lErr("Error while parsing search page", page)
			lErr(err)
			if serr, ok := err.(*json.SyntaxError); ok { //In case crap was still given, we are looking at it.
				lErr("Occurred at offset: ", serr.Offset)
			}
			break

		}

		if len(dats.Images) == 0 {
			lInfo("Pages are all over") //Does not mean that process is over.
			break
		} //exit due to finishing all pages

		for _, dat := range dats.Images {
			imgchan <- trim(dat)
		}

	}

	close(imgchan)
}
<commit_msg>Returning to original site, not a mirror<commit_after>package main

import (
	"encoding/json"
	"net/url"
	"path"
	"strconv"
	"sync"
	//	"github.com/davecgh/go-spew/spew"
)

var (
	derpiURL = url.URL{
		Scheme: "https",
		Host:   "derpibooru.org",
	}
	derpiquery = make(url.Values)
)

//RawImage contains data we got from API that needs to be modified before further usage
type RawImage struct {
	Imgid          string `json:"id"`
	URL            string `json:"image"`
	Score          int    `json:"score"`
	OriginalFormat string `json:"original_format"`
	Faves          int    `json:"faves"`
}

//Image contains data needed to filter fetch and save image
type Image struct {
	Imgid    string
	URL      *url.URL
	Filename string
	Score    int
	Faves    int
}

//Search returns to us array of searched images...
type Search struct {
	Images []RawImage `json:"search"`
}

//ImageCh is a channel of image data. You can put images into channel by parsing
//Derpibooru API by id(s) or  by tags and you can download images that are already
//in channel
type ImageCh chan Image

//Push gets unmarchalled JSON info, massages it and plugs it into channel so it
//would be processed in other places
func trim(dat RawImage) Image {

	fn := dat.Imgid + "." + dat.OriginalFormat
	tu, _ := url.Parse(dat.URL)
	tu.Scheme = derpiURL.Scheme
	tu.Path = path.Dir(tu.Path) + "/" + fn

	return Image{
		Imgid:    dat.Imgid,
		Filename: fn,
		URL:      tu,
		Score:    dat.Score,
		Faves:    dat.Faves,
	}
}

//ParseImg gets image IDs, fetches information about those images from Derpibooru and pushes them into the channel.
func (imgchan ImageCh) ParseImg(ids []int, key string) {

	for _, imgid := range ids {

		if isInterrupted() {
			break
		}

		derpiURL.Path = strconv.Itoa(imgid) + ".json"
		derpiURL.RawQuery = ""

		lInfo("Getting image info at:", derpiURL.String())
		if key != "" {
			derpiquery.Add("key", key)
		}
		derpiURL.RawQuery = derpiquery.Encode()
		body, err := getJSON(derpiURL.String())
		if err != nil {
			lErr(err)
			break
		}
		var dat RawImage
		if err := json.Unmarshal(body, &dat); //transforming json into native map

		err != nil {
			lErr(err)
			continue
		}

		imgchan <- trim(dat)
	}

	close(imgchan) //closing channel, we are done here

	return
}

//DlImg reads image data from channel and downloads specified images to disc
func (imgchan ImageCh) downloadImages(opts *Config) {

	lInfo("Worker started; reading channel") //nice notification that we are not forgotten
	var n int
	var size int64
	var l sync.Mutex
	var wg sync.WaitGroup
	for k := 0; k < 4; k++ {
		wg.Add(1)
		go func() {
			for imgdata := range imgchan {

				lInfo("Saving as", imgdata.Filename)

				tsize, ok := imgdata.saveImage(opts)
				l.Lock()
				size += tsize
				if ok {
					n++
				}
				l.Unlock()
			}
			wg.Done()
		}()
	}
	wg.Wait()
	lInfof("Downloaded %d images, for a total of %s", n, fmtbytes(float64(size)))
}

//ParseTag gets image tags, fetches information about all images it could from Derpibooru and pushes them into the channel.
func (imgchan ImageCh) ParseTag(opts *TagOpts, key string) {

	//Unlike main, I don't see how I could separate bits out to decrease complexity
	derpiURL.Path = "search.json"
	derpiquery.Add("sbq", opts.Tag)
	derpiURL.RawQuery = derpiquery.Encode()
	lInfo("Searching as", derpiURL.String())

	if key != "" {
		derpiquery.Add("key", key)
	}

	for page := opts.StartPage; opts.StopPage == 0 || page <= opts.StopPage; page++ {

		if isInterrupted() {
			break
		}

		lInfo("Searching page", page)
		derpiquery.Set("page", strconv.Itoa(page))
		derpiURL.RawQuery = derpiquery.Encode()

		body, err := getJSON(derpiURL.String())
		if err != nil {
			lErr("Error while getting json from page ", page)
			lErr(err)
			break
		}

		var dats Search
		err = json.Unmarshal(body, &dats)

		if err != nil {
			lErr("Error while parsing search page", page)
			lErr(err)
			if serr, ok := err.(*json.SyntaxError); ok { //In case crap was still given, we are looking at it.
				lErr("Occurred at offset: ", serr.Offset)
			}
			break

		}

		if len(dats.Images) == 0 {
			lInfo("Pages are all over") //Does not mean that process is over.
			break
		} //exit due to finishing all pages

		for _, dat := range dats.Images {
			imgchan <- trim(dat)
		}

	}

	close(imgchan)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
)

type (
	// Sources struct to model a Type we want to ingest into the elasticsearch index
	// and the links we want to crawl/scrape information to store in our index/type
	Sources struct {
		Type  string
		Links []string
	}

	// Settings struct to model the settings we want to run our hermes application with.
	Settings struct {
		ElasticsearchHost  string `json:"esHost"`           // host address for the elasticsearch instance
		ElasticsearchIndex string `json:"esIndex"`          // index name you are going to ingest data into
		CrawlDelay         int    `json:"crawlDelay"`       // delay time for the crawler to abide to
		CancelDuration     int    `json:"cancelDuration"`   // time duration for canceling the crawler (immediate cancel)
		CancelAtURL        string `json:"cancelUrl"`        // specific URL to cancel the crawler at
		StopDuration       int    `json:"stopDuration"`     // time duration for stopping the crawler (processes links on queue after duration time)
		StopAtURL          string `json:"stopUrl"`          // specific URL to stop the crawler at for a specific "root"
		MemStatsInterval   int    `json:"memStatsInterval"` // display memory statistics at a given interval
		UserAgent          string `json:"userAgent"`        // set the user agent string for the crawler... to be polite and identify yourself
		WorkerIdleTTL      int    `json:"workerTimeout"`    // time-to-live for a host URL's goroutine
	}
)

// ParseLinks will parse the local data.json file that is in the same directory as the executable.
// The json file will be a "master" list of links we are going to crawl through.
func ParseLinks() Sources {
	var s Sources
	data, errRead := ioutil.ReadFile("./data.json")
	if errRead != nil {
		panic(errRead)
	}

	errUnmarshal := json.Unmarshal(data, &s)
	if errUnmarshal != nil {
		panic(errUnmarshal)
	}

	return s
}

// ParseSettings will parse a local settings.json file that is in the same directory as the executable.
// The json file will be all the configuration set by the user for the application.
func ParseSettings() Settings {
	var s Settings
	data, errRead := ioutil.ReadFile("./settings.json")
	if errRead != nil {
		panic(errRead)
	}

	errUnmarshal := json.Unmarshal(data, &s)
	if errUnmarshal != nil {
		panic(errUnmarshal)
	}

	return s
}
<commit_msg>changed types from int to time.Duration for interval purposes<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"time"
)

type (
	// Sources struct to model a Type we want to ingest into the elasticsearch index
	// and the links we want to crawl/scrape information to store in our index/type
	Sources struct {
		Type  string
		Links []string
	}

	// Settings struct to model the settings we want to run our hermes application with.
	Settings struct {
		ElasticsearchHost  string        `json:"esHost"`           // host address for the elasticsearch instance
		ElasticsearchIndex string        `json:"esIndex"`          // index name you are going to ingest data into
		CrawlDelay         time.Duration `json:"crawlDelay"`       // delay time for the crawler to abide to
		CancelDuration     time.Duration `json:"cancelDuration"`   // time duration for canceling the crawler (immediate cancel)
		CancelAtURL        string        `json:"cancelUrl"`        // specific URL to cancel the crawler at
		StopDuration       time.Duration `json:"stopDuration"`     // time duration for stopping the crawler (processes links on queue after duration time)
		StopAtURL          string        `json:"stopUrl"`          // specific URL to stop the crawler at for a specific "root"
		MemStatsInterval   time.Duration `json:"memStatsInterval"` // display memory statistics at a given interval
		UserAgent          string        `json:"userAgent"`        // set the user agent string for the crawler... to be polite and identify yourself
		WorkerIdleTTL      time.Duration `json:"workerTimeout"`    // time-to-live for a host URL's goroutine
	}
)

// ParseLinks will parse the local data.json file that is in the same directory as the executable.
// The json file will be a "master" list of links we are going to crawl through.
func ParseLinks() Sources {
	var s Sources
	data, errRead := ioutil.ReadFile("./data.json")
	if errRead != nil {
		panic(errRead)
	}

	errUnmarshal := json.Unmarshal(data, &s)
	if errUnmarshal != nil {
		panic(errUnmarshal)
	}

	return s
}

// ParseSettings will parse a local settings.json file that is in the same directory as the executable.
// The json file will be all the configuration set by the user for the application.
func ParseSettings() Settings {
	var s Settings
	data, errRead := ioutil.ReadFile("./settings.json")
	if errRead != nil {
		panic(errRead)
	}

	errUnmarshal := json.Unmarshal(data, &s)
	if errUnmarshal != nil {
		panic(errUnmarshal)
	}

	return s
}
<|endoftext|>


<commit_before>package grayt

import (
	"image"
	"log"
	"math/rand"
	"time"
)

type strategy struct {
}

func (s *strategy) traceImage(pxHigh, pxWide int, scene Scene, quality int) image.Image {

	acc := newAccumulator(pxHigh, pxWide)

	done := make(chan struct{})
	go func() {
		final := false
		for {
			select {
			case <-done:
				final = true
			default:
			}
			log.Printf("%4.1f%%\n", float64(acc.getTotal())/float64(quality*pxWide*pxHigh)*100)
			if final {
				log.Print("Done.")
				done <- struct{}{}
				return
			}
			time.Sleep(time.Second)
		}
	}()

	w := newWorld(scene.Entities)
	pxPitch := 2.0 / float64(pxWide)
	for i := 0; i < quality; i++ {
		for pxX := 0; pxX < pxWide; pxX++ {
			for pxY := 0; pxY < pxHigh; pxY++ {
				x := (float64(pxX-pxWide/2) + rand.Float64()) * pxPitch
				y := (float64(pxY-pxHigh/2) + rand.Float64()) * pxPitch * -1.0
				r := scene.Camera.MakeRay(x, y)
				r.Dir = r.Dir.Unit()
				acc.add(pxX, pxY, tracePath(w, r))
			}
		}
	}

	done <- struct{}{}
	<-done

	return acc.toImage(1.0)
}
<commit_msg>Add better status indicator<commit_after>package grayt

import (
	"image"
	"log"
	"math/rand"
	"time"
)

type strategy struct {
}

func (s *strategy) traceImage(pxHigh, pxWide int, scene Scene, quality int) image.Image {

	start := time.Now()

	acc := newAccumulator(pxHigh, pxWide)

	done := make(chan struct{})
	go func() {

		final := false

		var throughputSmoothed float64
		samples := 0
		now := time.Now()

		for {
			select {
			case <-done:
				final = true
			default:
			}

			var nowDelta time.Duration
			newNow := time.Now()
			nowDelta, now = newNow.Sub(now), newNow
			newSamples := acc.getTotal()
			var samplesDelta int
			samplesDelta, samples = newSamples-samples, newSamples
			throughput := float64(samplesDelta) / nowDelta.Seconds()
			const alpha = 0.001
			throughputSmoothed = throughputSmoothed*(1.0-alpha) + throughput*alpha

			elapsed := time.Now().Sub(start)
			elapsed = time.Nanosecond * time.Duration(elapsed.Nanoseconds()/1e7*1e7)

			log.Printf("%10s %10f\n", elapsed, throughputSmoothed)
			if final {
				log.Print("Done.")
				done <- struct{}{}
				return
			}
			time.Sleep(10 * time.Millisecond)
		}
	}()

	w := newWorld(scene.Entities)
	pxPitch := 2.0 / float64(pxWide)
	for i := 0; i < quality; i++ {
		for pxX := 0; pxX < pxWide; pxX++ {
			for pxY := 0; pxY < pxHigh; pxY++ {
				x := (float64(pxX-pxWide/2) + rand.Float64()) * pxPitch
				y := (float64(pxY-pxHigh/2) + rand.Float64()) * pxPitch * -1.0
				r := scene.Camera.MakeRay(x, y)
				r.Dir = r.Dir.Unit()
				acc.add(pxX, pxY, tracePath(w, r))
			}
		}
	}

	done <- struct{}{}
	<-done

	return acc.toImage(1.0)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
)

func main() {
	//favorites := []string{"zai", "fogged", "draskyl", "sing_sing", "d"}

	//https://api.twitch.tv/channels/sing_sing
	fmt.Println("Hello, 世界" + clientID())
}

func clientID() string {
	file, e := ioutil.ReadFile("./client.id")
	if e != nil {
		panic(e)
	}
	return string(file)
}
<commit_msg>parsing json<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
)

func main() {
	//favorites := []string{"zai", "fogged", "draskyl", "sing_sing", "d"}
	// parse with https://api.twitch.tv/kraken/streams?game=Dota+2&limit=10&channel=zai,sing_sing,draskyl

	// https://api.twitch.tv/kraken/streams/

	requestURL := "https://api.twitch.tv/kraken/streams?game=Dota+2&limit=10"
	res, err := http.Get(requestURL)
	if err != nil {
		log.Fatal(err)
	}
	streams, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	var dat JSONResult
	if err := json.Unmarshal(streams, &dat); err != nil {
		panic(err)
	}

	for _, g := range dat.Streams {
		fmt.Println("Stream: " + g.Channel.Name + " - " + g.Channel.Status + " - " + g.Channel.URL)
	}

	//https://api.twitch.tv/channels/sing_sing stream == null?

}

func clientID() string {
	file, e := ioutil.ReadFile("./client.id")
	if e != nil {
		panic(e)
	}
	return string(file)
}

// JSON structs
type JSONResult struct {
	Streams []JSONStreams `json:"streams"`
}

type JSONStreams struct {
	Channel JSONChannel `json:"channel"`
}

type JSONChannel struct {
	Name   string `json:"display_name"`
	URL    string `json:"url"`
	Status string `json:"status"`
	Views  int    `json:"views"`
}
<|endoftext|>


<commit_before>package storage

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/url"
)

func init() {
	log.SetFlags(log.Lshortfile)
}

type Metadata map[string]string

type ParsedURLNameQuery struct {
	Name  string
	Query url.Values
}

func ParseURLNameQuery(name string) (parsedURL ParsedURLNameQuery, err error) {
	uri, err := url.Parse(name)
	if err != nil {
		return
	}
	parsedURL.Query = uri.Query()
	uri.RawQuery = ""
	parsedURL.Name = uri.String()
	return
}

type BlobStorageStruct struct {
	AccountName string
	AccountKey  []byte
	UseHTTPS    bool
	BaseURL     string
	ApiVersion  string
}

func BlobStorageClientToJson(c BlobStorageClient) string {
	c2 := BlobStorageStruct{
		AccountName: c.client.accountName,
		AccountKey:  c.client.accountKey,
		UseHTTPS:    c.client.useHTTPS,
		BaseURL:     c.client.baseURL,
		ApiVersion:  c.client.apiVersion,
	}
	txt, _ := json.MarshalIndent(c2, "", "  ")
	return string(txt)
}

type SnapshotResponse struct {
	StatusCode int
	Headers    http.Header
}

func showRequest(name string, container string, verb string, uri string, headers map[string]string) (string, error) {
	s := struct {
		Name      string
		Container string
		Verb      string
		URI       string
		Headers   map[string]string
	}{name, container, verb, uri, headers}
	jbytes, err := json.MarshalIndent(s, "", "  ")
	return string(jbytes), err
}

func (b BlobStorageClient) Snapshot(container, name string, metaSnap Metadata) (res SnapshotResponse, err error) {

	////////////////////////////
	jbytes, _ := json.MarshalIndent(metaSnap, "", "  ")
	log.Printf("metaSnap = ...\n%s", string(jbytes))
	/////////////////////////////

	verb := "PUT"
	path := fmt.Sprintf("%s/%s", container, name)
	// blob cmd
	urlValues := url.Values{"comp": {"snapshot"}}

	uri := b.client.getEndpoint(blobServiceName, path, urlValues)
	headers := b.client.getStandardHeaders()
	headers["Content-Length"] = fmt.Sprintf("%v", 0)

	// Merge original blob metadata with snap metadata
	metaBlob, err := b.client.GetBlobService().GetBlobMetadata(container, name)
	if err != nil {
		return
	}

	////////////////////////////
	jbytes, _ = json.MarshalIndent(metaBlob, "", "  ")
	log.Printf("metaBnap = ...\n%s", string(jbytes))
	/////////////////////////////

	for key, value := range metaBlob {
		hv := fmt.Sprintf("x-ms-meta-%s", key)
		headers[hv] = value
	}
	for key, value := range metaSnap {
		hv := fmt.Sprintf("x-ms-meta-%s", key)
		headers[hv] = value
	}

	//req, _ := showRequest(name, container, verb, uri, headers)
	resp, err := b.client.exec(verb, uri, headers, nil)
	if err != nil {
		return
	}
	// No response body!!!
	defer resp.body.Close()

	res = SnapshotResponse{
		StatusCode: resp.statusCode,
		Headers:    resp.headers,
	}

	//////////////////////////////////////////////
	metaBlob, err = b.client.GetBlobService().GetBlobMetadata(container, name)
	if err != nil {
		return
	}
	jbytes, _ = json.MarshalIndent(metaBlob, "", "  ")
	log.Printf("metaBnap = ...\n%s", string(jbytes))
	//////////////////////////////////////////////

	return
}
<commit_msg>Removed snapshot tag copy<commit_after>package storage

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/url"
)

func init() {
	log.SetFlags(log.Lshortfile)
}

type Metadata map[string]string

type ParsedURLNameQuery struct {
	Name  string
	Query url.Values
}

func ParseURLNameQuery(name string) (parsedURL ParsedURLNameQuery, err error) {
	uri, err := url.Parse(name)
	if err != nil {
		return
	}
	parsedURL.Query = uri.Query()
	uri.RawQuery = ""
	parsedURL.Name = uri.String()
	return
}

type BlobStorageStruct struct {
	AccountName string
	AccountKey  []byte
	UseHTTPS    bool
	BaseURL     string
	ApiVersion  string
}

func BlobStorageClientToJson(c BlobStorageClient) string {
	c2 := BlobStorageStruct{
		AccountName: c.client.accountName,
		AccountKey:  c.client.accountKey,
		UseHTTPS:    c.client.useHTTPS,
		BaseURL:     c.client.baseURL,
		ApiVersion:  c.client.apiVersion,
	}
	txt, _ := json.MarshalIndent(c2, "", "  ")
	return string(txt)
}

type SnapshotResponse struct {
	StatusCode int
	Headers    http.Header
}

func renderRequest(name string, container string, verb string, uri string, headers map[string]string) string {
	s := struct {
		Name      string
		Container string
		Verb      string
		URI       string
		Headers   map[string]string
	}{name, container, verb, uri, headers}
	jbytes, err := json.MarshalIndent(s, "", "  ")
	if err != nil {
		return "unable to render request"
	}
	return string(jbytes)
}

func (b BlobStorageClient) Snapshot(container, name string, metaSnap Metadata) (res SnapshotResponse, err error) {
	debug := false

	if debug == true {
		jbytes, _ := json.MarshalIndent(metaSnap, "", "  ")
		log.Printf("metaSnap headers to add = ...\n%s", string(jbytes))
	}

	verb := "PUT"
	path := fmt.Sprintf("%s/%s", container, name)
	urlValues := url.Values{"comp": {"snapshot"}}

	uri := b.client.getEndpoint(blobServiceName, path, urlValues)
	headers := b.client.getStandardHeaders()
	headers["Content-Length"] = fmt.Sprintf("%v", 0)

	if debug == true {
		log.Printf("base request = %s", renderRequest(name, container, verb, uri, headers))
		//
	}

	// Add snapshot tags
	for key, value := range metaSnap {
		hv := fmt.Sprintf("x-ms-meta-%s", key)
		//log.Printf("metasnap headers[%s] = %s", hv, value)
		headers[hv] = value
	}

	if debug == true {
		log.Printf("snapshot request = %s", renderRequest(name, container, verb, uri, headers))
		//
	}

	resp, err := b.client.exec(verb, uri, headers, nil)
	if err != nil {
		return res, err
	}
	defer resp.body.Close()

	res = SnapshotResponse{
		StatusCode: resp.statusCode,
		Headers:    resp.headers,
	}

	if debug == true {
		metaBlob, err := b.client.GetBlobService().GetBlobMetadata(container, name)
		if err != nil {
			return res, err
		}
		jbytes, _ := json.MarshalIndent(metaBlob, "", "  ")
		log.Printf("after snapshot: basename metadata = ...\n%s", string(jbytes))

		snapname := name + "?snapshot=" + res.Headers.Get("X-Ms-Snapshot")
		log.Printf("snapname = %s", snapname)
		metaBlob, err = b.client.GetBlobService().GetBlobMetadata(container, snapname)
		if err != nil {
			return res, err
		}
		jbytes, _ = json.MarshalIndent(metaBlob, "", "  ")
		log.Printf("after snapshot: snapshot metadata = ...\n%s", string(jbytes))
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"sync"

	"health_monitor/api"
	"health_monitor/disk"
	"health_monitor/live"
	"health_monitor/setup"
	"health_monitor/utils"
	"health_monitor/webui"
)

// Flags holds the health_monitor command line arguments
type Flags struct {
	NoWebUI *bool
	NoCLI   *bool
	Quite   *bool
}

func main() {
	var (
		wg    sync.WaitGroup
		flags Flags
		chans [5]chan bool //Number of modules
	)

	for i := range chans {
		chans[i] = make(chan bool)
	}

	controlModule(chans, &wg)

	flags.NoWebUI = flag.Bool("nowebui", false, "Disables the web ui")
	flags.NoCLI = flag.Bool("nocli", false, "Disables cli")
	flags.Quite = flag.Bool("quite", false, "Disables all notifications except email")

	flag.Parse()

	go webui.RunServer(setup.ConfigVars.Port)
	if (*flags.NoCLI == true) || (*flags.NoWebUI == false) {
		fmt.Printf("[*] Server is up and running at 127.0.0.1:%s\n", setup.ConfigVars.Port)
	}
}

func controlModule(chans [5]chan bool, wg *sync.WaitGroup) {
	api.ControlChan = make(chan utils.Status)
	for {
		data := <-api.ControlChan
		switch data.Module {
		case "live":
			if data.Run {
				wg.Add(1)
				go live.Live(chans[0], wg)
			} else {
				chans[0] <- true
			}
		case "target":
			break
		case "disk":
			if data.Run {
				wg.Add(1)
				go disk.Disk(chans[2], wg)
			} else {
				chans[2] <- true
			}
		}
	}
}
<commit_msg>[misc] Added initialisation process<commit_after>package main

import (
	"flag"
	"fmt"
	"sync"

	"health_monitor/api"
	"health_monitor/disk"
	"health_monitor/live"
	"health_monitor/setup"
	"health_monitor/utils"
	"health_monitor/webui"
)

// Flags holds the health_monitor command line arguments
type Flags struct {
	NoWebUI *bool
	NoCLI   *bool
	Quite   *bool
}

func main() {
	var (
		wg    sync.WaitGroup
		flags Flags
		chans [5]chan bool //Number of modules
	)

	for i := range chans {
		chans[i] = make(chan bool)
	}

	flags.NoWebUI = flag.Bool("nowebui", false, "Disables the web ui")
	flags.NoCLI = flag.Bool("nocli", false, "Disables cli")
	flags.Quite = flag.Bool("quite", false, "Disables all notifications except email")

	flag.Parse()

	go webui.RunServer(setup.ConfigVars.Port)
	if (*flags.NoCLI == true) || (*flags.NoWebUI == false) {
		fmt.Printf("[*] Server is up and running at 127.0.0.1:%s\n", setup.ConfigVars.Port)
	}
	runModules(chans, &wg)
	controlModule(chans, &wg)
	setup.SaveStatus()
}

func controlModule(chans [5]chan bool, wg *sync.WaitGroup) {
	api.ControlChan = make(chan utils.Status)
	for {
		data := <-api.ControlChan
		switch data.Module {
		case "live":
			if data.Run && !setup.ModulesStatus.Live {
				wg.Add(1)
				go live.Live(chans[0], wg)
			} else if setup.ModulesStatus.Live {
				chans[0] <- true
			}
		case "target":
			break
		case "disk":
			if data.Run && !setup.ModulesStatus.Live {
				wg.Add(1)
				go disk.Disk(chans[2], wg)
			} else if setup.ModulesStatus.Live {
				chans[2] <- true
			}
		}
	}
}

func runModules(chans [5]chan bool, wg *sync.WaitGroup) {
	if setup.ModulesStatus.Live {
		wg.Add(1)
		go live.Live(chans[0], wg)
	}
	if setup.ModulesStatus.Target {

	}
	if setup.ModulesStatus.Disk {
		wg.Add(1)
		go disk.Disk(chans[2], wg)
	}
}
<|endoftext|>