<commit_before>// Author: Gergely Födémesi fgergo@gmail.com

// Boringstreamer looks for mp3 files and broadcasts via http (live streaming.)
//
// $ boringstreamer
//
// or
//
// c:\>boringstreamer.exe
//
// recursively looks for .mp3 files starting from "/" and broadcasts on port 4444 for at most 42 concurrent http clients.
//
// See -h for details.
//
// Browse to listen (e.g. http://localhost:4444/)
package main

import (
	"bytes"
	"bufio"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/tcolgate/mp3"

	_ "net/http/pprof"		// TODO(fgergo) remove when finished
)

var (
	addr           = flag.String("addr", ":4444", "listen on address (:port or host:port)")
	maxConnections = flag.Int("max", 42, "set maximum number of streaming connections")
	recursively    = flag.Bool("r", true, "recursively look for music starting from path")
	verbose        = flag.Bool("v", false, "display verbose messages")
)

var debugging bool // controlled by hidden command line argument -debug

// like /dev/null
type nullWriter struct {}

func (nw nullWriter) Write(p []byte) (n int, err error) {
	return len(p), nil
}

type streamFrame []byte

// client's event
type broadcastResult struct {
	qid int
	err error
}

// After a start() mux broadcasts audio stream to subscribed clients (ie. to http servers).
// Clients subscribe() and unsubscribe by writing to result chanel.
type mux struct {
	sync.Mutex

	clients map[int]chan streamFrame // set of listener clients to be notified
	result  chan broadcastResult     // clients share broadcast success-failure here
}

// subscribe(ch) adds ch to the set of channels to be received on by the clients when a new audio frame is available.
// Returns uniq client id (qid) for ch and a broadcast result channel for the client.
// Returns -1, nil if too many clients are already listening.
// clients: qid, br := m.subscribe(ch)
func (m *mux) subscribe(ch chan streamFrame) (int, chan broadcastResult) {
	m.Lock()
	// search for available qid
	qid := 0
	_, ok := m.clients[qid]
	for ; ok; _, ok = m.clients[qid] {
		if qid >= *maxConnections-1 {
			m.Unlock()
			return -1, nil
		}
		qid++
	}
	m.clients[qid] = ch
	m.Unlock()
	if *verbose {
		fmt.Printf("New connection (qid: %v), streaming to %v connections, at %v\n", qid, len(m.clients), time.Now().Format(time.Stamp))
	}

	return qid, m.result
}

// start() initializes a multiplexer for raw audio streams
// e.g: m := new(mux).start(path)
func (m *mux) start(path string) *mux {
	m.result = make(chan broadcastResult)
	m.clients = make(map[int]chan streamFrame)

	// flow structure: fs -> nextFile -> nextStream -> nextFrame -> subscribed http servers -> browsers
	nextFile := make(chan string)       // next file to be broadcast
	nextStream := make(chan io.Reader)  // next raw audio stream
	nextFrame := make(chan streamFrame) // next audio frame

	// generate randomized list of files available from path
	rand.Seed(time.Now().Unix()) // minimal randomness
	rescan := make(chan chan string)
	go func() {
		if path == "-" {
			return
		}

		for {
			files := <-rescan

			t0 := time.Now()
			notified := false
			filepath.Walk(path, func(wpath string, info os.FileInfo, err error) error {
				// notify user if no audio files are found after 4 seconds of walking path recursively
				dt := time.Now().Sub(t0)
				if dt > 4*time.Second && !notified && *verbose {
					fmt.Printf("Still looking for first audio file under %#v to broadcast, after %v... Maybe try -h flag.\n", path, dt)
					notified = true
				}

				if err != nil {
					return nil
				}
				if !info.Mode().IsRegular() {
					return nil
				}
				probably := strings.HasSuffix(strings.ToLower(info.Name()), ".mp3") // probably mp3
				if !info.IsDir() && !probably {
					return nil
				}

				files <- wpath // found file

				return nil
			})
			close(files)
			time.Sleep(1 * time.Second) // if no files are found, poll at least with 1Hz
		}
	}()

	// buffer and shuffle
	go func() {
		if path == "-" {
			return
		}

		for {
			files := make(chan string)
			rescan <- files

			shuffled := make([]string, 0) // randomized set of files

			for f := range files {
				select {
				case <-time.After(100 * time.Millisecond): // start playing as soon as possible, but wait at least 0.1 second for shuffling
					nextFile <- f
					if *verbose {
						fmt.Printf("Next: %v\n", f)
					}
				default:
					// shuffle files for random playback
					// (random permutation)
					if len(shuffled) == 0 {
						shuffled = append(shuffled, f)
					} else {
						i := rand.Intn(len(shuffled))
						shuffled = append(shuffled, shuffled[i])
						shuffled[i] = f
					}
				}
			}

			// queue shuffled files
			for _, f := range shuffled {
				nextFile <- f
				if *verbose {
					fmt.Printf("Next: %v\n", f)
				}
			}
		}
	}()

	// open file
	go func() {
		if path == "-" {
			nextStream <- os.Stdin
			return
		}

		for {
			filename := <-nextFile
			f, err := os.Open(filename)
			if err != nil {
				if debugging {
					log.Printf("Skipped \"%v\", err=%v", filename, err)
				}
				continue
			}
			nextStream <- bufio.NewReaderSize(f, 1024*1024)
			if *verbose {
				fmt.Printf("Now playing: %v\n", filename)
			}
		}
	}()

	// decode stream to frames and delay for frame duration
	go func() {
		nullwriter := new(nullWriter)
		var cumwait time.Duration
		for {
			streamReader := <-nextStream
			d := mp3.NewDecoder(streamReader)
			var f mp3.Frame
			for {
				t0 := time.Now()
				tmp := log.Prefix()
				if !debugging {
					log.SetOutput(nullwriter) // hack to silence mp3 debug/log output
				} else {
					log.SetPrefix("info: mp3 decode msg: ")
				}
				err := d.Decode(&f)
				log.SetPrefix(tmp)
				if !debugging {
					log.SetOutput(os.Stderr)
				}
				if err == io.EOF {
					break
				}
				if err != nil {
					if debugging {
						log.Printf("Skipping frame, d.Decode() err=%v", err)
					}
					continue
				}
				buf, err := ioutil.ReadAll(f.Reader())
				if err != nil {
					if debugging {
						log.Printf("Skipping frame, ioutil.ReadAll() err=%v", err)
					}
					continue
				}
				nextFrame <- buf

				towait := f.Duration() - time.Now().Sub(t0)
				cumwait += towait // towait can be negative -> cumwait
				if cumwait > 1*time.Second {
					time.Sleep(cumwait)
					cumwait = 0
				}
			}
		}
	}()

	// broadcast frame to clients
	go func() {
		for {
			f := <-nextFrame
			// notify clients of new audio frame or let them quit
			m.Lock()
			for _, ch := range m.clients {
				m.Unlock()
				ch <- f
				br := <-m.result // handle quitting clients
				if br.err != nil {
					m.Lock()
					close(m.clients[br.qid])
					delete(m.clients, br.qid)
					nclients := len(m.clients)
					m.Unlock()
					if debugging {
						log.Printf("Connection exited, qid: %v, error %v. Now streaming to %v connections.", br.qid, br.err, nclients)
					} else if *verbose {
						fmt.Printf("Connection exited, qid: %v. Now streaming to %v connections, at %v\n", br.qid, nclients, time.Now().Format(time.Stamp))
					}
				}
				m.Lock()
			}
			m.Unlock()
		}
	}()

	return m
}

type streamHandler struct {
	*mux
}

// chrome and firefox play mp3 audio stream directly
// details: https://tools.ietf.org/html/draft-pantos-http-live-streaming-20
// search for "Packed Audio"
func (sh streamHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	now := time.Now().UTC()
	frames := make(chan streamFrame)
	qid, br := sh.subscribe(frames)
	if qid < 0 {
		log.Printf("Error: new connection request denied, already serving %v connections. See -h for details.", *maxConnections)
		w.WriteHeader(http.StatusTooManyRequests)
		return
	}

	w.Header().Set("Date", now.Format(http.TimeFormat))
	w.Header().Set("Connection", "keep-alive")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Content-Type", "audio/mpeg")
	w.Header().Set("Server", "BoringStreamer/4.0")

	// some browsers need ID3 tag to identify first frame as audio media to be played
	// minimal ID3 header to designate audio stream
	b := []byte{0x49, 0x44, 0x33, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
	_, err := io.Copy(w, bytes.NewReader(b))
	if err == nil {
		// broadcast mp3 stream to w
		broadcastTimeout := 44 * time.Second // timeout for slow clients
		result := make(chan error)
		m := sync.Mutex{}
		for {
			buf := <-frames

			go func(r chan error, b []byte) {
				m.Lock()
				_, err = io.Copy(w, bytes.NewReader(b))
				m.Unlock()
				r <- err
			}(result, buf)

			select {
			case err = <-result:
				if err != nil {
					break
				}
				br <- broadcastResult{qid, nil} // frame streamed, no error, send ack
			case <-time.After(broadcastTimeout): // it's an error if io.Copy() is not finished within broadcastTimeout, ServeHTTP should exit
				err = errors.New(fmt.Sprintf("timeout: %v", broadcastTimeout))
			}

			if err != nil {
				break
			}
		}
	}
	br <- broadcastResult{qid, err} // error, send nack
}

func main() {
	flag.Usage = func() {
		fmt.Printf("Usage: %s [flags] [path]\n", os.Args[0])
		fmt.Println("then browse to listen. (e.g. http://localhost:4444/)")
		fmt.Printf("%v does not follow links.\n", os.Args[0])
		fmt.Printf("To stream from standard input: %v -\n\n", os.Args[0])
		fmt.Println("flags:")
		flag.PrintDefaults()
	}
	flag.Parse()
	if len(flag.Args()) > 1 && flag.Args()[1] != "-debug" {
		flag.Usage()
		os.Exit(1)
	}

	path := "/"
	switch len(flag.Args()) {
	case 0:
		if *verbose {
			fmt.Printf("Using path %#v, see -h for details.\n", path)
		}
	case 1:
		path = flag.Args()[0]
	case 2:
		path = flag.Args()[0]
		debugging = true
	}

	// check if path is available
	if path != "-" {
		matches, err := filepath.Glob(path)
		if err != nil || len(matches) < 1 {
			fmt.Fprintf(os.Stderr, "Error: \"%v\" unavailable, nothing to play.\n", path)
			os.Exit(1)
		}

		err = os.Chdir(path)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error: \"%v\" unavailable, nothing to play. Error: %v\n", path, err)
			os.Exit(1)
		}
		path, err = os.Getwd()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error: \"%v\" unavailable, nothing to play. Error: %v\n", path, err)
			os.Exit(1)
		}

		if *verbose {
			fmt.Printf("Looking for files available from \"%v\" ...\n", path)
		}
	}

	if *verbose {
		fmt.Printf("Waiting for connections on %v\n", *addr)
	}

	// TODO(fgergo), remove when finished
	if debugging {
		// start profile serving page: http://ip.ad.dr.ess:6060/debug/pprof
		go func() {
			log.Println(http.ListenAndServe(":6060", nil))
		}()
	}

	// initialize and start mp3 streamer
	err := http.ListenAndServe(*addr, streamHandler{new(mux).start(path)})
	if err != nil {
		fmt.Fprintf(os.Stderr, "Exiting, error: %v\n", err) // log.Fatalf() race with log.SetPrefix()
		os.Exit(1)
	}
}
<commit_msg>add BUGS section to describe player bug, when different .mp3 files have different mp3 parameters<commit_after>// Author: Gergely Födémesi fgergo@gmail.com

/*

Boringstreamer looks for mp3 files and broadcasts via http (live streaming.)
	$ boringstreamer
or
	c:\>boringstreamer.exe

recursively looks for .mp3 files starting from "/" and broadcasts on port 4444 for
at most 42 concurrent http clients.

Details: see -h.

Browse to listen (e.g. http://localhost:4444/)

Bugs

A browser or player feature/bug:  Usually happens when boringstreamer streams
different mp3s with different sample rates (e.g. 44100 and 48000). If the sample
rate or bitrate or number of channels changes during playing, the player stops
playing the stream. Boringstreamer handles the different files, but the players stop playing mp3s.

Workaround 1: Refresh page in the browser when mp3 playing is stopped.

Workaround 2: Change all mp3s to uniform format. Doesn't matter which format, it just should be uniform.
For example with ffmpeg:
	ffmpeg -i source.mp3 -vn -ar 44100 -ac 2 -ab 128 -f mp3 output.mp3
*/
package main

import (
	"bytes"
	"bufio"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/tcolgate/mp3"

	_ "net/http/pprof"		// TODO(fgergo) remove when finished
)

var (
	addr           = flag.String("addr", ":4444", "listen on address (:port or host:port)")
	maxConnections = flag.Int("max", 42, "set maximum number of streaming connections")
	recursively    = flag.Bool("r", true, "recursively look for music starting from path")
	verbose        = flag.Bool("v", false, "display verbose messages")
)

var debugging bool // controlled by hidden command line argument -debug

// like /dev/null
type nullWriter struct {}

func (nw nullWriter) Write(p []byte) (n int, err error) {
	return len(p), nil
}

type streamFrame []byte

// client's event
type broadcastResult struct {
	qid int
	err error
}

// After a start() mux broadcasts audio stream to subscribed clients (ie. to http servers).
// Clients subscribe() and unsubscribe by writing to result chanel.
type mux struct {
	sync.Mutex

	clients map[int]chan streamFrame // set of listener clients to be notified
	result  chan broadcastResult     // clients share broadcast success-failure here
}

// subscribe(ch) adds ch to the set of channels to be received on by the clients when a new audio frame is available.
// Returns uniq client id (qid) for ch and a broadcast result channel for the client.
// Returns -1, nil if too many clients are already listening.
// clients: qid, br := m.subscribe(ch)
func (m *mux) subscribe(ch chan streamFrame) (int, chan broadcastResult) {
	m.Lock()
	// search for available qid
	qid := 0
	_, ok := m.clients[qid]
	for ; ok; _, ok = m.clients[qid] {
		if qid >= *maxConnections-1 {
			m.Unlock()
			return -1, nil
		}
		qid++
	}
	m.clients[qid] = ch
	m.Unlock()
	if *verbose {
		fmt.Printf("New connection (qid: %v), streaming to %v connections, at %v\n", qid, len(m.clients), time.Now().Format(time.Stamp))
	}

	return qid, m.result
}

// start() initializes a multiplexer for raw audio streams
// e.g: m := new(mux).start(path)
func (m *mux) start(path string) *mux {
	m.result = make(chan broadcastResult)
	m.clients = make(map[int]chan streamFrame)

	// flow structure: fs -> nextFile -> nextStream -> nextFrame -> subscribed http servers -> browsers
	nextFile := make(chan string)       // next file to be broadcast
	nextStream := make(chan io.Reader)  // next raw audio stream
	nextFrame := make(chan streamFrame) // next audio frame

	// generate randomized list of files available from path
	rand.Seed(time.Now().Unix()) // minimal randomness
	rescan := make(chan chan string)
	go func() {
		if path == "-" {
			return
		}

		for {
			files := <-rescan

			t0 := time.Now()
			notified := false
			filepath.Walk(path, func(wpath string, info os.FileInfo, err error) error {
				// notify user if no audio files are found after 4 seconds of walking path recursively
				dt := time.Now().Sub(t0)
				if dt > 4*time.Second && !notified && *verbose {
					fmt.Printf("Still looking for first audio file under %#v to broadcast, after %v... Maybe try -h flag.\n", path, dt)
					notified = true
				}

				if err != nil {
					return nil
				}
				if !info.Mode().IsRegular() {
					return nil
				}
				probably := strings.HasSuffix(strings.ToLower(info.Name()), ".mp3") // probably mp3
				if !info.IsDir() && !probably {
					return nil
				}

				files <- wpath // found file

				return nil
			})
			close(files)
			time.Sleep(1 * time.Second) // if no files are found, poll at least with 1Hz
		}
	}()

	// buffer and shuffle
	go func() {
		if path == "-" {
			return
		}

		for {
			files := make(chan string)
			rescan <- files

			shuffled := make([]string, 0) // randomized set of files

			for f := range files {
				select {
				case <-time.After(100 * time.Millisecond): // start playing as soon as possible, but wait at least 0.1 second for shuffling
					nextFile <- f
					if *verbose {
						fmt.Printf("Next: %v\n", f)
					}
				default:
					// shuffle files for random playback
					// (random permutation)
					if len(shuffled) == 0 {
						shuffled = append(shuffled, f)
					} else {
						i := rand.Intn(len(shuffled))
						shuffled = append(shuffled, shuffled[i])
						shuffled[i] = f
					}
				}
			}

			// queue shuffled files
			for _, f := range shuffled {
				nextFile <- f
				if *verbose {
					fmt.Printf("Next: %v\n", f)
				}
			}
		}
	}()

	// open file
	go func() {
		if path == "-" {
			nextStream <- os.Stdin
			return
		}

		for {
			filename := <-nextFile
			f, err := os.Open(filename)
			if err != nil {
				if debugging {
					log.Printf("Skipped \"%v\", err=%v", filename, err)
				}
				continue
			}
			nextStream <- bufio.NewReaderSize(f, 1024*1024)
			if *verbose {
				fmt.Printf("Now playing: %v\n", filename)
			}
		}
	}()

	// decode stream to frames and delay for frame duration
	go func() {
		nullwriter := new(nullWriter)
		var cumwait time.Duration
		for {
			streamReader := <-nextStream
			d := mp3.NewDecoder(streamReader)
			var f mp3.Frame
			for {
				t0 := time.Now()
				tmp := log.Prefix()
				if !debugging {
					log.SetOutput(nullwriter) // hack to silence mp3 debug/log output
				} else {
					log.SetPrefix("info: mp3 decode msg: ")
				}
				err := d.Decode(&f)
				log.SetPrefix(tmp)
				if !debugging {
					log.SetOutput(os.Stderr)
				}
				if err == io.EOF {
					break
				}
				if err != nil {
					if debugging {
						log.Printf("Skipping frame, d.Decode() err=%v", err)
					}
					continue
				}
				buf, err := ioutil.ReadAll(f.Reader())
				if err != nil {
					if debugging {
						log.Printf("Skipping frame, ioutil.ReadAll() err=%v", err)
					}
					continue
				}
				nextFrame <- buf

				towait := f.Duration() - time.Now().Sub(t0)
				cumwait += towait // towait can be negative -> cumwait
				if cumwait > 1*time.Second {
					time.Sleep(cumwait)
					cumwait = 0
				}
			}
		}
	}()

	// broadcast frame to clients
	go func() {
		for {
			f := <-nextFrame
			// notify clients of new audio frame or let them quit
			m.Lock()
			for _, ch := range m.clients {
				m.Unlock()
				ch <- f
				br := <-m.result // handle quitting clients
				if br.err != nil {
					m.Lock()
					close(m.clients[br.qid])
					delete(m.clients, br.qid)
					nclients := len(m.clients)
					m.Unlock()
					if debugging {
						log.Printf("Connection exited, qid: %v, error %v. Now streaming to %v connections.", br.qid, br.err, nclients)
					} else if *verbose {
						fmt.Printf("Connection exited, qid: %v. Now streaming to %v connections, at %v\n", br.qid, nclients, time.Now().Format(time.Stamp))
					}
				}
				m.Lock()
			}
			m.Unlock()
		}
	}()

	return m
}

type streamHandler struct {
	*mux
}

// chrome and firefox play mp3 audio stream directly
// details: https://tools.ietf.org/html/draft-pantos-http-live-streaming-20
// search for "Packed Audio"
func (sh streamHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	now := time.Now().UTC()
	frames := make(chan streamFrame)
	qid, br := sh.subscribe(frames)
	if qid < 0 {
		log.Printf("Error: new connection request denied, already serving %v connections. See -h for details.", *maxConnections)
		w.WriteHeader(http.StatusTooManyRequests)
		return
	}

	w.Header().Set("Date", now.Format(http.TimeFormat))
	w.Header().Set("Connection", "keep-alive")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Content-Type", "audio/mpeg")
	w.Header().Set("Server", "BoringStreamer/4.0")

	// some browsers need ID3 tag to identify first frame as audio media to be played
	// minimal ID3 header to designate audio stream
	b := []byte{0x49, 0x44, 0x33, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
	_, err := io.Copy(w, bytes.NewReader(b))
	if err == nil {
		// broadcast mp3 stream to w
		broadcastTimeout := 44 * time.Second // timeout for slow clients
		result := make(chan error)
		m := sync.Mutex{}
		for {
			buf := <-frames

			go func(r chan error, b []byte) {
				m.Lock()
				_, err = io.Copy(w, bytes.NewReader(b))
				m.Unlock()
				r <- err
			}(result, buf)

			select {
			case err = <-result:
				if err != nil {
					break
				}
				br <- broadcastResult{qid, nil} // frame streamed, no error, send ack
			case <-time.After(broadcastTimeout): // it's an error if io.Copy() is not finished within broadcastTimeout, ServeHTTP should exit
				err = errors.New(fmt.Sprintf("timeout: %v", broadcastTimeout))
			}

			if err != nil {
				break
			}
		}
	}
	br <- broadcastResult{qid, err} // error, send nack
}

func main() {
	flag.Usage = func() {
		fmt.Printf("Usage: %s [flags] [path]\n", os.Args[0])
		fmt.Println("then browse to listen. (e.g. http://localhost:4444/)")
		fmt.Printf("%v does not follow links.\n", os.Args[0])
		fmt.Printf("To stream from standard input: %v -\n\n", os.Args[0])
		fmt.Println("flags:")
		flag.PrintDefaults()
	}
	flag.Parse()
	if len(flag.Args()) > 1 && flag.Args()[1] != "-debug" {
		flag.Usage()
		os.Exit(1)
	}

	path := "/"
	switch len(flag.Args()) {
	case 0:
		if *verbose {
			fmt.Printf("Using path %#v, see -h for details.\n", path)
		}
	case 1:
		path = flag.Args()[0]
	case 2:
		path = flag.Args()[0]
		debugging = true
	}

	// check if path is available
	if path != "-" {
		matches, err := filepath.Glob(path)
		if err != nil || len(matches) < 1 {
			fmt.Fprintf(os.Stderr, "Error: \"%v\" unavailable, nothing to play.\n", path)
			os.Exit(1)
		}

		err = os.Chdir(path)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error: \"%v\" unavailable, nothing to play. Error: %v\n", path, err)
			os.Exit(1)
		}
		path, err = os.Getwd()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error: \"%v\" unavailable, nothing to play. Error: %v\n", path, err)
			os.Exit(1)
		}

		if *verbose {
			fmt.Printf("Looking for files available from \"%v\" ...\n", path)
		}
	}

	if *verbose {
		fmt.Printf("Waiting for connections on %v\n", *addr)
	}

	// TODO(fgergo), remove when finished
	if debugging {
		// start profile serving page: http://ip.ad.dr.ess:6060/debug/pprof
		go func() {
			log.Println(http.ListenAndServe(":6060", nil))
		}()
	}

	// initialize and start mp3 streamer
	err := http.ListenAndServe(*addr, streamHandler{new(mux).start(path)})
	if err != nil {
		fmt.Fprintf(os.Stderr, "Exiting, error: %v\n", err) // log.Fatalf() race with log.SetPrefix()
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package e2e

import (
	"testing"

	"gotest.tools/v3/icmd"

	. "github.com/docker/compose-cli/utils/e2e"
)

func TestCascadeStop(t *testing.T) {
	c := NewParallelE2eCLI(t, binDir)

	const projectName = "e2e-cascade-stop"

	t.Run("abort-on-container-exit", func(t *testing.T) {
		res := c.RunDockerOrExitError("compose", "-f", "./fixtures/cascade-stop-test/compose.yaml", "--project-name", projectName, "up", "--abort-on-container-exit")
		res.Assert(t, icmd.Expected{ExitCode: 1, Out: `should_fail_1 exited with code 1`})
		res.Assert(t, icmd.Expected{ExitCode: 1, Out: `Aborting on container exit...`})
	})

	t.Run("exit-code-from", func(t *testing.T) {
		res := c.RunDockerOrExitError("compose", "-f", "./fixtures/cascade-stop-test/compose.yaml", "--project-name", projectName, "up", "--exit-code-from=sleep")
		res.Assert(t, icmd.Expected{ExitCode: 137, Out: `should_fail_1 exited with code 1`})
		res.Assert(t, icmd.Expected{ExitCode: 137, Out: `Aborting on container exit...`})
	})

	t.Run("exit-code-from unknown", func(t *testing.T) {
		res := c.RunDockerOrExitError("compose", "-f", "./fixtures/cascade-stop-test/compose.yaml", "--project-name", projectName, "up", "--exit-code-from=unknown")
		res.Assert(t, icmd.Expected{ExitCode: 1, Err: `no such service: unknown`})
	})
}
<commit_msg>Add cascade test cleanup with `down`<commit_after>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package e2e

import (
	"testing"

	"gotest.tools/v3/icmd"

	. "github.com/docker/compose-cli/utils/e2e"
)

func TestCascadeStop(t *testing.T) {
	c := NewParallelE2eCLI(t, binDir)

	const projectName = "e2e-cascade-stop"

	t.Run("abort-on-container-exit", func(t *testing.T) {
		res := c.RunDockerOrExitError("compose", "-f", "./fixtures/cascade-stop-test/compose.yaml", "--project-name", projectName, "up", "--abort-on-container-exit")
		res.Assert(t, icmd.Expected{ExitCode: 1, Out: `should_fail_1 exited with code 1`})
		res.Assert(t, icmd.Expected{ExitCode: 1, Out: `Aborting on container exit...`})
	})

	t.Run("exit-code-from", func(t *testing.T) {
		res := c.RunDockerOrExitError("compose", "-f", "./fixtures/cascade-stop-test/compose.yaml", "--project-name", projectName, "up", "--exit-code-from=sleep")
		res.Assert(t, icmd.Expected{ExitCode: 137, Out: `should_fail_1 exited with code 1`})
		res.Assert(t, icmd.Expected{ExitCode: 137, Out: `Aborting on container exit...`})
	})

	t.Run("exit-code-from unknown", func(t *testing.T) {
		res := c.RunDockerOrExitError("compose", "-f", "./fixtures/cascade-stop-test/compose.yaml", "--project-name", projectName, "up", "--exit-code-from=unknown")
		res.Assert(t, icmd.Expected{ExitCode: 1, Err: `no such service: unknown`})
	})

	t.Run("down", func(t *testing.T) {
		_ = c.RunDockerCmd("compose", "--project-name", projectName, "down")
	})
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package utils_test

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/utils"
)

type insecureClientSuite struct {
	Server *httptest.Server
}

var _ = gc.Suite(&insecureClientSuite{})

type trivialResponseHandler struct{}

func (t *trivialResponseHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Greetings!\n")
}

func (s *insecureClientSuite) SetUpSuite(c *gc.C) {
}

func (s *insecureClientSuite) TearDownSuite(c *gc.C) {
}

func (s *insecureClientSuite) SetUpTest(c *gc.C) {
	s.Server = httptest.NewTLSServer(&trivialResponseHandler{})
}

func (s *insecureClientSuite) TearDownTest(c *gc.C) {
	if s.Server != nil {
		s.Server.Close()
	}
}

func (s *insecureClientSuite) TestDefaultClientFails(c *gc.C) {
	_, err := http.Get(s.Server.URL)
	c.Assert(err, gc.ErrorMatches, "(.|\n)*x509: certificate signed by unknown authority")
}

func (s *insecureClientSuite) TestInsecureClientSucceeds(c *gc.C) {
	response, err := utils.GetNonValidatingHTTPClient().Get(s.Server.URL)
	c.Assert(err, gc.IsNil)
	defer response.Body.Close()
	body, err := ioutil.ReadAll(response.Body)
	c.Assert(err, gc.IsNil)
	c.Check(string(body), gc.Equals, "Greetings!\n")
}

func (s *insecureClientSuite) TestInsecureClientCached(c *gc.C) {
	client1 := utils.GetNonValidatingHTTPClient()
	client2 := utils.GetNonValidatingHTTPClient()
	c.Check(client1, gc.Equals, client2)
}
<commit_msg>Add tests<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package utils_test

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/utils"
)

type httpSuite struct {
	Server *httptest.Server
}

var _ = gc.Suite(&httpSuite{})

type trivialResponseHandler struct{}

func (t *trivialResponseHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Greetings!\n")
}

func (s *httpSuite) SetUpSuite(c *gc.C) {
}

func (s *httpSuite) TearDownSuite(c *gc.C) {
}

func (s *httpSuite) SetUpTest(c *gc.C) {
	s.Server = httptest.NewTLSServer(&trivialResponseHandler{})
}

func (s *httpSuite) TearDownTest(c *gc.C) {
	if s.Server != nil {
		s.Server.Close()
	}
}

func (s *httpSuite) TestDefaultClientFails(c *gc.C) {
	_, err := http.Get(s.Server.URL)
	c.Assert(err, gc.ErrorMatches, "(.|\n)*x509: certificate signed by unknown authority")
}

func (s *httpSuite) TestValidatingClientGetter(c *gc.C) {
	client1 := utils.GetValidatingHTTPClient()
	client2 := utils.GetHTTPClient(utils.VerifySSLHostnames)
	c.Check(client1, gc.Equals, client2)
}

func (s *httpSuite) TestNonValidatingClientGetter(c *gc.C) {
	client1 := utils.GetNonValidatingHTTPClient()
	client2 := utils.GetHTTPClient(utils.NoVerifySSLHostnames)
	c.Check(client1, gc.Equals, client2)
}

func (s *httpSuite) TestValidatingClientFails(c *gc.C) {
	client := utils.GetValidatingHTTPClient()
	_, err := client.Get(s.Server.URL)
	c.Assert(err, gc.ErrorMatches, "(.|\n)*x509: certificate signed by unknown authority")
}

func (s *httpSuite) TestInsecureClientSucceeds(c *gc.C) {
	response, err := utils.GetNonValidatingHTTPClient().Get(s.Server.URL)
	c.Assert(err, gc.IsNil)
	defer response.Body.Close()
	body, err := ioutil.ReadAll(response.Body)
	c.Assert(err, gc.IsNil)
	c.Check(string(body), gc.Equals, "Greetings!\n")
}

func (s *httpSuite) TestInsecureClientCached(c *gc.C) {
	client1 := utils.GetNonValidatingHTTPClient()
	client2 := utils.GetNonValidatingHTTPClient()
	c.Check(client1, gc.Equals, client2)
}
<|endoftext|>


<commit_before>package mpawsses

import (
	"errors"
	"flag"
	"os"
	"time"

	"github.com/crowdmob/goamz/aws"
	mp "github.com/mackerelio/go-mackerel-plugin"
	ses "github.com/naokibtn/go-ses"
)

var graphdef = map[string]mp.Graphs{
	"ses.send24h": {
		Label: "SES Send (last 24h)",
		Unit:  "float",
		Metrics: []mp.Metrics{
			{Name: "Max24HourSend", Label: "Max"},
			{Name: "SentLast24Hours", Label: "Sent"},
		},
	},
	"ses.max_send_rate": {
		Label: "SES Max Send Rate",
		Unit:  "float",
		Metrics: []mp.Metrics{
			{Name: "MaxSendRate", Label: "MaxRate"},
		},
	},
	"ses.stats": {
		Label: "SES Stats",
		Unit:  "int",
		Metrics: []mp.Metrics{
			{Name: "Complaints", Label: "Complaints"},
			{Name: "DeliveryAttempts", Label: "DeliveryAttempts"},
			{Name: "Bounces", Label: "Bounces"},
			{Name: "Rejects", Label: "Rejects"},
		},
	},
}

// SESPlugin mackerel plugin for Amazon SES
type SESPlugin struct {
	Endpoint        string
	AccessKeyID     string
	SecretAccessKey string
}

// FetchMetrics interface for mackerel plugin
func (p SESPlugin) FetchMetrics() (map[string]float64, error) {
	if p.Endpoint == "" {
		return nil, errors.New("no endpoint")
	}

	auth, err := aws.GetAuth(p.AccessKeyID, p.SecretAccessKey, "", time.Now())
	if err != nil {
		return nil, err
	}

	sescfg := ses.Config{
		AccessKeyID:     auth.AccessKey,
		SecretAccessKey: auth.SecretKey,
		SecurityToken:   auth.Token(),
		Endpoint:        p.Endpoint,
	}

	stat := make(map[string]float64)
	quota, err := sescfg.GetSendQuota()
	if err == nil {
		stat["SentLast24Hours"] = quota.SentLast24Hours
		stat["Max24HourSend"] = quota.Max24HourSend
		stat["MaxSendRate"] = quota.MaxSendRate
	}

	datapoints, err := sescfg.GetSendStatistics()
	if err == nil {
		latest := ses.SendDataPoint{
			Timestamp: time.Unix(0, 0),
		}

		for _, dp := range datapoints {
			if latest.Timestamp.Before(dp.Timestamp) {
				latest = dp
			}
		}

		stat["Complaints"] = float64(latest.Complaints)
		stat["DeliveryAttempts"] = float64(latest.DeliveryAttempts)
		stat["Bounces"] = float64(latest.Bounces)
		stat["Rejects"] = float64(latest.Rejects)
	}

	return stat, nil
}

// GraphDefinition interface for mackerel plugin
func (p SESPlugin) GraphDefinition() map[string]mp.Graphs {
	return graphdef
}

// Do the plugin
func Do() {
	optEndpoint := flag.String("endpoint", "", "AWS Endpoint")
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	var ses SESPlugin

	ses.Endpoint = *optEndpoint
	ses.AccessKeyID = *optAccessKeyID
	ses.SecretAccessKey = *optSecretAccessKey

	helper := mp.NewMackerelPlugin(ses)
	helper.Tempfile = *optTempfile

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<commit_msg>[aws-ses] ses.stats is unit type<commit_after>package mpawsses

import (
	"errors"
	"flag"
	"os"
	"time"

	"github.com/crowdmob/goamz/aws"
	mp "github.com/mackerelio/go-mackerel-plugin"
	ses "github.com/naokibtn/go-ses"
)

var graphdef = map[string]mp.Graphs{
	"ses.send24h": {
		Label: "SES Send (last 24h)",
		Unit:  "float",
		Metrics: []mp.Metrics{
			{Name: "Max24HourSend", Label: "Max"},
			{Name: "SentLast24Hours", Label: "Sent"},
		},
	},
	"ses.max_send_rate": {
		Label: "SES Max Send Rate",
		Unit:  "float",
		Metrics: []mp.Metrics{
			{Name: "MaxSendRate", Label: "MaxRate"},
		},
	},
	"ses.stats": {
		Label: "SES Stats",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "Complaints", Label: "Complaints"},
			{Name: "DeliveryAttempts", Label: "DeliveryAttempts"},
			{Name: "Bounces", Label: "Bounces"},
			{Name: "Rejects", Label: "Rejects"},
		},
	},
}

// SESPlugin mackerel plugin for Amazon SES
type SESPlugin struct {
	Endpoint        string
	AccessKeyID     string
	SecretAccessKey string
}

// FetchMetrics interface for mackerel plugin
func (p SESPlugin) FetchMetrics() (map[string]float64, error) {
	if p.Endpoint == "" {
		return nil, errors.New("no endpoint")
	}

	auth, err := aws.GetAuth(p.AccessKeyID, p.SecretAccessKey, "", time.Now())
	if err != nil {
		return nil, err
	}

	sescfg := ses.Config{
		AccessKeyID:     auth.AccessKey,
		SecretAccessKey: auth.SecretKey,
		SecurityToken:   auth.Token(),
		Endpoint:        p.Endpoint,
	}

	stat := make(map[string]float64)
	quota, err := sescfg.GetSendQuota()
	if err == nil {
		stat["SentLast24Hours"] = quota.SentLast24Hours
		stat["Max24HourSend"] = quota.Max24HourSend
		stat["MaxSendRate"] = quota.MaxSendRate
	}

	datapoints, err := sescfg.GetSendStatistics()
	if err == nil {
		latest := ses.SendDataPoint{
			Timestamp: time.Unix(0, 0),
		}

		for _, dp := range datapoints {
			if latest.Timestamp.Before(dp.Timestamp) {
				latest = dp
			}
		}

		stat["Complaints"] = float64(latest.Complaints)
		stat["DeliveryAttempts"] = float64(latest.DeliveryAttempts)
		stat["Bounces"] = float64(latest.Bounces)
		stat["Rejects"] = float64(latest.Rejects)
	}

	return stat, nil
}

// GraphDefinition interface for mackerel plugin
func (p SESPlugin) GraphDefinition() map[string]mp.Graphs {
	return graphdef
}

// Do the plugin
func Do() {
	optEndpoint := flag.String("endpoint", "", "AWS Endpoint")
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	var ses SESPlugin

	ses.Endpoint = *optEndpoint
	ses.AccessKeyID = *optAccessKeyID
	ses.SecretAccessKey = *optSecretAccessKey

	helper := mp.NewMackerelPlugin(ses)
	helper.Tempfile = *optTempfile

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<|endoftext|>


<commit_before>//+build smoke

package smoke

import (
	"flag"
	"os"
	"testing"

	"github.com/opentable/sous/util/firsterr"
	"github.com/opentable/sous/util/testmatrix"
)

func TestMain(m *testing.M) {
	flag.Parse()
	testmatrix.Quiet = quiet()
	sup = testmatrix.Init(matrix, newFixture, func() error {
		return firsterr.Parallel().Set(
			func(e *error) { *e = resetSingularity() },
			func(e *error) { *e = stopPIDs() },
			func(e *error) { *e = nukeDockerRegistry() },
		)
	})
	exitCode := m.Run()
	sup.PrintSummary()
	os.Exit(exitCode)
}

func resetSingularity() error {
	envDesc := getEnvDesc()
	singularity := newSingularity(envDesc.SingularityURL())
	return singularity.Reset()
}

func nukeDockerRegistry() error {
	if err := doCMD("../../integration/test-registry", "docker-compose", "-sfv", "registry"); err != nil {
		return err
	}
	if err := doCMD(".", "docker", "volume", "rm", "test-registry_registrydata"); err != nil {
		return err
	}
	return doCMD("../../integration/test-registry", "docker-compose", "up", "-d", "registry")
}
<commit_msg>test/smoke: correct docker-compose invocation<commit_after>//+build smoke

package smoke

import (
	"flag"
	"os"
	"testing"

	"github.com/opentable/sous/util/firsterr"
	"github.com/opentable/sous/util/testmatrix"
)

func TestMain(m *testing.M) {
	flag.Parse()
	testmatrix.Quiet = quiet()
	sup = testmatrix.Init(matrix, newFixture, func() error {
		return firsterr.Parallel().Set(
			func(e *error) { *e = resetSingularity() },
			func(e *error) { *e = stopPIDs() },
			func(e *error) { *e = nukeDockerRegistry() },
		)
	})
	exitCode := m.Run()
	sup.PrintSummary()
	os.Exit(exitCode)
}

func resetSingularity() error {
	envDesc := getEnvDesc()
	singularity := newSingularity(envDesc.SingularityURL())
	return singularity.Reset()
}

func nukeDockerRegistry() error {
	if err := doCMD("../../integration/test-registry", "docker-compose", "rm", "-sfv", "registry"); err != nil {
		return err
	}
	if err := doCMD(".", "docker", "volume", "rm", "test-registry_registrydata"); err != nil {
		return err
	}
	return doCMD("../../integration/test-registry", "docker-compose", "up", "-d", "registry")
}
<|endoftext|>


<commit_before>//+build smoke

package smoke

import (
	"flag"
	"os"
	"strings"
	"testing"

	"github.com/opentable/sous/util/firsterr"
	"github.com/opentable/sous/util/testmatrix"
)

func TestMain(m *testing.M) {
	flag.Parse()
	testmatrix.Quiet = quiet()
	sup = testmatrix.Init(matrix, newFixture, func() error {
		return firsterr.Parallel().Set(
			func(e *error) { *e = resetSingularity() },
			func(e *error) { *e = stopPIDs() },
			func(e *error) { *e = nukeDockerRegistry() },
		)
	})
	exitCode := m.Run()
	sup.PrintSummary()
	os.Exit(exitCode)
}

func resetSingularity() error {
	envDesc := getEnvDesc()
	singularity := newSingularity(envDesc.SingularityURL())
	return singularity.Reset()
}

func nukeDockerRegistry() error {
	wd := "../../integration/test-registry"
	cid, err := doCMDCombinedOut(wd, "docker-compose", "ps", "-q", "registry")
	if err != nil {
		return err
	}
	if strings.TrimSpace(cid) != "" {
		if err := doCMD("", "docker-compose", "rm", "-sfv", "registry"); err != nil {
			return err
		}
	}
	vols, err := doCMDCombinedOut(wd, "docker", "volumes", "ls")
	if err != nil {
		return err
	}
	if strings.Contains(vols, "test-registry_registrydata") {
		if err := doCMD(".", "docker", "volume", "rm", "test-registry_registrydata"); err != nil {
			return err
		}
	}
	return doCMD("../../integration/test-registry", "docker-compose", "up", "-d", "registry")
}
<commit_msg>test/smoke: fix docker-compose dir<commit_after>//+build smoke

package smoke

import (
	"flag"
	"os"
	"strings"
	"testing"

	"github.com/opentable/sous/util/firsterr"
	"github.com/opentable/sous/util/testmatrix"
)

func TestMain(m *testing.M) {
	flag.Parse()
	testmatrix.Quiet = quiet()
	sup = testmatrix.Init(matrix, newFixture, func() error {
		return firsterr.Parallel().Set(
			func(e *error) { *e = resetSingularity() },
			func(e *error) { *e = stopPIDs() },
			func(e *error) { *e = nukeDockerRegistry() },
		)
	})
	exitCode := m.Run()
	sup.PrintSummary()
	os.Exit(exitCode)
}

func resetSingularity() error {
	envDesc := getEnvDesc()
	singularity := newSingularity(envDesc.SingularityURL())
	return singularity.Reset()
}

func nukeDockerRegistry() error {
	wd := "../../integration/test-registry"
	cid, err := doCMDCombinedOut(wd, "docker-compose", "ps", "-q", "registry")
	if err != nil {
		return err
	}
	if strings.TrimSpace(cid) != "" {
		if err := doCMD(wd, "docker-compose", "rm", "-sfv", "registry"); err != nil {
			return err
		}
	}
	vols, err := doCMDCombinedOut(wd, "docker", "volumes", "ls")
	if err != nil {
		return err
	}
	if strings.Contains(vols, "test-registry_registrydata") {
		if err := doCMD(wd, "docker", "volume", "rm", "test-registry_registrydata"); err != nil {
			return err
		}
	}
	return doCMD(wd, "docker-compose", "up", "-d", "registry")
}
<|endoftext|>


<commit_before>package whoson

import "time"

type ProtocolType int
type MethodType int
type ResultType int

const (
	/*
		" 1<<10"    1024
		" 2<<10"    2048
		" 4<<10"    4096
		" 8<<10"    8192
		"16<<10"   16384
		"32<<10"   32768
		"64<<10"   65536
		" 1<<20" 1048576 1M
	*/
	maxQueues       = 8 << 10
	udpByteSize     = 1472
	CRLF            = "\r\n"
	SessionTimeOut  = 10 * time.Second
	StoreDataExpire = 30 * time.Minute

	pUnkownProtocol ProtocolType = iota
	pTCP
	pUDP

	mUnkownMethod MethodType = iota
	mLogin
	mLogout
	mQuery
	mQuit

	rPositive ResultType = iota
	rNegative
	rBadRequest
)

var (
	MainStore Store = nil

	method = map[MethodType]string{
		mUnkownMethod: "NONE",
		mLogin:        "LOGIN",
		mLogout:       "LOGOUT",
		mQuery:        "QUERY",
		mQuit:         "QUIT",
	}

	result = map[ResultType]string{
		rPositive:   "+",
		rNegative:   "-",
		rBadRequest: "*",
	}

	methodFromString = map[string]MethodType{}
)

func init() {
	for i, v := range method {
		methodFromString[v] = i
	}
}
<commit_msg>Add global variables Logger and IDGenerator<commit_after>package whoson

import (
	"time"

	katsubushi "github.com/kayac/go-katsubushi"
	"go.uber.org/zap"
)

type ProtocolType int
type MethodType int
type ResultType int

const (
	/*
		" 1<<10"    1024
		" 2<<10"    2048
		" 4<<10"    4096
		" 8<<10"    8192
		"16<<10"   16384
		"32<<10"   32768
		"64<<10"   65536
		" 1<<20" 1048576 1M
	*/
	maxQueues       = 8 << 10
	udpByteSize     = 1472
	CRLF            = "\r\n"
	SessionTimeOut  = 10 * time.Second
	StoreDataExpire = 30 * time.Minute

	pUnkownProtocol ProtocolType = iota
	pTCP
	pUDP

	mUnkownMethod MethodType = iota
	mLogin
	mLogout
	mQuery
	mQuit

	rPositive ResultType = iota
	rNegative
	rBadRequest
)

var (
	MainStore   Store = nil
	Logger      *zap.Logger
	IDGenerator *katsubushi.Generator

	method = map[MethodType]string{
		mUnkownMethod: "NONE",
		mLogin:        "LOGIN",
		mLogout:       "LOGOUT",
		mQuery:        "QUERY",
		mQuit:         "QUIT",
	}

	result = map[ResultType]string{
		rPositive:   "+",
		rNegative:   "-",
		rBadRequest: "*",
	}

	methodFromString = map[string]MethodType{}
)

func init() {
	for i, v := range method {
		methodFromString[v] = i
	}
}
<|endoftext|>


<commit_before>package remember

import (
	"github.com/syndtr/goleveldb/leveldb"
)

var (
	Db     *db
	DbPath string
)

func init() {
	DbPath = "testdb"

	leveldbP, err := leveldb.OpenFile(DbPath, nil)
	if err != nil {
		panic(err)
	}

	Db = &db{
		leveldbP,
	}
}

type db struct {
	*leveldb.DB
}

func (d *db) Get(key []byte) (value []byte, err error) {
	value, err = d.DB.Get(key, nil)
	return
}

func (d *db) Put(key, value []byte) (err error) {
	err = d.DB.Put(key, value, nil)
	return
}

func (d *db) Delete(key []byte) (err error) {
	err = d.DB.Delete(key, nil)
	return
}
<commit_msg>add sync.Mutex to db<commit_after>package remember

import (
	"sync"

	"github.com/syndtr/goleveldb/leveldb"
)

var (
	Db     *db
	DbPath string
)

func init() {
	DbPath = "testdb"

	leveldbP, err := leveldb.OpenFile(DbPath, nil)
	if err != nil {
		panic(err)
	}

	Db = &db{
		DB: leveldbP,
	}
}

type db struct {
	*leveldb.DB
	sync.Mutex
}

func (d *db) Get(key []byte) (value []byte, err error) {
	value, err = d.DB.Get(key, nil)
	return
}

func (d *db) Put(key, value []byte) (err error) {
	err = d.DB.Put(key, value, nil)
	return
}

func (d *db) Delete(key []byte) (err error) {
	err = d.DB.Delete(key, nil)
	return
}
<|endoftext|>


<commit_before>// Package database is a library help for interact with database by model
package gomodel

import (
	"database/sql"

	"github.com/cosiner/gohper/bitset"
)

type (
	// Model represent a database model
	Model interface {
		Table() string
		// Vals store values of fields to given slice
		Vals(fields uint, vals []interface{})
		Ptrs(fields uint, ptrs []interface{})
	}

	// DB holds database connection, all typeinfos, and sql cache
	DB struct {
		// driver string
		*sql.DB
		types map[string]*TypeInfo
		*Cacher

		ModelCount int
	}
)

var (
	FieldCount = bitset.BitCountUint
)

// Open create a database manager and connect to database server
func Open(driver, dsn string, maxIdle, maxOpen int) (*DB, error) {
	db := NewDB()
	err := db.Connect(driver, dsn, maxIdle, maxOpen)

	return db, err
}

// New create a new db
func NewDB() *DB {
	return &DB{
		types:      make(map[string]*TypeInfo),
		ModelCount: 10,
	}
}

// Connect connect to database server
func (db *DB) Connect(driver, dsn string, maxIdle, maxOpen int) error {
	db_, err := sql.Open(driver, dsn)
	if err != nil {
		return err
	}

	db_.SetMaxIdleConns(maxIdle)
	db_.SetMaxOpenConns(maxOpen)
	db.DB = db_
	db.Cacher = NewCacher(Types, db)

	return nil
}

// registerType save type info of model
func (db *DB) registerType(v Model, table string) *TypeInfo {
	ti := parseTypeInfo(v, db)
	db.types[table] = ti

	return ti
}

// TypeInfo return type information of given model
// if type info not exist, it will parseTypeInfo it and save type info
func (db *DB) TypeInfo(v Model) *TypeInfo {
	table := v.Table()
	if ti, has := db.types[table]; has {
		return ti
	}

	return db.registerType(v, table)
}

func FieldVals(fields uint, v Model) []interface{} {
	args := make([]interface{}, FieldCount(fields))
	v.Vals(fields, args)

	return args
}

func FieldPtrs(fields uint, v Model) []interface{} {
	ptrs := make([]interface{}, FieldCount(fields))
	v.Ptrs(fields, ptrs)

	return ptrs
}

func (db *DB) Insert(v Model, fields uint, needId bool) (int64, error) {
	return db.ArgsInsert(v, fields, needId, FieldVals(fields, v))
}

func (db *DB) ArgsInsert(v Model, fields uint, needId bool, args ...interface{}) (int64, error) {
	stmt, err := db.TypeInfo(v).InsertStmt(fields)

	return StmtExec(stmt, err, needId, args...)
}

func (db *DB) Update(v Model, fields, whereFields uint) (int64, error) {
	c1, c2 := FieldCount(fields), FieldCount(whereFields)
	args := make([]interface{}, c1+c2)
	v.Vals(fields, args)
	v.Vals(whereFields, args[c1:])

	return db.ArgsUpdate(v, fields, whereFields, args)
}

func (db *DB) ArgsUpdate(v Model, fields, whereFields uint, args ...interface{}) (int64, error) {
	stmt, err := db.TypeInfo(v).UpdateStmt(fields, whereFields)

	return StmtExec(stmt, err, false, args...)
}

func (db *DB) Delete(v Model, whereFields uint) (int64, error) {
	return db.ArgsDelete(v, whereFields, FieldVals(whereFields, v))
}

func (db *DB) ArgsDelete(v Model, whereFields uint, args ...interface{}) (int64, error) {
	stmt, err := db.TypeInfo(v).DeleteStmt(whereFields)

	return StmtExec(stmt, err, false, args...)
}

// One select one row from database
func (db *DB) One(v Model, fields, whereFields uint) error {
	stmt, err := db.TypeInfo(v).SelectOneStmt(fields, whereFields)
	scanner, rows := StmtQuery(stmt, err, FieldVals(whereFields, v)...)

	return scanner.One(rows, FieldPtrs(fields, v)...)
}

func (db *DB) Limit(s Store, v Model, fields, whereFields uint, start, count int) error {
	c := FieldCount(whereFields)
	args := make([]interface{}, c+2)
	v.Vals(whereFields, args)
	args[c], args[c+1] = start, count

	return db.ArgsLimit(s, v, fields, whereFields, args)
}

func (db *DB) ArgsLimit(s Store, v Model, fields, whereFields uint, args ...interface{}) error {
	stmt, err := db.TypeInfo(v).SelectLimitStmt(fields, whereFields)
	scanner, rows := StmtQuery(stmt, err, args...)

	return scanner.Limit(rows, s, args[len(args)-1].(int))
}

func (db *DB) All(s Store, v Model, fields, whereFields uint) error {
	return db.ArgsAll(s, v, fields, whereFields, FieldVals(whereFields, v))
}

// ArgsAll select all rows, the last two argument must be "start" and "count"
func (db *DB) ArgsAll(s Store, v Model, fields, whereFields uint, args ...interface{}) error {
	stmt, err := db.TypeInfo(v).SelectAllStmt(fields, whereFields)
	scanner, rows := StmtQuery(stmt, err, args...)

	return scanner.All(rows, s, db.ModelCount)
}

// Count return count of rows for model, arguments was extracted from Model
func (db *DB) Count(v Model, whereFields uint) (count int64, err error) {
	return db.ArgsCount(v, whereFields, FieldVals(whereFields, v))
}

//Args Count return count of rows for model use custome arguments
func (db *DB) ArgsCount(v Model, whereFields uint,
	args ...interface{}) (count int64, err error) {
	ti := db.TypeInfo(v)

	stmt, err := ti.CountStmt(whereFields)
	scanner, rows := StmtQuery(stmt, err, args...)

	err = scanner.One(rows, &count)

	return
}

// ExecUpdate execute a update operation, return resolved result
func (db *DB) ExecUpdate(s string, needId bool, args ...interface{}) (ret int64, err error) {
	res, err := db.Exec(s, args...)

	return ResolveResult(res, err, needId)
}

// StmtExec execute stmt with given arguments and resolve the result if error is nil
func StmtExec(stmt *sql.Stmt, err error, needId bool, args ...interface{}) (int64, error) {
	if err != nil {
		return 0, err
	}

	res, err := stmt.Exec(args...)

	return ResolveResult(res, err, needId)
}

// StmtQuery execute the query stmt, error stored in Scanner
func StmtQuery(stmt *sql.Stmt, err error, args ...interface{}) (Scanner, *sql.Rows) {
	if err != nil {
		return Scanner{err}, nil
	}

	rows, err := stmt.Query(args...)
	if err != nil {
		return Scanner{err}, nil
	}

	return normalScanner, rows
}

// ResolveResult resolve sql result, if need id, return last insert id
// else return affected rows count
func ResolveResult(res sql.Result, err error, needId bool) (int64, error) {
	if err != nil {
		return 0, err
	}

	if needId {
		return res.LastInsertId()
	} else {
		return res.RowsAffected()
	}
}
<commit_msg>fix bug of variable arguments<commit_after>// Package database is a library help for interact with database by model
package gomodel

import (
	"database/sql"

	"github.com/cosiner/gohper/bitset"
)

type (
	// Model represent a database model
	Model interface {
		Table() string
		// Vals store values of fields to given slice
		Vals(fields uint, vals []interface{})
		Ptrs(fields uint, ptrs []interface{})
	}

	// DB holds database connection, all typeinfos, and sql cache
	DB struct {
		// driver string
		*sql.DB
		types map[string]*TypeInfo
		*Cacher

		ModelCount int
	}
)

var (
	FieldCount = bitset.BitCountUint
)

// Open create a database manager and connect to database server
func Open(driver, dsn string, maxIdle, maxOpen int) (*DB, error) {
	db := NewDB()
	err := db.Connect(driver, dsn, maxIdle, maxOpen)

	return db, err
}

// New create a new db
func NewDB() *DB {
	return &DB{
		types:      make(map[string]*TypeInfo),
		ModelCount: 10,
	}
}

// Connect connect to database server
func (db *DB) Connect(driver, dsn string, maxIdle, maxOpen int) error {
	db_, err := sql.Open(driver, dsn)
	if err != nil {
		return err
	}

	db_.SetMaxIdleConns(maxIdle)
	db_.SetMaxOpenConns(maxOpen)
	db.DB = db_
	db.Cacher = NewCacher(Types, db)

	return nil
}

// registerType save type info of model
func (db *DB) registerType(v Model, table string) *TypeInfo {
	ti := parseTypeInfo(v, db)
	db.types[table] = ti

	return ti
}

// TypeInfo return type information of given model
// if type info not exist, it will parseTypeInfo it and save type info
func (db *DB) TypeInfo(v Model) *TypeInfo {
	table := v.Table()
	if ti, has := db.types[table]; has {
		return ti
	}

	return db.registerType(v, table)
}

func FieldVals(fields uint, v Model) []interface{} {
	args := make([]interface{}, FieldCount(fields))
	v.Vals(fields, args)

	return args
}

func FieldPtrs(fields uint, v Model) []interface{} {
	ptrs := make([]interface{}, FieldCount(fields))
	v.Ptrs(fields, ptrs)

	return ptrs
}

func (db *DB) Insert(v Model, fields uint, needId bool) (int64, error) {
	return db.ArgsInsert(v, fields, needId, FieldVals(fields, v)...)
}

func (db *DB) ArgsInsert(v Model, fields uint, needId bool, args ...interface{}) (int64, error) {
	stmt, err := db.TypeInfo(v).InsertStmt(fields)

	return StmtExec(stmt, err, needId, args...)
}

func (db *DB) Update(v Model, fields, whereFields uint) (int64, error) {
	c1, c2 := FieldCount(fields), FieldCount(whereFields)
	args := make([]interface{}, c1+c2)
	v.Vals(fields, args)
	v.Vals(whereFields, args[c1:])

	return db.ArgsUpdate(v, fields, whereFields, args...)
}

func (db *DB) ArgsUpdate(v Model, fields, whereFields uint, args ...interface{}) (int64, error) {
	stmt, err := db.TypeInfo(v).UpdateStmt(fields, whereFields)

	return StmtExec(stmt, err, false, args...)
}

func (db *DB) Delete(v Model, whereFields uint) (int64, error) {
	return db.ArgsDelete(v, whereFields, FieldVals(whereFields, v)...)
}

func (db *DB) ArgsDelete(v Model, whereFields uint, args ...interface{}) (int64, error) {
	stmt, err := db.TypeInfo(v).DeleteStmt(whereFields)

	return StmtExec(stmt, err, false, args...)
}

// One select one row from database
func (db *DB) One(v Model, fields, whereFields uint) error {
	stmt, err := db.TypeInfo(v).SelectOneStmt(fields, whereFields)
	scanner, rows := StmtQuery(stmt, err, FieldVals(whereFields, v)...)

	return scanner.One(rows, FieldPtrs(fields, v)...)
}

func (db *DB) Limit(s Store, v Model, fields, whereFields uint, start, count int) error {
	c := FieldCount(whereFields)
	args := make([]interface{}, c+2)
	v.Vals(whereFields, args)
	args[c], args[c+1] = start, count

	return db.ArgsLimit(s, v, fields, whereFields, args...)
}

func (db *DB) ArgsLimit(s Store, v Model, fields, whereFields uint, args ...interface{}) error {
	stmt, err := db.TypeInfo(v).SelectLimitStmt(fields, whereFields)
	scanner, rows := StmtQuery(stmt, err, args...)

	return scanner.Limit(rows, s, args[len(args)-1].(int))
}

func (db *DB) All(s Store, v Model, fields, whereFields uint) error {
	return db.ArgsAll(s, v, fields, whereFields, FieldVals(whereFields, v)...)
}

// ArgsAll select all rows, the last two argument must be "start" and "count"
func (db *DB) ArgsAll(s Store, v Model, fields, whereFields uint, args ...interface{}) error {
	stmt, err := db.TypeInfo(v).SelectAllStmt(fields, whereFields)
	scanner, rows := StmtQuery(stmt, err, args...)

	return scanner.All(rows, s, db.ModelCount)
}

// Count return count of rows for model, arguments was extracted from Model
func (db *DB) Count(v Model, whereFields uint) (count int64, err error) {
	return db.ArgsCount(v, whereFields, FieldVals(whereFields, v)...)
}

//Args Count return count of rows for model use custome arguments
func (db *DB) ArgsCount(v Model, whereFields uint,
	args ...interface{}) (count int64, err error) {
	ti := db.TypeInfo(v)

	stmt, err := ti.CountStmt(whereFields)
	scanner, rows := StmtQuery(stmt, err, args...)

	err = scanner.One(rows, &count)

	return
}

// ExecUpdate execute a update operation, return resolved result
func (db *DB) ExecUpdate(s string, needId bool, args ...interface{}) (int64, error) {
	res, err := db.Exec(s, args...)

	return ResolveResult(res, err, needId)
}

// StmtExec execute stmt with given arguments and resolve the result if error is nil
func StmtExec(stmt *sql.Stmt, err error, needId bool, args ...interface{}) (int64, error) {
	if err != nil {
		return 0, err
	}

	res, err := stmt.Exec(args...)

	return ResolveResult(res, err, needId)
}

// StmtQuery execute the query stmt, error stored in Scanner
func StmtQuery(stmt *sql.Stmt, err error, args ...interface{}) (Scanner, *sql.Rows) {
	if err != nil {
		return Scanner{err}, nil
	}

	rows, err := stmt.Query(args...)
	if err != nil {
		return Scanner{err}, nil
	}

	return normalScanner, rows
}

// ResolveResult resolve sql result, if need id, return last insert id
// else return affected rows count
func ResolveResult(res sql.Result, err error, needId bool) (int64, error) {
	if err != nil {
		return 0, err
	}

	if needId {
		return res.LastInsertId()
	} else {
		return res.RowsAffected()
	}
}
<|endoftext|>


<commit_before><commit_msg>fix error message on secret/cm update failure<commit_after><|endoftext|>


<commit_before><commit_msg>no need to explicitly translate title, Fmt does it for us<commit_after><|endoftext|>


<commit_before>// Copyright 2018 DataArt. All rights reserved.
// Use of this source code is governed by an Apache-style
// license that can be found in the LICENSE file.

package transport

import (
	"github.com/devicehive/devicehive-go/internal/transport"
	"github.com/devicehive/devicehive-go/test/stubs"
	"github.com/matryer/is"
	"net/http"
	"testing"
	"time"
)

const testHTTPTimeout = 300 * time.Millisecond

func TestHTTPRequestId(t *testing.T) {
	httpTestSrv, addr := stubs.StartHTTPTestServer()
	defer httpTestSrv.Close()

	is := is.New(t)

	httpTestSrv.SetRequestHandler(func(reqData map[string]interface{}, rw http.ResponseWriter, r *http.Request) {
		is.True(reqData["requestId"] != "")
		rw.Write([]byte("{}"))
	})

	httpTsp, err := transport.Create(addr)

	is.NoErr(err)

	if err != nil {
		return
	}

	_, tspErr := httpTsp.Request("", nil, testHTTPTimeout)

	if tspErr != nil {
		t.Errorf("%s: %v", tspErr.Name(), tspErr)
	}
}

func TestHTTPTimeout(t *testing.T) {
	httpTestSrv, addr := stubs.StartHTTPTestServer()
	defer httpTestSrv.Close()

	is := is.New(t)

	httpTestSrv.SetRequestHandler(func(reqData map[string]interface{}, rw http.ResponseWriter, r *http.Request) {
		<-time.After(testWSTimeout + 1*time.Second)
		rw.Write([]byte("{\"result\": \"success\"}"))
	})

	httpTsp, err := transport.Create(addr)

	is.NoErr(err)

	res, tspErr := httpTsp.Request("", nil, testHTTPTimeout)

	is.True(res == nil)
	is.Equal(tspErr.Name(), transport.TimeoutErr)
}

func TestHTTPSubscription(t *testing.T) {
	httpTestSrv, addr := stubs.StartHTTPTestServer()
	defer httpTestSrv.Close()

	is := is.New(t)

	pollRequestsCount := 0
	const allowedPollRequestsCount = 3
	httpTestSrv.SetRequestHandler(func(reqData map[string]interface{}, rw http.ResponseWriter, r *http.Request) {
		if pollRequestsCount >= allowedPollRequestsCount {
			t.Error("HTTP transport must stop polling after unsubscribe")
			return
		}

		<-time.After(testWSTimeout + 1*time.Second)
		rw.Write([]byte(`[{"id": 1,"command": "command 1"},{"id": 2,"command": "command 2"}]`))
		pollRequestsCount++
	})

	httpTsp, err := transport.Create(addr)
	is.NoErr(err)

	tspChan, subscriptionId, tspErr := httpTsp.Subscribe("device/command/poll?deviceId=device-1", nil)
	if tspErr != nil {
		t.Fatalf("%s: %v", tspErr.Name(), tspErr)
	}

	is.True(subscriptionId != "")

	select {
	case data, ok := <-tspChan.DataChan:
		is.True(ok)
		is.True(data != nil)
	case <-time.After(2 * time.Second):
		t.Error("subscription event timeout")
	}

	select {
	case data, ok := <-tspChan.DataChan:
		is.True(ok)
		is.True(data != nil)
	case <-time.After(2 * time.Second):
		t.Error("subscription event timeout")
	}

	httpTsp.Unsubscribe(subscriptionId)
}
<commit_msg>Fix HTTP transport subscription test<commit_after>// Copyright 2018 DataArt. All rights reserved.
// Use of this source code is governed by an Apache-style
// license that can be found in the LICENSE file.

package transport

import (
	"github.com/devicehive/devicehive-go/internal/transport"
	"github.com/devicehive/devicehive-go/test/stubs"
	"github.com/matryer/is"
	"net/http"
	"testing"
	"time"
)

const testHTTPTimeout = 300 * time.Millisecond

func TestHTTPRequestId(t *testing.T) {
	httpTestSrv, addr := stubs.StartHTTPTestServer()
	defer httpTestSrv.Close()

	is := is.New(t)

	httpTestSrv.SetRequestHandler(func(reqData map[string]interface{}, rw http.ResponseWriter, r *http.Request) {
		is.True(reqData["requestId"] != "")
		rw.Write([]byte("{}"))
	})

	httpTsp, err := transport.Create(addr)

	is.NoErr(err)

	if err != nil {
		return
	}

	_, tspErr := httpTsp.Request("", nil, testHTTPTimeout)

	if tspErr != nil {
		t.Errorf("%s: %v", tspErr.Name(), tspErr)
	}
}

func TestHTTPTimeout(t *testing.T) {
	httpTestSrv, addr := stubs.StartHTTPTestServer()
	defer httpTestSrv.Close()

	is := is.New(t)

	httpTestSrv.SetRequestHandler(func(reqData map[string]interface{}, rw http.ResponseWriter, r *http.Request) {
		<-time.After(testHTTPTimeout + 1*time.Second)
		rw.Write([]byte("{\"result\": \"success\"}"))
	})

	httpTsp, err := transport.Create(addr)

	is.NoErr(err)

	res, tspErr := httpTsp.Request("", nil, testHTTPTimeout)

	is.True(res == nil)
	is.Equal(tspErr.Name(), transport.TimeoutErr)
}

func TestHTTPSubscription(t *testing.T) {
	httpTestSrv, addr := stubs.StartHTTPTestServer()
	defer httpTestSrv.Close()

	is := is.New(t)

	pollReqHandled := false
	httpTestSrv.SetRequestHandler(func(reqData map[string]interface{}, rw http.ResponseWriter, r *http.Request) {
		if pollReqHandled {
			t.Fatal("HTTP transport must stop polling after unsubscription")
		}

		rw.Write([]byte(`[{"id": 1,"command": "command 1"},{"id": 2,"command": "command 2"}]`))
		pollReqHandled = true
	})

	httpTsp, err := transport.Create(addr)
	is.NoErr(err)

	tspChan, subscriptionId, tspErr := httpTsp.Subscribe("device/command/poll?deviceId=device-1", nil)
	if tspErr != nil {
		t.Fatalf("%s: %v", tspErr.Name(), tspErr)
	}

	is.True(subscriptionId != "")

	select {
	case data, ok := <-tspChan.DataChan:
		is.True(ok)
		is.True(data != nil)
	case err := <- tspChan.ErrChan:
		t.Fatal(err)
	case <-time.After(testHTTPTimeout):
		t.Error("subscription event timeout")
	}

	httpTsp.Unsubscribe(subscriptionId)
}
<|endoftext|>


<commit_before>package dupefinder

import (
	"bufio"
	"crypto/sha256"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"runtime"
	"strings"
	"sync"
)

type fileHash struct {
	Hash     string
	Filename string
}

const header = `# This is a dupefinder catalog
#
# See https://github.com/rubenv/dupefinder for more info

`

// Catalog of hash to filename mappings
type DupeCatalog map[string]string

// Generate a catalog file based on a set of folders
func Generate(catalog string, folders ...string) error {
	err := validateFolders(folders...)
	if err != nil {
		return err
	}

	out, err := os.Create(catalog)
	if err != nil {
		return err
	}
	defer out.Close()

	_, err = out.WriteString(header)
	if err != nil {
		return err
	}

	errs := make(chan error)
	filenames := make(chan string, 100)
	entries := make(chan fileHash, 100)

	go walkAllFolders(errs, filenames, folders...)
	go hashFiles(errs, filenames, entries)

	for {
		entry, ok := <-entries
		if !ok {
			break
		}

		_, err := out.WriteString(fmt.Sprintf("%s %s\n", entry.Hash, entry.Filename))
		if err != nil {
			return err
		}
	}

	select {
	case err := <-errs:
		if err != nil {
			return err
		}
	default:
	}

	return nil
}

// Detect duplicates. Set echo to true to print duplicates, rm to delete them.
func Detect(catalog string, echo, rm bool, folders ...string) error {
	err := validateFolders(folders...)
	if err != nil {
		return err
	}

	catalogEntries, err := ParseCatalog(catalog)
	if err != nil {
		return err
	}

	errs := make(chan error)
	filenames := make(chan string, 100)
	entries := make(chan fileHash, 100)

	go walkAllFolders(errs, filenames, folders...)
	go hashFiles(errs, filenames, entries)

	deleted := int64(0)
	for {
		entry, ok := <-entries
		if !ok {
			break
		}

		if orig, ok := catalogEntries[entry.Hash]; ok {
			fi, err := os.Stat(entry.Filename)
			if err != nil {
				return err
			}

			deleted += fi.Size()

			if echo {
				fmt.Printf("Would delete %s (matches %s)\n", entry.Filename, orig)
			}

			if rm {
				fmt.Printf("Deleting %s (matches %s)\n", entry.Filename, orig)
				err := os.Remove(entry.Filename)
				if err != nil {
					return err
				}
			}
		}
	}

	fmt.Printf("Size saved: %d bytes\n", deleted)

	select {
	case err := <-errs:
		if err != nil {
			return err
		}
	default:
	}

	return nil
}

func validateFolders(folders ...string) error {
	for _, f := range folders {
		isfolder, err := isFolder(f)
		if err != nil {
			return err
		}
		if !isfolder {
			return fmt.Errorf("%s is not a folder", f)
		}
	}

	return nil
}

func isFolder(filename string) (bool, error) {
	f, err := os.Open(filename)
	if err != nil {
		return false, err
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return false, err
	}

	return fi.IsDir(), nil
}

func walkAllFolders(errs chan error, filenames chan string, folders ...string) {
	defer close(filenames)

	for _, f := range folders {
		err := walkFolder(f, filenames)
		if err != nil {
			errs <- err
			return
		}
	}
}

func walkFolder(filename string, out chan string) error {
	fi, err := ioutil.ReadDir(filename)
	if err != nil {
		return err
	}

	for _, child := range fi {
		fullname := path.Join(filename, child.Name())
		if child.IsDir() {
			err := walkFolder(fullname, out)
			if err != nil {
				return err
			}
		} else if child.Mode().IsRegular() {
			out <- fullname
		}
	}

	return nil
}

func hashFiles(errs chan error, filenames chan string, entries chan fileHash) {
	defer close(entries)

	var wg sync.WaitGroup

	wg.Add(runtime.NumCPU())
	for i := 0; i < runtime.NumCPU(); i++ {
		go func() {
			defer wg.Done()
			for {
				filename, ok := <-filenames
				if !ok {
					return
				}

				hash, err := hashFile(filename)
				if err != nil {
					errs <- err
					return
				}

				entries <- fileHash{
					Hash:     hash,
					Filename: filename,
				}
			}
		}()
	}

	wg.Wait()
}

func hashFile(filename string) (string, error) {
	file, err := os.Open(filename)
	if err != nil {
		return "", err
	}
	defer file.Close()

	hash := sha256.New()
	if _, err := io.Copy(hash, file); err != nil {
		return "", err
	}

	return fmt.Sprintf("%x", hash.Sum([]byte{})), nil
}

// Parse the catalog file at filename
func ParseCatalog(filename string) (DupeCatalog, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	return ParseCatalogReader(file)
}

// Parse a catalog file using an io.Reader
func ParseCatalogReader(reader io.Reader) (DupeCatalog, error) {
	result := DupeCatalog{}

	bufreader := bufio.NewReader(reader)

	done := false
	for !done {
		line, err := bufreader.ReadString('\n')
		if err == io.EOF {
			done = true
		} else if err != nil {
			return nil, err
		}

		line = strings.TrimSpace(line)
		if line == "" || line[0] == '#' {
			continue
		}

		parts := strings.SplitN(line, " ", 2)
		if len(parts) != 2 {
			return nil, fmt.Errorf("Malformed line: %#v", line)
		}

		result[parts[0]] = parts[1]
	}

	return result, nil
}
<commit_msg>Show number of files deleted<commit_after>package dupefinder

import (
	"bufio"
	"crypto/sha256"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"runtime"
	"strings"
	"sync"
)

type fileHash struct {
	Hash     string
	Filename string
}

const header = `# This is a dupefinder catalog
#
# See https://github.com/rubenv/dupefinder for more info

`

// Catalog of hash to filename mappings
type DupeCatalog map[string]string

// Generate a catalog file based on a set of folders
func Generate(catalog string, folders ...string) error {
	err := validateFolders(folders...)
	if err != nil {
		return err
	}

	out, err := os.Create(catalog)
	if err != nil {
		return err
	}
	defer out.Close()

	_, err = out.WriteString(header)
	if err != nil {
		return err
	}

	errs := make(chan error)
	filenames := make(chan string, 100)
	entries := make(chan fileHash, 100)

	go walkAllFolders(errs, filenames, folders...)
	go hashFiles(errs, filenames, entries)

	for {
		entry, ok := <-entries
		if !ok {
			break
		}

		_, err := out.WriteString(fmt.Sprintf("%s %s\n", entry.Hash, entry.Filename))
		if err != nil {
			return err
		}
	}

	select {
	case err := <-errs:
		if err != nil {
			return err
		}
	default:
	}

	return nil
}

// Detect duplicates. Set echo to true to print duplicates, rm to delete them.
func Detect(catalog string, echo, rm bool, folders ...string) error {
	err := validateFolders(folders...)
	if err != nil {
		return err
	}

	catalogEntries, err := ParseCatalog(catalog)
	if err != nil {
		return err
	}

	errs := make(chan error)
	filenames := make(chan string, 100)
	entries := make(chan fileHash, 100)

	go walkAllFolders(errs, filenames, folders...)
	go hashFiles(errs, filenames, entries)

	deleted := int64(0)
	deletedCount := int64(0)
	for {
		entry, ok := <-entries
		if !ok {
			break
		}

		if orig, ok := catalogEntries[entry.Hash]; ok {
			fi, err := os.Stat(entry.Filename)
			if err != nil {
				return err
			}

			deleted += fi.Size()
			deletedCount += 1

			if echo {
				fmt.Printf("Would delete %s (matches %s)\n", entry.Filename, orig)
			}

			if rm {
				fmt.Printf("Deleting %s (matches %s)\n", entry.Filename, orig)
				err := os.Remove(entry.Filename)
				if err != nil {
					return err
				}
			}
		}
	}

	fmt.Printf("Size saved: %d bytes (%d files)\n", deleted, deletedCount)

	select {
	case err := <-errs:
		if err != nil {
			return err
		}
	default:
	}

	return nil
}

func validateFolders(folders ...string) error {
	for _, f := range folders {
		isfolder, err := isFolder(f)
		if err != nil {
			return err
		}
		if !isfolder {
			return fmt.Errorf("%s is not a folder", f)
		}
	}

	return nil
}

func isFolder(filename string) (bool, error) {
	f, err := os.Open(filename)
	if err != nil {
		return false, err
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return false, err
	}

	return fi.IsDir(), nil
}

func walkAllFolders(errs chan error, filenames chan string, folders ...string) {
	defer close(filenames)

	for _, f := range folders {
		err := walkFolder(f, filenames)
		if err != nil {
			errs <- err
			return
		}
	}
}

func walkFolder(filename string, out chan string) error {
	fi, err := ioutil.ReadDir(filename)
	if err != nil {
		return err
	}

	for _, child := range fi {
		fullname := path.Join(filename, child.Name())
		if child.IsDir() {
			err := walkFolder(fullname, out)
			if err != nil {
				return err
			}
		} else if child.Mode().IsRegular() {
			out <- fullname
		}
	}

	return nil
}

func hashFiles(errs chan error, filenames chan string, entries chan fileHash) {
	defer close(entries)

	var wg sync.WaitGroup

	wg.Add(runtime.NumCPU())
	for i := 0; i < runtime.NumCPU(); i++ {
		go func() {
			defer wg.Done()
			for {
				filename, ok := <-filenames
				if !ok {
					return
				}

				hash, err := hashFile(filename)
				if err != nil {
					errs <- err
					return
				}

				entries <- fileHash{
					Hash:     hash,
					Filename: filename,
				}
			}
		}()
	}

	wg.Wait()
}

func hashFile(filename string) (string, error) {
	file, err := os.Open(filename)
	if err != nil {
		return "", err
	}
	defer file.Close()

	hash := sha256.New()
	if _, err := io.Copy(hash, file); err != nil {
		return "", err
	}

	return fmt.Sprintf("%x", hash.Sum([]byte{})), nil
}

// Parse the catalog file at filename
func ParseCatalog(filename string) (DupeCatalog, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	return ParseCatalogReader(file)
}

// Parse a catalog file using an io.Reader
func ParseCatalogReader(reader io.Reader) (DupeCatalog, error) {
	result := DupeCatalog{}

	bufreader := bufio.NewReader(reader)

	done := false
	for !done {
		line, err := bufreader.ReadString('\n')
		if err == io.EOF {
			done = true
		} else if err != nil {
			return nil, err
		}

		line = strings.TrimSpace(line)
		if line == "" || line[0] == '#' {
			continue
		}

		parts := strings.SplitN(line, " ", 2)
		if len(parts) != 2 {
			return nil, fmt.Errorf("Malformed line: %#v", line)
		}

		result[parts[0]] = parts[1]
	}

	return result, nil
}
<|endoftext|>


<commit_before>// Package fd provides a simple API to pass file descriptors
// between different OS processes.
//
// It can be useful if you want to inherit network connections
// from another process without closing them.
//
// Example scenario:
//
//   1) Running server receives a "let's upgrade" message
//   2) Server opens a Unix domain socket for the "upgrade"
//   3) Server starts a new copy of itself and passes Unix domain socket name
//   4) New copy starts reading for the socket
//   5) Server sends its state over the socket, also sending the number
//      of network connections to inherit, then it sends those connections
//      using fd.Put()
//   6) New copy reads the state and inherits connections using fd.Get(),
//      checks that everything is OK and sends the "OK" message to the socket
//   7) Server receives "OK" message and kills itself
package fd

import (
	"net"
	"os"
	"syscall"
)

// Get receives file descriptors over Unix domain socket.
//
// Num specifies the expected number of file descriptors in one message.
// Internal files' names to be assigned are specified via optional filenames
// argument.
//
// Use net.FileConn() if you're receiving a network connection. Don't
// forget to close the returned *os.File though.
func Get(via *net.UnixConn, num int, filenames []string) ([]*os.File, error) {
	if num < 1 {
		return nil, nil
	}

	// get the underlying socket
	viaf, err := via.File()
	if err != nil {
		return nil, err
	}
	socket := int(viaf.Fd())
	defer viaf.Close()

	// recvmsg
	buf := make([]byte, syscall.CmsgSpace(num*4))
	_, _, _, _, err = syscall.Recvmsg(socket, nil, buf, 0)
	if err != nil {
		return nil, err
	}

	// parse control msgs
	var msgs []syscall.SocketControlMessage
	msgs, err = syscall.ParseSocketControlMessage(buf)

	// convert fds to files
	res := make([]*os.File, 0, len(msgs))
	for i := 0; i < len(msgs) && err == nil; i++ {
		var fds []int
		fds, err = syscall.ParseUnixRights(&msgs[i])

		for fi, fd := range fds {
			var filename string
			if fi < len(filenames) {
				filename = filenames[fi]
			}

			res = append(res, os.NewFile(uintptr(fd), filename))
		}
	}

	return res, err
}

// Put file descriptors into Unix domain socket.
//
// Please note that the number of descriptors in one message is limited
// and is rather small.
// Use conn.File() to get a file if you want to put a network connection.
func Put(via *net.UnixConn, files ...*os.File) error {
	if len(files) == 0 {
		return nil
	}

	viaf, err := via.File()
	if err != nil {
		return err
	}
	socket := int(viaf.Fd())
	defer viaf.Close()

	fds := make([]int, len(files))
	for i := range files {
		fds[i] = int(files[i].Fd())
	}

	rights := syscall.UnixRights(fds...)
	return syscall.Sendmsg(socket, nil, rights, nil, 0)
}
<commit_msg>fix Get and Put descriptions<commit_after>// Package fd provides a simple API to pass file descriptors
// between different OS processes.
//
// It can be useful if you want to inherit network connections
// from another process without closing them.
//
// Example scenario:
//
//   1) Running server receives a "let's upgrade" message
//   2) Server opens a Unix domain socket for the "upgrade"
//   3) Server starts a new copy of itself and passes Unix domain socket name
//   4) New copy starts reading for the socket
//   5) Server sends its state over the socket, also sending the number
//      of network connections to inherit, then it sends those connections
//      using fd.Put()
//   6) New copy reads the state and inherits connections using fd.Get(),
//      checks that everything is OK and sends the "OK" message to the socket
//   7) Server receives "OK" message and kills itself
package fd

import (
	"net"
	"os"
	"syscall"
)

// Get receives file descriptors from a Unix domain socket.
//
// Num specifies the expected number of file descriptors in one message.
// Internal files' names to be assigned are specified via optional filenames
// argument.
//
// Use net.FileConn() if you're receiving a network connection. Don't
// forget to close the returned *os.File though.
func Get(via *net.UnixConn, num int, filenames []string) ([]*os.File, error) {
	if num < 1 {
		return nil, nil
	}

	// get the underlying socket
	viaf, err := via.File()
	if err != nil {
		return nil, err
	}
	socket := int(viaf.Fd())
	defer viaf.Close()

	// recvmsg
	buf := make([]byte, syscall.CmsgSpace(num*4))
	_, _, _, _, err = syscall.Recvmsg(socket, nil, buf, 0)
	if err != nil {
		return nil, err
	}

	// parse control msgs
	var msgs []syscall.SocketControlMessage
	msgs, err = syscall.ParseSocketControlMessage(buf)

	// convert fds to files
	res := make([]*os.File, 0, len(msgs))
	for i := 0; i < len(msgs) && err == nil; i++ {
		var fds []int
		fds, err = syscall.ParseUnixRights(&msgs[i])

		for fi, fd := range fds {
			var filename string
			if fi < len(filenames) {
				filename = filenames[fi]
			}

			res = append(res, os.NewFile(uintptr(fd), filename))
		}
	}

	return res, err
}

// Put sends file descriptors to Unix domain socket.
//
// Please note that the number of descriptors in one message is limited
// and is rather small.
// Use conn.File() to get a file if you want to put a network connection.
func Put(via *net.UnixConn, files ...*os.File) error {
	if len(files) == 0 {
		return nil
	}

	viaf, err := via.File()
	if err != nil {
		return err
	}
	socket := int(viaf.Fd())
	defer viaf.Close()

	fds := make([]int, len(files))
	for i := range files {
		fds[i] = int(files[i].Fd())
	}

	rights := syscall.UnixRights(fds...)
	return syscall.Sendmsg(socket, nil, rights, nil, 0)
}
<|endoftext|>


<commit_before>package kayvee

import (
	"encoding/json"
)

// Log Levels:
const Unknown = "unknown"
const Critical = "critical"
const Error = "error"
const Warning = "warning"
const Info = "info"
const Trace = "trace"

// Format converts a map to a string of space-delimited key=val pairs
func Format(data map[string]interface{}) string {
	formattedString, _ := json.Marshal(data)
	return string(formattedString)
}

// FormatLog is similar to Format, but takes additional reserved params to promote logging best-practices
func FormatLog(source string, level string, title string, data map[string]interface{}) string {
	if data == nil {
		data = make(map[string]interface{})
	}
	data["source"] = source
	data["level"] = level
	data["title"] = title
	return Format(data)
}
<commit_msg>use LogLevel type for log level constants<commit_after>package kayvee

import (
	"encoding/json"
)

// Log Levels:

type LogLevel string

const Unknown LogLevel = "unknown"
const Critical LogLevel = "critical"
const Error LogLevel = "error"
const Warning LogLevel = "warning"
const Info LogLevel = "info"
const Trace LogLevel = "trace"

// Format converts a map to a string of space-delimited key=val pairs
func Format(data map[string]interface{}) string {
	formattedString, _ := json.Marshal(data)
	return string(formattedString)
}

// FormatLog is similar to Format, but takes additional reserved params to promote logging best-practices
func FormatLog(source string, level LogLevel, title string, data map[string]interface{}) string {
	if data == nil {
		data = make(map[string]interface{})
	}
	data["source"] = source
	data["level"] = level
	data["title"] = title
	return Format(data)
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

type grtCmd struct {
	digest   string
	method   string
	protocol string
	domain   string
	endpoint string
	getVars  map[string]string
	body     string
}

func NewGrtCmd(method, endpoint string) *grtCmd {
	cmd := &grtCmd{
		method:   method,
		protocol: "http",
		domain:   "gerrit.dev.returnpath.net",
		endpoint: endpoint,
	}

	cmd.SetDigest()

	return cmd
}

func (this *grtCmd) GetDigest() string {
	var client http.Client

	resp, err := client.Get(this.protocol + "://" + this.domain + this.endpoint)
	if err != nil {
		log.Fatal(err)
	}
	req, err := http.NewRequest(this.method, this.protocol+"://"+this.domain+this.endpoint, nil)
	if err != nil {
		log.Fatal(err)
	}

	user := GetConfigValue("gerrit.user")
	pass := GetConfigValue("gerrit.pass")
	auth := GetAuthorization(user, pass, resp)

	return GetAuthString(auth, req.URL, req.Method, 1)
}

func (this *grtCmd) SetDigest() {
	this.digest = this.GetDigest()
}

func (this *grtCmd) Call() string {
	var client http.Client
	getQry := ""

	for k, v := range this.getVars {
		getQry += k + "=" + v + "&"
	}

	if len(getQry) > 0 {
		getQry = "?" + getQry[:len(getQry)-1]
	}

	resp, err := client.Get(this.protocol + "://" + this.domain + this.endpoint + getQry)
	if err != nil {
		log.Fatal(err)
	}

	req, err := http.NewRequest(this.method, this.protocol+"://"+this.domain+this.endpoint+getQry, nil)
	if err != nil {
		log.Fatal(err)
	}

	resp, err = client.Do(req)
	if err != nil {
		log.Fatal(err)
	}

	req.Header.Add("Authorization", this.digest)

	resp, err = client.Do(req)

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}

	val := string(body)
	val = val[strings.Index(string(val), "\n"):]

	return val
}
<commit_msg>Removing excess client.Do from Call()<commit_after>package main

import (
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

type grtCmd struct {
	digest   string
	method   string
	protocol string
	domain   string
	endpoint string
	getVars  map[string]string
	body     string
}

func NewGrtCmd(method, endpoint string) *grtCmd {
	cmd := &grtCmd{
		method:   method,
		protocol: "http",
		domain:   "gerrit.dev.returnpath.net",
		endpoint: endpoint,
	}

	cmd.SetDigest()

	return cmd
}

func (this *grtCmd) GetDigest() string {
	var client http.Client

	resp, err := client.Get(this.protocol + "://" + this.domain + this.endpoint)
	if err != nil {
		log.Fatal(err)
	}
	req, err := http.NewRequest(this.method, this.protocol+"://"+this.domain+this.endpoint, nil)
	if err != nil {
		log.Fatal(err)
	}

	user := GetConfigValue("gerrit.user")
	pass := GetConfigValue("gerrit.pass")
	auth := GetAuthorization(user, pass, resp)

	return GetAuthString(auth, req.URL, req.Method, 1)
}

func (this *grtCmd) SetDigest() {
	this.digest = this.GetDigest()
}

func (this *grtCmd) Call() string {
	var client http.Client
	getQry := ""

	for k, v := range this.getVars {
		getQry += k + "=" + v + "&"
	}

	if len(getQry) > 0 {
		getQry = "?" + getQry[:len(getQry)-1]
	}

	resp, err := client.Get(this.protocol + "://" + this.domain + this.endpoint + getQry)
	if err != nil {
		log.Fatal(err)
	}

	req, err := http.NewRequest(this.method, this.protocol+"://"+this.domain+this.endpoint+getQry, nil)
	if err != nil {
		log.Fatal(err)
	}

	req.Header.Add("Authorization", this.digest)

	resp, err = client.Do(req)
	if err != nil {
		log.Fatal(err)
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}

	val := string(body)
	val = val[strings.Index(string(val), "\n"):]

	return val
}
<|endoftext|>


<commit_before>//Command to test application without deploy:
//goapp serve app.yaml
//Command to deploy/update application:
//goapp deploy -application golangnode0 -version 0

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"html/template"
	"io/ioutil"
	"net/http"
	"strconv"
	"time"

	"appengine"
	"appengine/runtime"
	"appengine/urlfetch"
)

//predefined parameters
var maxNodes int = 10
var isAliveCheckPeriod int = 500 //in millisecs

//changeable parameters
var statusContent string = "Default status"
var statusLog string = "Log: "
var statusLog2 string = "Log2: "

//nodesStates := make(map[int]map[string]string)
/*
example for this map
var nodesStates map[int]map[string]string{
	1: map[string]string{
		"alive":"1",
		"hasTask":"true",
		"taskStatus":"completed",
		"taskResult":"some_result_for_node"
	},
}
*/

type webPage struct {
	Title string
}

type nodeStats struct {
	NodeID           int    `json:"ID"`
	NodeCount        int    `json:"nodeCount"`
	HasTask          bool   `json:"hasTask"`
	TaskStatus       string `json:"taskStatus"` //running-copleted-loaded
	TaskResult       string `json:"taskResult"`
	TaskFragmentBody string `json:"taskFragmentBody"`
	TaskBody         string `json:"taskBody"`
}

type echoMessage struct {
	Title   string `json:"title"`
	Content string `json:"content"`
}

//types for periodical functions
type pFunc func()
type pFuncInt func(int, *http.Request)

//wrong func for Google App Engine deployment. Need to use appengine libs...=(
func echo() {

	url := "http://golangappnode1.appspot.com/status"

	var jsonStr = []byte(`{"msg":"Hello!"}`)
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}

	defer resp.Body.Close()

	body, _ := ioutil.ReadAll(resp.Body)
	statusContent = string(body)

}

func startPage(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		templatePage, _ := template.ParseFiles("start.html")
		templatePage.Execute(w, &webPage{"simplePage"})
	case "POST":
		r.ParseForm()
		//go echo()
		//fmt.Fprintf(w, "Successful read command/input from web-interface! Input contains - "+r.FormValue("nodeId")+" "+r.FormValue("echoContent"))
	}
}

func statusServer(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		fmt.Fprintf(w, "Get status - "+statusContent)
	case "POST":
		buf := new(bytes.Buffer)
		buf.ReadFrom(r.Body)
		newStr := buf.String()

		/*inputMsg := echoMessage{}
		err2 := json.NewDecoder(r.Body).Decode(&inputMsg)
		if err2 != nil {
			panic(err2)
		}*/

		thisNodeStats := nodeStats{
			1,
			2,
			false,
			"not running",
			"empty",
			"empty fragment",
			"empty",
		}

		jsonNodeStats, err1 := json.Marshal(thisNodeStats)
		if err1 != nil {
			panic(err1)
		}

		fmt.Fprintf(w, "Get data by params in POST - OK "+string(jsonNodeStats))
		//statusContent = "POST request handled, " + "Node id: " + string(nodeSends.id) + ", Echo content: " + nodeSends.content
		statusContent = "POST request handled, " + newStr //+ "Input message object content: " + inputMsg.Title + inputMsg.Content
	}
}

//Functions for isAlive checking realization
func checkIsAlive(nodeId int, req *http.Request) {
	ctx := appengine.NewContext(req)
	client := http.Client{Transport: &urlfetch.Transport{Context: ctx}}

	nodeUrl := "http://goappnode" + strconv.Itoa(nodeId) + ".appspot.com/"

	resp, err := client.Get(nodeUrl)
	if err != nil {
		panic(err)
	}

	if resp.StatusCode != 200 {
		statusLog += "Node #" + strconv.Itoa(nodeId) + " - offline"
	} else {

		statusLog += "Node #" + strconv.Itoa(nodeId) + " - online"
	}
}

func checkIsAliveCtx(nodeId int, c appengine.Context) {
	ctx := c
	client := http.Client{Transport: &urlfetch.Transport{Context: ctx}}

	nodeUrl := "http://goappnode" + strconv.Itoa(nodeId) + ".appspot.com/"

	resp, err := client.Get(nodeUrl)
	if err != nil {
		panic(err)
	}

	if resp.StatusCode != 200 {
		statusLog += "Node #" + strconv.Itoa(nodeId) + " - offline"
	} else {

		statusLog += "Node #" + strconv.Itoa(nodeId) + " - online"
	}
}

func alivePeriodicTest(r *http.Request, done chan int) {
	for i := 0; i < 10; i++ {
		checkIsAlive(1, r)
	}
	done <- 0
}

func aliveTest(c appengine.Context) {
	for i := 0; i < 10; i++ {
		time.Sleep(150 * time.Millisecond)
		checkIsAliveCtx(1, c)
	}
}

func periodicTask(period time.Duration, task pFuncInt, taskArg int, taskReq *http.Request) {
	for {
		task(taskArg, taskReq)
		time.Sleep(period * time.Millisecond)
	}
}

func isAliveServer(w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, 1)
}

func logServer(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, statusLog + statusLog2)
}

func checkAliveStart(w http.ResponseWriter, r *http.Request) {
	//go periodicTask(200, checkIsAlive, 1, r)
	//checkIsAlive(1, r)
	//alivePeriodicTest(r)
	//ctx := appengine.NewContext(r)
	//runtime.RunInBackground(ctx, aliveTest)
	go func() {
		for i:=1; i<5; i++ {
			statusLog += " oLOLo"
		}

	}
	go func() {
		for i:=1; i<5; i++ {
			statusLogTwo += " oLOLo"
		}
	}
}

/*
func checkAliveStop(w http.ResponseWriter, r *http.Request) {

}
*/

func testEcho(w http.ResponseWriter, r *http.Request) {
	msg := echoMessage{
		"Message is",
		"",
	}

	r.ParseForm()
	c := appengine.NewContext(r)
	msg.Content = r.FormValue("echoContent")

	jsonMessage, err2 := json.Marshal(msg)
	if err2 != nil {
		panic(err2)
	}

	//jsonStr := []byte(`{"message":"` + r.FormValue("echoContent") + `"}`)
	jsonStr := []byte(jsonMessage)
	buf := bytes.NewBuffer(jsonStr)
	client := http.Client{Transport: &urlfetch.Transport{Context: c}}
	resp, err := client.Post("http://goappnode"+r.FormValue("nodeId")+".appspot.com/status", "application/octet-stream", buf)
	if err != nil {
		statusContent = err.Error()
		fmt.Println(err)
	}
	respBody, _ := ioutil.ReadAll(resp.Body)
	statusContent = "Response from node - " + string(respBody)
}

func init() {
	//view pages
	http.HandleFunc("/", startPage)
	//service pages
	http.HandleFunc("/echo", testEcho)
	http.HandleFunc("/status", statusServer)
	http.HandleFunc("/isalive", isAliveServer)
	http.HandleFunc("/startcheck", checkAliveStart)
	http.HandleFunc("/logs", logServer)

	//Wrong code for App Enine - server cant understand what it need to show
	//http.ListenAndServe(":80", nil)
}

//this func not needed for deploy on Google App Engine, init() func replace main()
/*
func main() {
	//fmt.Println("Hello, test server started on 8080 port.\n - /helloworld - show title page\n - /showinfo - show information about this thing")
	//http.ListenAndServe(":8080", nil)
	go sender()
}*/
<commit_msg>goroutines<commit_after>//Command to test application without deploy:
//goapp serve app.yaml
//Command to deploy/update application:
//goapp deploy -application golangnode0 -version 0

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"html/template"
	"io/ioutil"
	"net/http"
	"strconv"
	"time"

	"appengine"
	"appengine/urlfetch"
)

//predefined parameters
var maxNodes int = 10
var isAliveCheckPeriod int = 500 //in millisecs

//changeable parameters
var statusContent string = "Default status"
var statusLog string = "Log: "
var statusLog2 string = "Log2: "

//nodesStates := make(map[int]map[string]string)
/*
example for this map
var nodesStates map[int]map[string]string{
	1: map[string]string{
		"alive":"1",
		"hasTask":"true",
		"taskStatus":"completed",
		"taskResult":"some_result_for_node"
	},
}
*/

type webPage struct {
	Title string
}

type nodeStats struct {
	NodeID           int    `json:"ID"`
	NodeCount        int    `json:"nodeCount"`
	HasTask          bool   `json:"hasTask"`
	TaskStatus       string `json:"taskStatus"` //running-copleted-loaded
	TaskResult       string `json:"taskResult"`
	TaskFragmentBody string `json:"taskFragmentBody"`
	TaskBody         string `json:"taskBody"`
}

type echoMessage struct {
	Title   string `json:"title"`
	Content string `json:"content"`
}

//types for periodical functions
type pFunc func()
type pFuncInt func(int, *http.Request)

//wrong func for Google App Engine deployment. Need to use appengine libs...=(
func echo() {

	url := "http://golangappnode1.appspot.com/status"

	var jsonStr = []byte(`{"msg":"Hello!"}`)
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}

	defer resp.Body.Close()

	body, _ := ioutil.ReadAll(resp.Body)
	statusContent = string(body)

}

func startPage(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		templatePage, _ := template.ParseFiles("start.html")
		templatePage.Execute(w, &webPage{"simplePage"})
	case "POST":
		r.ParseForm()
		//go echo()
		//fmt.Fprintf(w, "Successful read command/input from web-interface! Input contains - "+r.FormValue("nodeId")+" "+r.FormValue("echoContent"))
	}
}

func statusServer(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		fmt.Fprintf(w, "Get status - "+statusContent)
	case "POST":
		buf := new(bytes.Buffer)
		buf.ReadFrom(r.Body)
		newStr := buf.String()

		/*inputMsg := echoMessage{}
		err2 := json.NewDecoder(r.Body).Decode(&inputMsg)
		if err2 != nil {
			panic(err2)
		}*/

		thisNodeStats := nodeStats{
			1,
			2,
			false,
			"not running",
			"empty",
			"empty fragment",
			"empty",
		}

		jsonNodeStats, err1 := json.Marshal(thisNodeStats)
		if err1 != nil {
			panic(err1)
		}

		fmt.Fprintf(w, "Get data by params in POST - OK "+string(jsonNodeStats))
		//statusContent = "POST request handled, " + "Node id: " + string(nodeSends.id) + ", Echo content: " + nodeSends.content
		statusContent = "POST request handled, " + newStr //+ "Input message object content: " + inputMsg.Title + inputMsg.Content
	}
}

//Functions for isAlive checking realization
func checkIsAlive(nodeId int, req *http.Request) {
	ctx := appengine.NewContext(req)
	client := http.Client{Transport: &urlfetch.Transport{Context: ctx}}

	nodeUrl := "http://goappnode" + strconv.Itoa(nodeId) + ".appspot.com/"

	resp, err := client.Get(nodeUrl)
	if err != nil {
		panic(err)
	}

	if resp.StatusCode != 200 {
		statusLog += "Node #" + strconv.Itoa(nodeId) + " - offline"
	} else {

		statusLog += "Node #" + strconv.Itoa(nodeId) + " - online"
	}
}

func checkIsAliveCtx(nodeId int, c appengine.Context) {
	ctx := c
	client := http.Client{Transport: &urlfetch.Transport{Context: ctx}}

	nodeUrl := "http://goappnode" + strconv.Itoa(nodeId) + ".appspot.com/"

	resp, err := client.Get(nodeUrl)
	if err != nil {
		panic(err)
	}

	if resp.StatusCode != 200 {
		statusLog += "Node #" + strconv.Itoa(nodeId) + " - offline"
	} else {

		statusLog += "Node #" + strconv.Itoa(nodeId) + " - online"
	}
}

func alivePeriodicTest(r *http.Request, done chan int) {
	for i := 0; i < 10; i++ {
		checkIsAlive(1, r)
	}
	done <- 0
}

func aliveTest(c appengine.Context) {
	for i := 0; i < 10; i++ {
		time.Sleep(150 * time.Millisecond)
		checkIsAliveCtx(1, c)
	}
}

func periodicTask(period time.Duration, task pFuncInt, taskArg int, taskReq *http.Request) {
	for {
		task(taskArg, taskReq)
		time.Sleep(period * time.Millisecond)
	}
}

func isAliveServer(w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, 1)
}

func logServer(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, statusLog+statusLog2)
}

func checkAliveStart(w http.ResponseWriter, r *http.Request) {
	//go periodicTask(200, checkIsAlive, 1, r)
	//checkIsAlive(1, r)
	//alivePeriodicTest(r)
	//ctx := appengine.NewContext(r)
	//runtime.RunInBackground(ctx, aliveTest)
	go func() {
		for i := 1; i < 5; i++ {
			statusLog += " oLOLo"
		}
	}()
	go func() {
		for i := 1; i < 5; i++ {
			statusLogTwo += " oLOLo"
		}
	}()
}

/*
func checkAliveStop(w http.ResponseWriter, r *http.Request) {

}
*/

func testEcho(w http.ResponseWriter, r *http.Request) {
	msg := echoMessage{
		"Message is",
		"",
	}

	r.ParseForm()
	c := appengine.NewContext(r)
	msg.Content = r.FormValue("echoContent")

	jsonMessage, err2 := json.Marshal(msg)
	if err2 != nil {
		panic(err2)
	}

	//jsonStr := []byte(`{"message":"` + r.FormValue("echoContent") + `"}`)
	jsonStr := []byte(jsonMessage)
	buf := bytes.NewBuffer(jsonStr)
	client := http.Client{Transport: &urlfetch.Transport{Context: c}}
	resp, err := client.Post("http://goappnode"+r.FormValue("nodeId")+".appspot.com/status", "application/octet-stream", buf)
	if err != nil {
		statusContent = err.Error()
		fmt.Println(err)
	}
	respBody, _ := ioutil.ReadAll(resp.Body)
	statusContent = "Response from node - " + string(respBody)
}

func init() {
	//view pages
	http.HandleFunc("/", startPage)
	//service pages
	http.HandleFunc("/echo", testEcho)
	http.HandleFunc("/status", statusServer)
	http.HandleFunc("/isalive", isAliveServer)
	http.HandleFunc("/startcheck", checkAliveStart)
	http.HandleFunc("/logs", logServer)

	//Wrong code for App Enine - server cant understand what it need to show
	//http.ListenAndServe(":80", nil)
}

//this func not needed for deploy on Google App Engine, init() func replace main()
/*
func main() {
	//fmt.Println("Hello, test server started on 8080 port.\n - /helloworld - show title page\n - /showinfo - show information about this thing")
	//http.ListenAndServe(":8080", nil)
	go sender()
}*/
<|endoftext|>