<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"runtime"
	"runtime/pprof"
	"sort"
	"strconv"
	"strings"
	"sync"
)

const (
	CmdDisplayMax = 32
	// max length of /proc/$PID/comm
	CommMax = 16
	// from ps_mem - average error due to truncation in the kernel
	// pss calculations
	PssAdjust    = .5
	pageSize     = 4096
	mapDetailLen = len("Size:                  4 kB")

	usage = `Usage: %s [OPTION...]
Simple, accurate RAM and swap reporting.

Options:
`
)

var (
	memProfile string
	cpuProfile string

	tyPss          = []byte("Pss:")
	tySwap         = []byte("Swap:")
	tyPrivateClean = []byte("Private_Clean:")
	tyPrivateDirty = []byte("Private_Dirty:")
)

// store info about a command (group of processes), similar to how
// ps_mem works.
type CmdMemInfo struct {
	PIDs    []int
	Name    string
	Pss     float64
	Shared  float64
	Swapped float64
}

type MapInfo struct {
	Inode uint64
	Name  string
}

// mapLine is a line from /proc/$PID/maps, or one of the same header
// lines from smaps.
func NewMapInfo(mapLine []byte) MapInfo {
	var mi MapInfo
	var err error
	pieces := splitSpaces(mapLine)
	if len(pieces) == 6 {
		mi.Name = string(pieces[5])
	}
	mi.Inode, err = ParseUint(pieces[4], 10, 64)
	if err != nil {
		panic(fmt.Sprintf("NewMapInfo: Atoi(%s): %s (%s)",
			string(pieces[4]), err, string(mapLine)))
	}
	return mi
}

func (mi MapInfo) IsAnon() bool {
	return mi.Inode == 0
}

// isDigit returns true if the rune d represents an ascii digit
// between 0 and 9, inclusive.
func isDigit(d uint8) bool {
	return d >= '0' && d <= '9'
}

// pidList returns a list of the process-IDs of every currently
// running process on the local system.
func pidList() ([]int, error) {
	procLs, err := ioutil.ReadDir("/proc")
	if err != nil {
		return nil, fmt.Errorf("ReadDir(/proc): %s", err)
	}

	pids := make([]int, 0, len(procLs))
	for _, pInfo := range procLs {
		if !isDigit(pInfo.Name()[0]) || !pInfo.IsDir() {
			continue
		}
		pidInt, err := strconv.Atoi(pInfo.Name())
		if err != nil {
			return nil, fmt.Errorf("Atoi(%s): %s", pInfo.Name(), err)
		}
		pids = append(pids, pidInt)
	}
	return pids, nil
}

// procName gets the process name for a worker.  It first checks the
// value of /proc/$PID/cmdline.  If setproctitle(3) has been called,
// it will use this.  Otherwise it uses the value of
// path.Base(/proc/$PID/exe), which has info on whether the executable
// has changed since the process was exec'ed.
func procName(pid int) (string, error) {
	p, err := os.Readlink(fmt.Sprintf("/proc/%d/exe", pid))
	// this would return an error if the PID doesn't
	// exist, or if the PID refers to a kernel thread.
	if err != nil {
		return "", nil
	}
	// cmdline is the null separated list of command line
	// arguments for the process, unless setproctitle(3)
	// has been called, in which case it is the
	argsB, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/cmdline", pid))
	if err != nil {
		return "", fmt.Errorf("ReadFile(%s): %s", fmt.Sprintf("/proc/%d/cmdline", pid), err)
	}
	args := strings.Split(string(argsB), "\000")
	n := args[0]

	nTrunc := n
	if len(n) > CommMax {
		nTrunc = n[:CommMax]
	}
	if strings.HasPrefix(p, nTrunc) {
		n = path.Base(p)
	}
	return n, nil
}

// splitSpaces returns a slice of byte slices which are the space
// delimited words from the original byte slice.  Unlike
// strings.Split($X, " "), runs of multiple spaces in a row are
// discarded.  NOTE WELL: this only checks for spaces (' '), other
// unicode whitespace isn't supported.
func splitSpaces(b []byte) [][]byte {
	// most lines in smaps have the form "Swap: 4 kB", so
	// preallocate the slice's array appropriately.
	res := make([][]byte, 0, 3)
	start := 0
	for i := 0; i < len(b)-1; i++ {
		if b[i] == ' ' {
			start = i + 1
		} else if b[i+1] == ' ' {
			res = append(res, b[start:i+1])
			start = i + 1
		}
	}
	if start != len(b) && b[start] != ' ' {
		res = append(res, b[start:])
	}
	return res
}

// procMem returns the amount of Pss, shared, and swapped out memory
// used.  The swapped out amount refers to anonymous pages only.
func procMem(pid int) (pss, shared, swap float64, err error) {
	fPath := fmt.Sprintf("/proc/%d/smaps", pid)
	f, err := os.Open(fPath)
	if err != nil {
		err = fmt.Errorf("os.Open(%s): %s", fPath, err)
		return
	}
	var priv float64
	//var curr MapInfo
	r := bufio.NewReaderSize(f, pageSize)
	for {
		var l []byte
		var isPrefix bool
		l, isPrefix, err = r.ReadLine()
		// this should never happen, so take the easy way out.
		if isPrefix {
			err = fmt.Errorf("ReadLine(%s): isPrefix", fPath)
		}
		if err != nil {
			// if we've got EOF, then we're simply done
			// processing smaps.
			if err == io.EOF {
				err = nil
				break
			}
			// otherwise error out
			err = fmt.Errorf("ReadLine(%s): %s", fPath, err)
			return
		}

		if len(l) != mapDetailLen {
			//curr = NewMapInfo(l)
			continue
		}
		pieces := splitSpaces(l)
		ty := pieces[0]
		var v uint64
		if bytes.Equal(ty, tyPss) {
			v, err = ParseUint(pieces[1], 10, 64)
			if err != nil {
				err = fmt.Errorf("Atoi(%s): %s", string(pieces[1]), err)
				return
			}
			pss += float64(v) + PssAdjust
		} else if bytes.Equal(ty, tyPrivateClean) || bytes.Equal(ty, tyPrivateDirty) {
			v, err = ParseUint(pieces[1], 10, 64)
			if err != nil {
				err = fmt.Errorf("Atoi(%s): %s", string(pieces[1]), err)
				return
			}
			priv += float64(v)
		} else if bytes.Equal(ty, tySwap) {
			v, err = ParseUint(pieces[1], 10, 64)
			if err != nil {
				err = fmt.Errorf("Atoi(%s): %s", string(pieces[1]), err)
				return
			}
			swap += float64(v)
		}
	}
	shared = pss - priv
	return
}

func worker(pidRequest chan int, wg *sync.WaitGroup, result chan *CmdMemInfo) {
	for pid := range pidRequest {
		var err error
		cmi := new(CmdMemInfo)

		cmi.PIDs = []int{pid}
		cmi.Name, err = procName(pid)
		if err != nil {
			log.Printf("procName(%d): %s", pid, err)
			wg.Done()
			continue
		} else if cmi.Name == "" {
			// XXX: This happens with kernel
			// threads. maybe warn? idk.
			wg.Done()
			continue
		}

		cmi.Pss, cmi.Shared, cmi.Swapped, err = procMem(pid)
		if err != nil {
			log.Printf("procMem(%d): %s", pid, err)
			wg.Done()
			continue
		}

		result <- cmi
		wg.Done()
	}
}

type byTotal []*CmdMemInfo

func (c byTotal) Len() int           { return len(c) }
func (c byTotal) Less(i, j int) bool { return c[i].Pss < c[j].Pss }
func (c byTotal) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usage, os.Args[0])
		flag.PrintDefaults()
	}

	flag.StringVar(&memProfile, "memprofile", "",
		"write memory profile to this file")
	flag.StringVar(&cpuProfile, "cpuprofile", "",
		"write cpu profile to this file")

	flag.Parse()
}

// startProfiling enables memory and/or CPU profiling if the
// appropriate command line flags have been set.
func startProfiling() {
	var err error
	// if we've passed in filenames to dump profiling data too,
	// start collecting profiling data.
	if memProfile != "" {
		runtime.MemProfileRate = 1
	}
	if cpuProfile != "" {
		var f *os.File
		if f, err = os.Create(cpuProfile); err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
	}
}

func stopProfiling() {
	if memProfile != "" {
		runtime.GC()
		f, err := os.Create(memProfile)
		if err != nil {
			log.Println(err)
		}
		pprof.WriteHeapProfile(f)
		f.Close()
	}
	if cpuProfile != "" {
		pprof.StopCPUProfile()
		cpuProfile = ""
	}
}

func main() {
	// if -memprof or -cpuprof haven't been set on the command
	// line, these are nops
	startProfiling()
	defer stopProfiling()

	// need to be root to read map info for other user's
	// processes.
	if os.Geteuid() != 0 {
		fmt.Printf("%s requires root privileges. (try 'sudo `which %s`)\n",
			os.Args[0], os.Args[0])
		return
	}

	pids, err := pidList()
	if err != nil {
		log.Printf("pidList: %s", err)
		return
	}

	var wg sync.WaitGroup
	work := make(chan int, len(pids))
	result := make(chan *CmdMemInfo, len(pids))

	// give us as much parallelism as possible
	nCPU := runtime.NumCPU()
	runtime.GOMAXPROCS(nCPU)
	for i := 0; i < nCPU; i++ {
		go worker(work, &wg, result)
	}

	wg.Add(len(pids))
	for _, pid := range pids {
		work <- pid
	}
	wg.Wait()

	// aggregate similar processes by command name.
	cmdMap := map[string]*CmdMemInfo{}
loop:
	for {
		// this only works correctly because we a channel
		// where the buffer size >= the number of potential
		// results.
		select {
		case c := <-result:
			n := c.Name
			if _, ok := cmdMap[n]; !ok {
				cmdMap[n] = c
				continue
			}
			cmdMap[n].PIDs = append(cmdMap[n].PIDs, c.PIDs...)
			cmdMap[n].Pss += c.Pss
			cmdMap[n].Shared += c.Shared
			cmdMap[n].Swapped += c.Swapped
		default:
			break loop
		}
	}

	// extract map values to a slice so we can sort them
	cmds := make([]*CmdMemInfo, 0, len(cmdMap))
	for _, c := range cmdMap {
		cmds = append(cmds, c)
	}
	sort.Sort(byTotal(cmds))

	// keep track of total RAM and swap usage
	var totPss, totSwap float64

	fmt.Printf("%10s%10s%10s\t%s\n", "MB RAM", "SHARED", "SWAPPED", "PROCESS (COUNT)")
	for _, c := range cmds {
		n := c.Name
		if len(n) > CmdDisplayMax {
			if n[0] == '[' {
				n = n[:strings.IndexRune(n, ']')+1]
			} else {
				n = n[:CmdDisplayMax]
			}
		}
		s := ""
		if c.Swapped > 0 {
			swap := float64(c.Swapped) / 1024.
			totSwap += swap
			s = fmt.Sprintf("%10.1f", swap)
		}
		pss := float64(c.Pss) / 1024.
		fmt.Printf("%10.1f%10.1f%10s\t%s (%d)\n", pss, float64(c.Shared)/1024., s, n, len(c.PIDs))
		totPss += pss
	}
	fmt.Printf("#%9.1f%20.1f\tTOTAL USED BY PROCESSES\n", totPss, totSwap)
}
<commit_msg>main: rename byTotal -> byPss<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"runtime"
	"runtime/pprof"
	"sort"
	"strconv"
	"strings"
	"sync"
)

const (
	CmdDisplayMax = 32
	// max length of /proc/$PID/comm
	CommMax = 16
	// from ps_mem - average error due to truncation in the kernel
	// pss calculations
	PssAdjust    = .5
	pageSize     = 4096
	mapDetailLen = len("Size:                  4 kB")

	usage = `Usage: %s [OPTION...]
Simple, accurate RAM and swap reporting.

Options:
`
)

var (
	memProfile string
	cpuProfile string

	tyPss          = []byte("Pss:")
	tySwap         = []byte("Swap:")
	tyPrivateClean = []byte("Private_Clean:")
	tyPrivateDirty = []byte("Private_Dirty:")
)

// store info about a command (group of processes), similar to how
// ps_mem works.
type CmdMemInfo struct {
	PIDs    []int
	Name    string
	Pss     float64
	Shared  float64
	Swapped float64
}

type MapInfo struct {
	Inode uint64
	Name  string
}

// mapLine is a line from /proc/$PID/maps, or one of the same header
// lines from smaps.
func NewMapInfo(mapLine []byte) MapInfo {
	var mi MapInfo
	var err error
	pieces := splitSpaces(mapLine)
	if len(pieces) == 6 {
		mi.Name = string(pieces[5])
	}
	mi.Inode, err = ParseUint(pieces[4], 10, 64)
	if err != nil {
		panic(fmt.Sprintf("NewMapInfo: Atoi(%s): %s (%s)",
			string(pieces[4]), err, string(mapLine)))
	}
	return mi
}

func (mi MapInfo) IsAnon() bool {
	return mi.Inode == 0
}

// isDigit returns true if the rune d represents an ascii digit
// between 0 and 9, inclusive.
func isDigit(d uint8) bool {
	return d >= '0' && d <= '9'
}

// pidList returns a list of the process-IDs of every currently
// running process on the local system.
func pidList() ([]int, error) {
	procLs, err := ioutil.ReadDir("/proc")
	if err != nil {
		return nil, fmt.Errorf("ReadDir(/proc): %s", err)
	}

	pids := make([]int, 0, len(procLs))
	for _, pInfo := range procLs {
		if !isDigit(pInfo.Name()[0]) || !pInfo.IsDir() {
			continue
		}
		pidInt, err := strconv.Atoi(pInfo.Name())
		if err != nil {
			return nil, fmt.Errorf("Atoi(%s): %s", pInfo.Name(), err)
		}
		pids = append(pids, pidInt)
	}
	return pids, nil
}

// procName gets the process name for a worker.  It first checks the
// value of /proc/$PID/cmdline.  If setproctitle(3) has been called,
// it will use this.  Otherwise it uses the value of
// path.Base(/proc/$PID/exe), which has info on whether the executable
// has changed since the process was exec'ed.
func procName(pid int) (string, error) {
	p, err := os.Readlink(fmt.Sprintf("/proc/%d/exe", pid))
	// this would return an error if the PID doesn't
	// exist, or if the PID refers to a kernel thread.
	if err != nil {
		return "", nil
	}
	// cmdline is the null separated list of command line
	// arguments for the process, unless setproctitle(3)
	// has been called, in which case it is the
	argsB, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/cmdline", pid))
	if err != nil {
		return "", fmt.Errorf("ReadFile(%s): %s", fmt.Sprintf("/proc/%d/cmdline", pid), err)
	}
	args := strings.Split(string(argsB), "\000")
	n := args[0]

	nTrunc := n
	if len(n) > CommMax {
		nTrunc = n[:CommMax]
	}
	if strings.HasPrefix(p, nTrunc) {
		n = path.Base(p)
	}
	return n, nil
}

// splitSpaces returns a slice of byte slices which are the space
// delimited words from the original byte slice.  Unlike
// strings.Split($X, " "), runs of multiple spaces in a row are
// discarded.  NOTE WELL: this only checks for spaces (' '), other
// unicode whitespace isn't supported.
func splitSpaces(b []byte) [][]byte {
	// most lines in smaps have the form "Swap: 4 kB", so
	// preallocate the slice's array appropriately.
	res := make([][]byte, 0, 3)
	start := 0
	for i := 0; i < len(b)-1; i++ {
		if b[i] == ' ' {
			start = i + 1
		} else if b[i+1] == ' ' {
			res = append(res, b[start:i+1])
			start = i + 1
		}
	}
	if start != len(b) && b[start] != ' ' {
		res = append(res, b[start:])
	}
	return res
}

// procMem returns the amount of Pss, shared, and swapped out memory
// used.  The swapped out amount refers to anonymous pages only.
func procMem(pid int) (pss, shared, swap float64, err error) {
	fPath := fmt.Sprintf("/proc/%d/smaps", pid)
	f, err := os.Open(fPath)
	if err != nil {
		err = fmt.Errorf("os.Open(%s): %s", fPath, err)
		return
	}
	var priv float64
	//var curr MapInfo
	r := bufio.NewReaderSize(f, pageSize)
	for {
		var l []byte
		var isPrefix bool
		l, isPrefix, err = r.ReadLine()
		// this should never happen, so take the easy way out.
		if isPrefix {
			err = fmt.Errorf("ReadLine(%s): isPrefix", fPath)
		}
		if err != nil {
			// if we've got EOF, then we're simply done
			// processing smaps.
			if err == io.EOF {
				err = nil
				break
			}
			// otherwise error out
			err = fmt.Errorf("ReadLine(%s): %s", fPath, err)
			return
		}

		if len(l) != mapDetailLen {
			//curr = NewMapInfo(l)
			continue
		}
		pieces := splitSpaces(l)
		ty := pieces[0]
		var v uint64
		if bytes.Equal(ty, tyPss) {
			v, err = ParseUint(pieces[1], 10, 64)
			if err != nil {
				err = fmt.Errorf("Atoi(%s): %s", string(pieces[1]), err)
				return
			}
			pss += float64(v) + PssAdjust
		} else if bytes.Equal(ty, tyPrivateClean) || bytes.Equal(ty, tyPrivateDirty) {
			v, err = ParseUint(pieces[1], 10, 64)
			if err != nil {
				err = fmt.Errorf("Atoi(%s): %s", string(pieces[1]), err)
				return
			}
			priv += float64(v)
		} else if bytes.Equal(ty, tySwap) {
			v, err = ParseUint(pieces[1], 10, 64)
			if err != nil {
				err = fmt.Errorf("Atoi(%s): %s", string(pieces[1]), err)
				return
			}
			swap += float64(v)
		}
	}
	shared = pss - priv
	return
}

func worker(pidRequest chan int, wg *sync.WaitGroup, result chan *CmdMemInfo) {
	for pid := range pidRequest {
		var err error
		cmi := new(CmdMemInfo)

		cmi.PIDs = []int{pid}
		cmi.Name, err = procName(pid)
		if err != nil {
			log.Printf("procName(%d): %s", pid, err)
			wg.Done()
			continue
		} else if cmi.Name == "" {
			// XXX: This happens with kernel
			// threads. maybe warn? idk.
			wg.Done()
			continue
		}

		cmi.Pss, cmi.Shared, cmi.Swapped, err = procMem(pid)
		if err != nil {
			log.Printf("procMem(%d): %s", pid, err)
			wg.Done()
			continue
		}

		result <- cmi
		wg.Done()
	}
}

type byPss []*CmdMemInfo

func (c byPss) Len() int           { return len(c) }
func (c byPss) Less(i, j int) bool { return c[i].Pss < c[j].Pss }
func (c byPss) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usage, os.Args[0])
		flag.PrintDefaults()
	}

	flag.StringVar(&memProfile, "memprofile", "",
		"write memory profile to this file")
	flag.StringVar(&cpuProfile, "cpuprofile", "",
		"write cpu profile to this file")

	flag.Parse()
}

// startProfiling enables memory and/or CPU profiling if the
// appropriate command line flags have been set.
func startProfiling() {
	var err error
	// if we've passed in filenames to dump profiling data too,
	// start collecting profiling data.
	if memProfile != "" {
		runtime.MemProfileRate = 1
	}
	if cpuProfile != "" {
		var f *os.File
		if f, err = os.Create(cpuProfile); err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
	}
}

func stopProfiling() {
	if memProfile != "" {
		runtime.GC()
		f, err := os.Create(memProfile)
		if err != nil {
			log.Println(err)
		}
		pprof.WriteHeapProfile(f)
		f.Close()
	}
	if cpuProfile != "" {
		pprof.StopCPUProfile()
		cpuProfile = ""
	}
}

func main() {
	// if -memprof or -cpuprof haven't been set on the command
	// line, these are nops
	startProfiling()
	defer stopProfiling()

	// need to be root to read map info for other user's
	// processes.
	if os.Geteuid() != 0 {
		fmt.Printf("%s requires root privileges. (try 'sudo `which %s`)\n",
			os.Args[0], os.Args[0])
		return
	}

	pids, err := pidList()
	if err != nil {
		log.Printf("pidList: %s", err)
		return
	}

	var wg sync.WaitGroup
	work := make(chan int, len(pids))
	result := make(chan *CmdMemInfo, len(pids))

	// give us as much parallelism as possible
	nCPU := runtime.NumCPU()
	runtime.GOMAXPROCS(nCPU)
	for i := 0; i < nCPU; i++ {
		go worker(work, &wg, result)
	}

	wg.Add(len(pids))
	for _, pid := range pids {
		work <- pid
	}
	wg.Wait()

	// aggregate similar processes by command name.
	cmdMap := map[string]*CmdMemInfo{}
loop:
	for {
		// this only works correctly because we a channel
		// where the buffer size >= the number of potential
		// results.
		select {
		case c := <-result:
			n := c.Name
			if _, ok := cmdMap[n]; !ok {
				cmdMap[n] = c
				continue
			}
			cmdMap[n].PIDs = append(cmdMap[n].PIDs, c.PIDs...)
			cmdMap[n].Pss += c.Pss
			cmdMap[n].Shared += c.Shared
			cmdMap[n].Swapped += c.Swapped
		default:
			break loop
		}
	}

	// extract map values to a slice so we can sort them
	cmds := make([]*CmdMemInfo, 0, len(cmdMap))
	for _, c := range cmdMap {
		cmds = append(cmds, c)
	}
	sort.Sort(byPss(cmds))

	// keep track of total RAM and swap usage
	var totPss, totSwap float64

	fmt.Printf("%10s%10s%10s\t%s\n", "MB RAM", "SHARED", "SWAPPED", "PROCESS (COUNT)")
	for _, c := range cmds {
		n := c.Name
		if len(n) > CmdDisplayMax {
			if n[0] == '[' {
				n = n[:strings.IndexRune(n, ']')+1]
			} else {
				n = n[:CmdDisplayMax]
			}
		}
		s := ""
		if c.Swapped > 0 {
			swap := float64(c.Swapped) / 1024.
			totSwap += swap
			s = fmt.Sprintf("%10.1f", swap)
		}
		pss := float64(c.Pss) / 1024.
		fmt.Printf("%10.1f%10.1f%10s\t%s (%d)\n", pss, float64(c.Shared)/1024., s, n, len(c.PIDs))
		totPss += pss
	}
	fmt.Printf("#%9.1f%20.1f\tTOTAL USED BY PROCESSES\n", totPss, totSwap)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"

	"github.com/dchest/uniuri"
	"github.com/ewhal/pygments"
	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

const (
	ADDRESS  = "http://localhost:9900"
	LENGTH   = 6
	PORT     = ":9900"
	USERNAME = ""
	PASS     = ""
	NAME     = ""
	DATABASE = USERNAME + ":" + PASS + "@/" + NAME + "?charset=utf8"
)

type Response struct {
	ID     string `json:"id"`
	HASH   string `json:"hash"`
	URL    string `json:"url"`
	SIZE   int    `json:"size"`
	DELKEY string `json:"delkey"`
}

type Page struct {
	Title string
	Body  []byte
	Raw   string
	Home  string
}

func check(err error) {
	if err != nil {
		fmt.Println(err)
	}
}

func generateName() string {
	s := uniuri.NewLen(LENGTH)
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	query, err := db.Query("select id from pastebin")
	for query.Next() {
		var id string
		err := query.Scan(&id)
		if err != nil {

		}
		if id == s {
			generateName()
		}
	}
	db.Close()

	return s

}
func hash(paste string) string {
	hasher := sha1.New()

	hasher.Write([]byte(paste))
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}

func save(raw string, lang string) []string {
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	sha := hash(raw)
	query, err := db.Query("select id, hash, data, delkey from pastebin")
	for query.Next() {
		var id, hash, paste, delkey string
		err := query.Scan(&id, &hash, &paste, &delkey)
		check(err)
		if hash == sha {
			url := ADDRESS + "/p/" + id
			return []string{id, hash, url, paste, delkey}
		}
	}
	id := generateName()
	var url string
	if lang == "" {
		url = ADDRESS + "/p/" + id
	} else {
		url = ADDRESS + "/p/" + id + "/" + lang
	}
	delKey := uniuri.NewLen(40)
	paste := html.EscapeString(raw)

	stmt, err := db.Prepare("INSERT INTO pastebin(id, hash, data, delkey) values(?,?,?,?)")
	check(err)
	_, err = stmt.Exec(id, sha, paste, delKey)
	check(err)
	db.Close()
	return []string{id, sha, url, paste, delKey}
}

func delHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	delkey := vars["delKey"]

	db, err := sql.Open("mysql", DATABASE)
	check(err)

	stmt, err := db.Prepare("delete from pastebin where delkey=?")
	check(err)

	res, err := stmt.Exec(html.EscapeString(delkey))
	check(err)

	_, err = res.RowsAffected()
	if err == sql.ErrNoRows {
		io.WriteString(w, "Error invalid paste")
	} else {
		io.WriteString(w, paste+" deleted")
	}
	db.Close()

}
func saveHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	output := vars["output"]
	switch r.Method {
	case "POST":
		paste := r.FormValue("p")
		lang := r.FormValue("lang")
		if paste == "" {
			http.Error(w, "Empty paste", 500)
			return
		}
		values := save(paste, lang)
		b := &Response{
			ID:     values[0],
			HASH:   values[1],
			URL:    values[2],
			SIZE:   len(values[3]),
			DELKEY: values[4],
		}

		switch output {
		case "json":

			w.Header().Set("Content-Type", "application/json")
			err := json.NewEncoder(w).Encode(b)

			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		case "xml":
			x, err := xml.MarshalIndent(b, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			w.Header().Set("Content-Type", "application/xml")
			w.Write(x)

		case "html":
			w.Header().Set("Content-Type", "text/html")
			io.WriteString(w, "
URL: "+b.URL+"
")
			io.WriteString(w, "
Delete Key: "+b.DELKEY+"
")

		case "redirect":
			http.Redirect(w, r, b.URL, 301)

		default:
			w.Header().Set("Content-Type", "plain/text")
			io.WriteString(w, b.URL+"\n")
			io.WriteString(w, "delete key: "+b.DELKEY+"\n")
		}
	}

}

func highlight(s string, lang string) (string, error) {

	highlight, err := pygments.Highlight(html.UnescapeString(s), html.EscapeString(lang), "html", "style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,noclasses=True,", "utf-8")
	if err != nil {
		return "", err
	}
	return highlight, nil

}

func getPaste(paste string, lang string) string {
	param1 := html.EscapeString(paste)
	db, err := sql.Open("mysql", DATABASE)
	var s string
	err = db.QueryRow("select data from pastebin where id=?", param1).Scan(&s)
	db.Close()
	check(err)

	if err == sql.ErrNoRows {
		return "Error invalid paste"
	} else {
		if lang == "" {
			return html.UnescapeString(s)
		} else {
			high, err := highlight(s, lang)
			check(err)
			return high

		}
	}

}

func pasteHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	lang := vars["lang"]
	s := getPaste(paste, lang)
	link := ADDRESS + "/raw/" + paste
	if lang == "" {
		p := &Page{
			Title: paste,
			Body:  []byte(s),
			Raw:   link,
			Home:  ADDRESS,
		}
		t, err := template.ParseFiles("assets/paste.html")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		t.Execute(w, p)

	} else {
		dat, err := ioutil.ReadFile("assets/syntax.html")
		check(err)
		fmt.Fprintf(w, string(dat), paste, paste, s, ADDRESS, link)

	}
}

func cloneHandler(w http.ResponseWriter, r *http.Request) {
}
func downloadHandler(w http.ResponseWriter, r *http.Request) {
}
func rawHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s := getPaste(paste, "")
	io.WriteString(w, s)

}

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/p/{pasteId}", pasteHandler)
	router.HandleFunc("/raw/{pasteId}", rawHandler)
	router.HandleFunc("/p/{pasteId}/{lang}", pasteHandler)
	router.HandleFunc("/{clone}/{pasteId}", cloneHandler)
	router.HandleFunc("/{download}/{pasteId}", downloadHandler)
	router.HandleFunc("/save", saveHandler)
	router.HandleFunc("/save/{output}", saveHandler)
	router.HandleFunc("/del/{pasteId}/{delKey}", delHandler)
	router.PathPrefix("/").Handler(http.StripPrefix("/", http.FileServer(http.Dir("assets/"))))
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<commit_msg>Cache templates<commit_after>package main

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"

	"github.com/dchest/uniuri"
	"github.com/ewhal/pygments"
	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

const (
	ADDRESS  = "http://localhost:9900"
	LENGTH   = 6
	PORT     = ":9900"
	USERNAME = ""
	PASS     = ""
	NAME     = ""
	DATABASE = USERNAME + ":" + PASS + "@/" + NAME + "?charset=utf8"
)

type Response struct {
	ID     string `json:"id"`
	HASH   string `json:"hash"`
	URL    string `json:"url"`
	SIZE   int    `json:"size"`
	DELKEY string `json:"delkey"`
}

type Page struct {
	Title string
	Body  []byte
	Raw   string
	Home  string
}

func check(err error) {
	if err != nil {
		fmt.Println(err)
	}
}

func generateName() string {
	s := uniuri.NewLen(LENGTH)
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	query, err := db.Query("select id from pastebin")
	for query.Next() {
		var id string
		err := query.Scan(&id)
		if err != nil {

		}
		if id == s {
			generateName()
		}
	}
	db.Close()

	return s

}
func hash(paste string) string {
	hasher := sha1.New()

	hasher.Write([]byte(paste))
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}

func save(raw string, lang string) []string {
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	sha := hash(raw)
	query, err := db.Query("select id, hash, data, delkey from pastebin")
	for query.Next() {
		var id, hash, paste, delkey string
		err := query.Scan(&id, &hash, &paste, &delkey)
		check(err)
		if hash == sha {
			url := ADDRESS + "/p/" + id
			return []string{id, hash, url, paste, delkey}
		}
	}
	id := generateName()
	var url string
	if lang == "" {
		url = ADDRESS + "/p/" + id
	} else {
		url = ADDRESS + "/p/" + id + "/" + lang
	}
	delKey := uniuri.NewLen(40)
	paste := html.EscapeString(raw)

	stmt, err := db.Prepare("INSERT INTO pastebin(id, hash, data, delkey) values(?,?,?,?)")
	check(err)
	_, err = stmt.Exec(id, sha, paste, delKey)
	check(err)
	db.Close()
	return []string{id, sha, url, paste, delKey}
}

func delHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	delkey := vars["delKey"]

	db, err := sql.Open("mysql", DATABASE)
	check(err)

	stmt, err := db.Prepare("delete from pastebin where delkey=?")
	check(err)

	res, err := stmt.Exec(html.EscapeString(delkey))
	check(err)

	_, err = res.RowsAffected()
	if err == sql.ErrNoRows {
		io.WriteString(w, "Error invalid paste")
	} else {
		io.WriteString(w, paste+" deleted")
	}
	db.Close()

}
func saveHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	output := vars["output"]
	switch r.Method {
	case "POST":
		paste := r.FormValue("p")
		lang := r.FormValue("lang")
		if paste == "" {
			http.Error(w, "Empty paste", 500)
			return
		}
		values := save(paste, lang)
		b := &Response{
			ID:     values[0],
			HASH:   values[1],
			URL:    values[2],
			SIZE:   len(values[3]),
			DELKEY: values[4],
		}

		switch output {
		case "json":

			w.Header().Set("Content-Type", "application/json")
			err := json.NewEncoder(w).Encode(b)

			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		case "xml":
			x, err := xml.MarshalIndent(b, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			w.Header().Set("Content-Type", "application/xml")
			w.Write(x)

		case "html":
			w.Header().Set("Content-Type", "text/html")
			io.WriteString(w, "
URL: "+b.URL+"
")
			io.WriteString(w, "
Delete Key: "+b.DELKEY+"
")

		case "redirect":
			http.Redirect(w, r, b.URL, 301)

		default:
			w.Header().Set("Content-Type", "plain/text")
			io.WriteString(w, b.URL+"\n")
			io.WriteString(w, "delete key: "+b.DELKEY+"\n")
		}
	}

}

func highlight(s string, lang string) (string, error) {

	highlight, err := pygments.Highlight(html.UnescapeString(s), html.EscapeString(lang), "html", "style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,noclasses=True,", "utf-8")
	if err != nil {
		return "", err
	}
	return highlight, nil

}

func getPaste(paste string, lang string) string {
	param1 := html.EscapeString(paste)
	db, err := sql.Open("mysql", DATABASE)
	var s string
	err = db.QueryRow("select data from pastebin where id=?", param1).Scan(&s)
	db.Close()
	check(err)

	if err == sql.ErrNoRows {
		return "Error invalid paste"
	} else {
		if lang == "" {
			return html.UnescapeString(s)
		} else {
			high, err := highlight(s, lang)
			check(err)
			return high

		}
	}

}

var templates = template.Must(template.ParseFiles("assets/paste.html"))
var syntax, _ = ioutil.ReadFile("assets/syntax.html")

func pasteHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	lang := vars["lang"]
	s := getPaste(paste, lang)
	link := ADDRESS + "/raw/" + paste
	if lang == "" {
		p := &Page{
			Title: paste,
			Body:  []byte(s),
			Raw:   link,
			Home:  ADDRESS,
		}
		err := templates.ExecuteTemplate(w, "assets/paste.html", p)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}

	} else {
		fmt.Fprintf(w, string(syntax), paste, paste, s, ADDRESS, link)

	}
}

func cloneHandler(w http.ResponseWriter, r *http.Request) {
}
func downloadHandler(w http.ResponseWriter, r *http.Request) {
}
func rawHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s := getPaste(paste, "")
	io.WriteString(w, s)

}

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/p/{pasteId}", pasteHandler)
	router.HandleFunc("/raw/{pasteId}", rawHandler)
	router.HandleFunc("/p/{pasteId}/{lang}", pasteHandler)
	router.HandleFunc("/{clone}/{pasteId}", cloneHandler)
	router.HandleFunc("/{download}/{pasteId}", downloadHandler)
	router.HandleFunc("/save", saveHandler)
	router.HandleFunc("/save/{output}", saveHandler)
	router.HandleFunc("/del/{pasteId}/{delKey}", delHandler)
	router.PathPrefix("/").Handler(http.StripPrefix("/", http.FileServer(http.Dir("assets/"))))
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"flag"

	"code.cloudfoundry.org/cflager"
	"github.com/cloudfoundry-community/splunk-firehose-nozzle/nozzle"
)

var (
	version string
	branch  string
	commit  string
	buildos string
)

func main() {
	logger, _ := cflager.New("splunk-nozzle-logger")
	logger.Info("Running splunk-firehose-nozzle")

	config := splunknozzle.NewConfigFromCmdFlags(version, branch, commit, buildos)

	splunkNozzle := splunknozzle.NewSplunkFirehoseNozzle(config)
	err := splunkNozzle.Run(logger)
	if err != nil {
		logger.Error("Failed to run splunk-firehose-nozzle", err)
	}
}
<commit_msg>Addback log info flag 	modified:   main.go<commit_after>package main

import (
	"flag"

	"code.cloudfoundry.org/cflager"
	"github.com/cloudfoundry-community/splunk-firehose-nozzle/nozzle"
)

var (
	version string
	branch  string
	commit  string
	buildos string
)

func main() {
	cflager.AddFlags(flag.CommandLine)

	logger, _ := cflager.New("splunk-nozzle-logger")
	logger.Info("Running splunk-firehose-nozzle")

	config := splunknozzle.NewConfigFromCmdFlags(version, branch, commit, buildos)

	splunkNozzle := splunknozzle.NewSplunkFirehoseNozzle(config)
	err := splunkNozzle.Run(logger)
	if err != nil {
		logger.Error("Failed to run splunk-firehose-nozzle", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"

	"github.com/LeSuisse/vault-gpg-plugin/gpg"
	"github.com/hashicorp/vault/helper/pluginutil"
	"github.com/hashicorp/vault/logical/plugin"
)

func main() {
	apiClientMeta := &pluginutil.APIClientMeta{}

	tlsConfig := apiClientMeta.GetTLSConfig()
	tlsProviderFunc := pluginutil.VaultPluginTLSProvider(tlsConfig)

	err := plugin.Serve(&plugin.ServeOpts{
		BackendFactoryFunc: gpg.Factory,
		TLSProviderFunc:    tlsProviderFunc,
	})
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}
}
<commit_msg>Parse command args otherwise TLS config options are not retrieved<commit_after>package main

import (
	"log"
	"os"

	"github.com/LeSuisse/vault-gpg-plugin/gpg"
	"github.com/hashicorp/vault/helper/pluginutil"
	"github.com/hashicorp/vault/logical/plugin"
)

func main() {
	apiClientMeta := &pluginutil.APIClientMeta{}
	flags := apiClientMeta.FlagSet()
	flags.Parse(os.Args)

	tlsConfig := apiClientMeta.GetTLSConfig()
	tlsProviderFunc := pluginutil.VaultPluginTLSProvider(tlsConfig)

	err := plugin.Serve(&plugin.ServeOpts{
		BackendFactoryFunc: gpg.Factory,
		TLSProviderFunc:    tlsProviderFunc,
	})
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"time"

	"github.com/Vivena/babelweb2/parser"
	"github.com/Vivena/babelweb2/ws"
)

type nodeslice []string

var nodes nodeslice
var staticRoot string

func (i *nodeslice) String() string {
	return fmt.Sprintf("%s", *i)
}

func (i *nodeslice) Set(value string) error {
	*i = append(*i, value)
	return nil
}

func connection(updates chan parser.SBabelUpdate, node string) {
	var conn net.Conn
	var err error

	for {
		log.Println("Trying", node)
		for {
			conn, err = net.Dial("tcp6", node)
			if err != nil {
				log.Println(err)
				time.Sleep(time.Second * 5)
			} else {
				break
			}
		}
		log.Println("Connected to", node)
		afterHours := func() {
			conn.Close()
			log.Printf("Connection to %v closed\n", node)
		}
		defer afterHours()
		fmt.Fprintf(conn, "monitor\n")
		r := bufio.NewReader(conn)
		s := parser.NewScanner(r)
		desc := parser.NewBabelDesc()
		err = desc.Fill(s)
		if err == io.EOF {
			log.Printf("Something wrong with %v:\n\tcouldn't get router id.\n", node)
		} else if err != nil {
			// Don't you even dare to reconnect to this unholy node!
			log.Printf("Oh, boy! %v is doomed:\n\t%v.\t", node, err)
			afterHours()
			return
		} else {
			ws.AddDesc(desc)
			err = desc.Listen(s, updates)
			if err != nil {
				log.Printf("Error while listening %v:\n\t%v.\n", node, err)
			}
			ws.RemoveDesc(desc.Id())
		}
		afterHours()
	}
}

func main() {
	var bwPort string

	flag.Var(&nodes, "node",
		"Babel node to connect to (default \"[::1]:33123\", "+
			"may be repeated)")
	flag.StringVar(&bwPort, "http", ":8080", "web server address")
	flag.StringVar(&staticRoot, "static", "./static/",
		"directory with static files")
	flag.Parse()
	if len(nodes) == 0 {
		nodes = nodeslice{"[::1]:33123"}
	}

	ws.Init()

	updates := make(chan parser.SBabelUpdate, 1024)
	defer close(updates)

	for i := 0; i < len(nodes); i++ {
		go connection(updates, nodes[i])
	}

	bcastGrp := ws.NewListenerGroup()
	handler := ws.Handler(bcastGrp)
	http.Handle("/", http.FileServer(http.Dir(staticRoot)))
	http.Handle("/ws", handler)
	go func() {
		log.Fatal(http.ListenAndServe(bwPort, nil))
	}()

	for {
		upd := <-updates
		bcastGrp.Iter(func(l *ws.Listener) {
			l.Channel <- upd
		})
	}
}
<commit_msg>Remove useless defer.<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"time"

	"github.com/Vivena/babelweb2/parser"
	"github.com/Vivena/babelweb2/ws"
)

type nodeslice []string

var nodes nodeslice
var staticRoot string

func (i *nodeslice) String() string {
	return fmt.Sprintf("%s", *i)
}

func (i *nodeslice) Set(value string) error {
	*i = append(*i, value)
	return nil
}

func connection(updates chan parser.SBabelUpdate, node string) {
	var conn net.Conn
	var err error

	for {
		log.Println("Trying", node)
		for {
			conn, err = net.Dial("tcp6", node)
			if err != nil {
				log.Println(err)
				time.Sleep(time.Second * 5)
			} else {
				break
			}
		}
		log.Println("Connected to", node)
		afterHours := func() {
			conn.Close()
			log.Printf("Connection to %v closed\n", node)
		}
		fmt.Fprintf(conn, "monitor\n")
		r := bufio.NewReader(conn)
		s := parser.NewScanner(r)
		desc := parser.NewBabelDesc()
		err = desc.Fill(s)
		if err == io.EOF {
			log.Printf("Something wrong with %v:\n\tcouldn't get router id.\n", node)
		} else if err != nil {
			// Don't you even dare to reconnect to this unholy node!
			log.Printf("Oh, boy! %v is doomed:\n\t%v.\t", node, err)
			afterHours()
			return
		} else {
			ws.AddDesc(desc)
			err = desc.Listen(s, updates)
			if err != nil {
				log.Printf("Error while listening %v:\n\t%v.\n", node, err)
			}
			ws.RemoveDesc(desc.Id())
		}
		afterHours()
	}
}

func main() {
	var bwPort string

	flag.Var(&nodes, "node",
		"Babel node to connect to (default \"[::1]:33123\", "+
			"may be repeated)")
	flag.StringVar(&bwPort, "http", ":8080", "web server address")
	flag.StringVar(&staticRoot, "static", "./static/",
		"directory with static files")
	flag.Parse()
	if len(nodes) == 0 {
		nodes = nodeslice{"[::1]:33123"}
	}

	ws.Init()

	updates := make(chan parser.SBabelUpdate, 1024)
	defer close(updates)

	for i := 0; i < len(nodes); i++ {
		go connection(updates, nodes[i])
	}

	bcastGrp := ws.NewListenerGroup()
	handler := ws.Handler(bcastGrp)
	http.Handle("/", http.FileServer(http.Dir(staticRoot)))
	http.Handle("/ws", handler)
	go func() {
		log.Fatal(http.ListenAndServe(bwPort, nil))
	}()

	for {
		upd := <-updates
		bcastGrp.Iter(func(l *ws.Listener) {
			l.Channel <- upd
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"strings"
)

func main() {
	flag.Parse()
	root := flag.Arg(0)
	target := flag.Arg(1)
	fmt.Println(fmt.Sprintf("smashing vendor @%s into %s", root, target))
	err := new(smasher).Run(root, target)
	fmt.Printf("errors returned %v\n", err)
}

type smasher struct {
	root   string
	target string
}

func (s *smasher) Run(root, target string) (err error) {
	s.root = root
	s.target = target
	if err = filepath.Walk(root, s.visit); err == nil {
		err = filepath.Walk(root, s.destroy)
	}
	return err
}

func (s *smasher) destroy(path string, f os.FileInfo, err error) error {
	if filepath.Base(path) == "vendor" {
		fmt.Printf("Hulk Crush: %s\n", path)
		err = os.RemoveAll(path)
	}
	return err
}

func (s *smasher) visit(path string, f os.FileInfo, err error) error {
	if filepath.Base(path) == "vendor" {
		fmt.Printf("Hulk Smash: %s\n", path)
		err = filepath.Walk(path, s.smash)
	}
	return err
}

func (s *smasher) smash(path string, f os.FileInfo, err error) error {
	p := strings.Split(path, "/vendor/")
	ref, _ := os.Stat(path)
	if !ref.IsDir() {
		err = Copy(s.target+p[len(p)-1], path)
	} else {
		os.RemoveAll(path)
		os.MkdirAll(path, 0777)
	}
	return err
}

func Copy(dst, src string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	out, err := SafeCreate(dst)
	if err != nil {
		return err
	}
	defer out.Close()
	_, err = io.Copy(out, in)
	cerr := out.Close()
	if err != nil {
		return err
	}
	return cerr
}

// SafeCreate creates a file, creating parent directories if needed
func SafeCreate(name ...string) (file *os.File, err error) {
	p, e := ensurePath(path.Join(name...))

	if e != nil {
		return nil, e
	}
	return os.Create(p)
}

func ensurePath(path string) (string, error) {
	base := filepath.Dir(path)
	e, _ := Exists(base)
	if e {
		return path, nil
	}

	// Create missing directory recursively
	err := os.MkdirAll(base, 0777)
	return path, err
}

//Exists - check if the given path exists
func Exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}
<commit_msg>safer path creation<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	pathlib "path"
	"path/filepath"
	"strings"
)

func main() {
	flag.Parse()
	root := flag.Arg(0)
	target := flag.Arg(1)
	fmt.Println(fmt.Sprintf("smashing vendor @%s into %s", root, target))
	err := new(smasher).Run(root, target)
	fmt.Printf("errors returned %v\n", err)
}

type smasher struct {
	root   string
	target string
}

func (s *smasher) Run(root, target string) (err error) {
	s.root = root
	s.target = target
	if err = filepath.Walk(root, s.visit); err == nil {
		err = filepath.Walk(root, s.destroy)
	}
	return err
}

func (s *smasher) destroy(path string, f os.FileInfo, err error) error {
	if filepath.Base(path) == "vendor" {
		fmt.Printf("Hulk Crush: %s\n", path)
		err = os.RemoveAll(path)
	}
	return err
}

func (s *smasher) visit(path string, f os.FileInfo, err error) error {
	if filepath.Base(path) == "vendor" {
		fmt.Printf("Hulk Smash: %s\n", path)
		err = filepath.Walk(path, s.smash)
	}
	return err
}

func (s *smasher) smash(path string, f os.FileInfo, err error) error {
	p := strings.Split(path, "/vendor/")
	ref, _ := os.Stat(path)
	if !ref.IsDir() {
		err = Copy(pathlib.Join(s.target, p[len(p)-1]), path)
	} else {
		os.RemoveAll(path)
		os.MkdirAll(path, 0777)
	}
	return err
}

func Copy(dst, src string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	out, err := SafeCreate(dst)
	if err != nil {
		return err
	}
	defer out.Close()
	_, err = io.Copy(out, in)
	cerr := out.Close()
	if err != nil {
		return err
	}
	return cerr
}

// SafeCreate creates a file, creating parent directories if needed
func SafeCreate(name ...string) (file *os.File, err error) {
	p, e := ensurePath(path.Join(name...))

	if e != nil {
		return nil, e
	}
	return os.Create(p)
}

func ensurePath(path string) (string, error) {
	base := filepath.Dir(path)
	e, _ := Exists(base)
	if e {
		return path, nil
	}

	// Create missing directory recursively
	err := os.MkdirAll(base, 0777)
	return path, err
}

//Exists - check if the given path exists
func Exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

var (
	// Version can be auto-set at build time using an ldflag
	//   go build -ldflags "-X main.Version `git describe --tags --always`"
	Version string

	// DepsFile specifies the Godeps file used by gdm
	DepsFile string = "Godeps"
)

const usage = `Go Dependency Manager (gdm), a lightweight tool for managing Go dependencies.

Usage:

    gdm [-f GODEPS_FILE] <command>

The commands are:

    restore   Check out revisions defined in Godeps file in GOPATH.
    save      Saves currently checked-out dependencies from GOPATH to Godeps file.
`

var ffile = flag.String("f", "Godeps", "Specify the name/location of Godeps file")

func main() {
	flag.Usage = usageExit
	flag.Parse()
	if *ffile != "" {
		DepsFile = *ffile
	}
	args := flag.Args()
	if len(args) < 1 {
		usageExit()
	}

	wd, err := os.Getwd()
	if err != nil {
		panic(err)
	}
	fmt.Println("======= Go Dependency Manager =======")
	fmt.Println("= working dir:", wd)
	gopath := getGoPath(wd)
	fmt.Println("= GOPATH:     ", gopath)
	fmt.Println("=====================================")

	switch args[0] {
	case "save", "bootstrap":
		save(wd, gopath)
	case "restore", "get", "sync", "checkout":
		restore(wd, gopath)
	}
}

func usageExit() {
	fmt.Println(usage)
	os.Exit(0)
}

func getGoPath(wd string) string {
	gopath := os.Getenv("GOPATH")
	if gopath == "" {
		fmt.Fprintf(os.Stderr, "GOPATH must be set to use gdm")
		os.Exit(1)
	}

	// Split out multiple GOPATHs if necessary
	if strings.Contains(gopath, ":") {
		paths := strings.Split(gopath, ":")
		for _, path := range paths {
			if strings.Contains(path, wd) {
				gopath = path
				break
			}
		}
	}

	if !strings.Contains(wd, gopath) {
		fmt.Fprintf(os.Stderr, "gdm can only be executed within a directory in the GOPATH")
		os.Exit(1)
	}
	return gopath
}

func save(wd, gopath string) {
	imports := ImportsFromPath(wd, gopath)

	f, err := os.Create(filepath.Join(wd, DepsFile))
	if err != nil {
		panic(err)
	}
	defer f.Close()
	w := bufio.NewWriter(f)
	for _, i := range imports {
		fmt.Printf("> Saving Import [%s] Revision [%s]\n", i.ImportPath, i.Rev)
		_, err = w.WriteString(fmt.Sprintf("%s %s\n", i.ImportPath, i.Rev))
		if err != nil {
			panic(err)
		}
	}
	w.Flush()
}

func restore(wd, gopath string) {
	imports := ImportsFromFile(filepath.Join(wd, DepsFile))
	for _, i := range imports {
		checkoutRevision(i, gopath)
	}
}

func checkoutRevision(i *Import, gopath string) {
	fullpath := filepath.Join(gopath, "src", i.ImportPath)
	cmdString := i.Repo.VCS.TagSyncCmd
	cmdString = strings.Replace(cmdString, "{tag}", i.Rev, 1)
	cmd := exec.Command(i.Repo.VCS.Cmd, strings.Split(cmdString, " ")...)
	cmd.Dir = fullpath
	fmt.Printf("> Executing [%s %s] in %s\n", i.Repo.VCS.Cmd, cmdString, fullpath)
	_, err := cmd.Output()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error checking out revision at %s, %s\n",
			fullpath, err.Error())
		os.Exit(1)
	}
}
<commit_msg>usage tweek<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

var (
	// Version can be auto-set at build time using an ldflag
	//   go build -ldflags "-X main.Version `git describe --tags --always`"
	Version string

	// DepsFile specifies the Godeps file used by gdm
	DepsFile string = "Godeps"
)

const usage = `Go Dependency Manager (gdm), a lightweight tool for managing Go dependencies.

Usage:

    gdm [-f GODEPS_FILE] <command>

The commands are:

    restore   Check out revisions defined in Godeps file to $GOPATH.
    save      Saves currently checked-out dependencies from $GOPATH to Godeps file.
`

var ffile = flag.String("f", "Godeps", "Specify the name/location of Godeps file")

func main() {
	flag.Usage = usageExit
	flag.Parse()
	if *ffile != "" {
		DepsFile = *ffile
	}
	args := flag.Args()
	if len(args) < 1 {
		usageExit()
	}

	wd, err := os.Getwd()
	if err != nil {
		panic(err)
	}
	fmt.Println("======= Go Dependency Manager =======")
	fmt.Println("= working dir:", wd)
	gopath := getGoPath(wd)
	fmt.Println("= GOPATH:     ", gopath)
	fmt.Println("=====================================")

	switch args[0] {
	case "save", "bootstrap":
		save(wd, gopath)
	case "restore", "get", "sync", "checkout":
		restore(wd, gopath)
	}
}

func usageExit() {
	fmt.Println(usage)
	os.Exit(0)
}

func getGoPath(wd string) string {
	gopath := os.Getenv("GOPATH")
	if gopath == "" {
		fmt.Fprintf(os.Stderr, "GOPATH must be set to use gdm")
		os.Exit(1)
	}

	// Split out multiple GOPATHs if necessary
	if strings.Contains(gopath, ":") {
		paths := strings.Split(gopath, ":")
		for _, path := range paths {
			if strings.Contains(path, wd) {
				gopath = path
				break
			}
		}
	}

	if !strings.Contains(wd, gopath) {
		fmt.Fprintf(os.Stderr, "gdm can only be executed within a directory in the GOPATH")
		os.Exit(1)
	}
	return gopath
}

func save(wd, gopath string) {
	imports := ImportsFromPath(wd, gopath)

	f, err := os.Create(filepath.Join(wd, DepsFile))
	if err != nil {
		panic(err)
	}
	defer f.Close()
	w := bufio.NewWriter(f)
	for _, i := range imports {
		fmt.Printf("> Saving Import [%s] Revision [%s]\n", i.ImportPath, i.Rev)
		_, err = w.WriteString(fmt.Sprintf("%s %s\n", i.ImportPath, i.Rev))
		if err != nil {
			panic(err)
		}
	}
	w.Flush()
}

func restore(wd, gopath string) {
	imports := ImportsFromFile(filepath.Join(wd, DepsFile))
	for _, i := range imports {
		checkoutRevision(i, gopath)
	}
}

func checkoutRevision(i *Import, gopath string) {
	fullpath := filepath.Join(gopath, "src", i.ImportPath)
	cmdString := i.Repo.VCS.TagSyncCmd
	cmdString = strings.Replace(cmdString, "{tag}", i.Rev, 1)
	cmd := exec.Command(i.Repo.VCS.Cmd, strings.Split(cmdString, " ")...)
	cmd.Dir = fullpath
	fmt.Printf("> Executing [%s %s] in %s\n", i.Repo.VCS.Cmd, cmdString, fullpath)
	_, err := cmd.Output()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error checking out revision at %s, %s\n",
			fullpath, err.Error())
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"
	"time"
)

func main() {
	log.Print("Jolie is ALIVE")

	InfluxTest()

	var err error
	db, err = NewMongoSession()
	if err != nil {
		log.Fatalf("Failed to connect storage: %s", err.Error())
	}

	go http.ListenAndServe(":8080", Api())

	consumer, err := connectConsumer()
	if err != nil {
		log.Fatalf("Failed to connect consumer: %s", err.Error())
	}

	queues, err := consumer.Consume()
	for {
		select {
		case gatewayStatus := <-queues.GatewayStatuses:
			log.Printf("Gateway status: %#v", gatewayStatus)
		case rxPacket := <-queues.RxPackets:
			log.Printf("RX packet: %#v", rxPacket)
		}
	}
}

func connectConsumer() (Consumer, error) {
	consumer, err := ConnectRabbitConsumer()
	if err != nil {
		return nil, err
	}

	err = consumer.Configure()
	if err != nil {
		return nil, err
	}

	return consumer, nil
}
<commit_msg>Fixed unused dependency<commit_after>package main

import (
	"log"
	"net/http"
)

func main() {
	log.Print("Jolie is ALIVE")

	InfluxTest()

	var err error
	db, err = NewMongoSession()
	if err != nil {
		log.Fatalf("Failed to connect storage: %s", err.Error())
	}

	go http.ListenAndServe(":8080", Api())

	consumer, err := connectConsumer()
	if err != nil {
		log.Fatalf("Failed to connect consumer: %s", err.Error())
	}

	queues, err := consumer.Consume()
	for {
		select {
		case gatewayStatus := <-queues.GatewayStatuses:
			log.Printf("Gateway status: %#v", gatewayStatus)
		case rxPacket := <-queues.RxPackets:
			log.Printf("RX packet: %#v", rxPacket)
		}
	}
}

func connectConsumer() (Consumer, error) {
	consumer, err := ConnectRabbitConsumer()
	if err != nil {
		return nil, err
	}

	err = consumer.Configure()
	if err != nil {
		return nil, err
	}

	return consumer, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Brian Lachniet. All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

/*
Gomir mirrors git repositories

TODO:
	- Add support for controlling concurrency during fetch/push
	- Add documentation describing where repos are stored
	- Add documentation describing what to do if something goes wrong during a fetch/push
	- Add documentation for a normal synchronization workflow
	- Add usage (gomir -h)
*/
package main

import (
	"fmt"
	"io"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"

	"github.com/fatih/color"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// Set via ldflags
var version string
var gitCommit string
var buildDate string

func main() {
	rootCmd := &cobra.Command{
		Use:  "gomir",
		Long: `Mirror Git repositories between two disconnected networks`,
	}

	addCmd := &cobra.Command{
		Use:   "add <fetchURL> <pushURL> [<localDest>]",
		Short: "Add a repository to mirror",
		Args:  cobra.RangeArgs(2, 3),
		Run: func(cmd *cobra.Command, args []string) {
			switch len(args) {
			case 2:
				add(args[0], args[1], "")
			case 3:
				add(args[0], args[1], args[2])
			default:
				fmt.Println("Wrong number of arguments")
				os.Exit(1)
			}
		},
	}

	fetchCmd := &cobra.Command{
		Use:   "fetch",
		Short: "Fetch changes for all mirroed repositories",
		Args:  cobra.NoArgs,
		Run: func(cmd *cobra.Command, args []string) {
			fetch()
		},
	}

	pushCmd := &cobra.Command{
		Use:   "push",
		Short: "Push changes for all mirrored repositories",
		Args:  cobra.NoArgs,
		Run: func(cmd *cobra.Command, args []string) {
			push()
		},
	}

	versionCmd := &cobra.Command{
		Use:   "version",
		Short: "Show gomir version information",
		Args:  cobra.NoArgs,
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Printf("  Version:    %v\n", version)
			fmt.Printf("  Git commit: %v\n", gitCommit)
			fmt.Printf("  Built:      %v\n", buildDate)
		},
	}

	rootCmd.AddCommand(addCmd, fetchCmd, pushCmd, versionCmd)
	rootCmd.Execute()
}

func add(fetchURL, pushURL, localDest string) {
	// Try to generate a localDest
	if localDest == "" {
		u, err := url.Parse(fetchURL)
		if err != nil {
			fmt.Println("Could not generate a localDest")
			os.Exit(1)
		}
		localDest = fmt.Sprintf("%v%v", u.Host, u.Path)
	}

	localDest = ensureGitExt(localDest)

	// Clone
	if err := gitCloneMirror(fetchURL, localDest); err != nil {
		fmt.Println("Error cloning repository")
		os.Exit(1)
	}

	// Set Push URL
	if err := gitSetOriginPushURL(localDest, pushURL); err != nil {
		fmt.Println("Error setting push URL")
		os.Exit(1)
	}
}

func fetch() {
	errCount := performOperationAsync(fetchSingle)
	if errCount > 0 {
		color.Red("Fetch failed for %v repos", errCount)
		os.Exit(1)
	}
}

func fetchSingle(gitDir string) bool {
	logFile, logger, err := getLog(gitDir, "FETCH: ")
	if err != nil {
		color.Red("Error opening log file for %v", gitDir)
		return false
	}
	defer logFile.Close()

	logger.Println("Start")
	success := gitFetchPrune(gitDir, logFile) == nil
	logger.Printf("Done, success:%v", success)
	return success
}

func push() {
	errCount := performOperationAsync(pushSingle)
	if errCount > 0 {
		color.Red("Push failed for %v repos", errCount)
		os.Exit(1)
	}
}

func pushSingle(gitDir string) bool {
	logFile, logger, err := getLog(gitDir, "PUSH: ")
	if err != nil {
		color.Red("Error opening log file for %v", gitDir)
		return false
	}
	defer logFile.Close()

	logger.Println("Start")

	// Where are we pushing to?
	pushURL, err := gitGetOriginPushURL(gitDir)
	if err != nil {
		logger.Printf("Error retrieving push URL: %+v", err)
		return false
	}

	// If pushing using file protocol and destination repository does
	// not alread exist, initialize it
	isFileProtocol := pushURL.Scheme == "" || strings.ToLower(pushURL.Scheme) == "file"
	if isFileProtocol {
		_, err := os.Stat(pushURL.Path)
		if err != nil && os.IsNotExist(err) {
			if err := gitInitBareRepo(pushURL.Path, logFile); err != nil {
				logger.Printf("Error initializing bare git repository: %+v", err)
				return false
			}
		}
	}

	// Push
	if err := gitPushMirror(gitDir, logFile); err != nil {
		logger.Printf("Error pushing: %+v", err)
		return false
	}

	// Update server info
	if isFileProtocol {
		if err := gitUpdateServerInfo(pushURL.Path, logFile); err != nil {
			logger.Printf("Error updating server info: %+v", err)
			return false
		}
	}

	logger.Println("Done")
	return true
}

func findGitDirs() []string {
	gitDirs := []string{}
	filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
		if err == nil &&
			info.IsDir() &&
			strings.ToLower(filepath.Base(path)) != ".git" &&
			strings.ToLower(filepath.Ext(path)) == ".git" {
			gitDirs = append(gitDirs, path)
			return filepath.SkipDir
		}
		return nil
	})
	return gitDirs
}

func ensureGitExt(str string) string {
	if strings.ToLower(filepath.Ext(str)) != ".git" {
		return fmt.Sprintf("%v.git", str)
	}
	return str
}

func getLog(gitDir, prefix string) (io.WriteCloser, *log.Logger, error) {
	logPath := fmt.Sprintf("%v.log", gitDir)
	logFile, err := os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return nil, nil, errors.Wrap(err, "Error opening log file")
	}

	logger := log.New(logFile, prefix, log.Ldate|log.Ltime|log.Lshortfile|log.LUTC)
	return logFile, logger, nil
}

type gitDirOperation func(gitDir string) bool

func performOperationAsync(op gitDirOperation) int64 {
	var wg sync.WaitGroup
	var errCount int64
	for _, gitDir := range findGitDirs() {
		wg.Add(1)
		go func(gitDir string) {
			defer wg.Done()

			if op(gitDir) {
				color.Green("[✔] %v", gitDir)
			} else {
				atomic.AddInt64(&errCount, 1)
				color.Red("[X] %v", gitDir)
			}
		}(gitDir)
	}

	wg.Wait()
	return errCount
}
<commit_msg>Update documentation<commit_after>// Copyright 2017 Brian Lachniet. All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

/*
Gomir mirrors git repositories

Gomir is built to mirror Git repositories between two disconnected networks. The
'source' network is the network that hosts the repositories that you want to mirror.
The 'destination' network is the network that you want to host your mirrored version
of the repositories.

Generally, you will make use of the 'add' and 'fetch' commands on the source network.
When connected to the destination network, you will make use of the 'push' command.

Below is an example of a normal workflow. We're mirroring two repositories from GitHub
to an SMB file share. The server name is 'gitserver' and the share is named 'repos'.

	# Connect to the source network
	gomir add https://github.com/blachniet/dotfiles.git file:////gitserver/repos/dotfiles.git
	gomir add https://github.com/spf13/cobra.git file:////gitserver/repos/cobra.git
	# Connect to the destination network
	gomir push

	# Time passes...

	# Connect to the source network
	gomir fetch
	# Connect to the destination network
	gomir push

TODO:
	- Add support for controlling concurrency during fetch/push
	- Add documentation describing where repos are stored
	- Add documentation describing what to do if something goes wrong during a fetch/push
	- Add documentation for a normal synchronization workflow
*/
package main

import (
	"fmt"
	"io"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"

	"github.com/fatih/color"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// Set via ldflags
var version string
var gitCommit string
var buildDate string

func main() {
	rootCmd := &cobra.Command{
		Use:  "gomir",
		Long: `Mirror Git repositories between two disconnected networks`,
	}

	addCmd := &cobra.Command{
		Use:   "add <fetchURL> <pushURL> [<localDest>]",
		Short: "Add a repository to mirror",
		Args:  cobra.RangeArgs(2, 3),
		Run: func(cmd *cobra.Command, args []string) {
			switch len(args) {
			case 2:
				add(args[0], args[1], "")
			case 3:
				add(args[0], args[1], args[2])
			default:
				fmt.Println("Wrong number of arguments")
				os.Exit(1)
			}
		},
	}

	fetchCmd := &cobra.Command{
		Use:   "fetch",
		Short: "Fetch changes for all mirroed repositories",
		Args:  cobra.NoArgs,
		Run: func(cmd *cobra.Command, args []string) {
			fetch()
		},
	}

	pushCmd := &cobra.Command{
		Use:   "push",
		Short: "Push changes for all mirrored repositories",
		Args:  cobra.NoArgs,
		Run: func(cmd *cobra.Command, args []string) {
			push()
		},
	}

	versionCmd := &cobra.Command{
		Use:   "version",
		Short: "Show gomir version information",
		Args:  cobra.NoArgs,
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Printf("  Version:    %v\n", version)
			fmt.Printf("  Git commit: %v\n", gitCommit)
			fmt.Printf("  Built:      %v\n", buildDate)
		},
	}

	rootCmd.AddCommand(addCmd, fetchCmd, pushCmd, versionCmd)
	rootCmd.Execute()
}

func add(fetchURL, pushURL, localDest string) {
	// Try to generate a localDest
	if localDest == "" {
		u, err := url.Parse(fetchURL)
		if err != nil {
			fmt.Println("Could not generate a localDest")
			os.Exit(1)
		}
		localDest = fmt.Sprintf("%v%v", u.Host, u.Path)
	}

	localDest = ensureGitExt(localDest)

	// Clone
	if err := gitCloneMirror(fetchURL, localDest); err != nil {
		fmt.Println("Error cloning repository")
		os.Exit(1)
	}

	// Set Push URL
	if err := gitSetOriginPushURL(localDest, pushURL); err != nil {
		fmt.Println("Error setting push URL")
		os.Exit(1)
	}
}

func fetch() {
	errCount := performOperationAsync(fetchSingle)
	if errCount > 0 {
		color.Red("Fetch failed for %v repos", errCount)
		os.Exit(1)
	}
}

func fetchSingle(gitDir string) bool {
	logFile, logger, err := getLog(gitDir, "FETCH: ")
	if err != nil {
		color.Red("Error opening log file for %v", gitDir)
		return false
	}
	defer logFile.Close()

	logger.Println("Start")
	success := gitFetchPrune(gitDir, logFile) == nil
	logger.Printf("Done, success:%v", success)
	return success
}

func push() {
	errCount := performOperationAsync(pushSingle)
	if errCount > 0 {
		color.Red("Push failed for %v repos", errCount)
		os.Exit(1)
	}
}

func pushSingle(gitDir string) bool {
	logFile, logger, err := getLog(gitDir, "PUSH: ")
	if err != nil {
		color.Red("Error opening log file for %v", gitDir)
		return false
	}
	defer logFile.Close()

	logger.Println("Start")

	// Where are we pushing to?
	pushURL, err := gitGetOriginPushURL(gitDir)
	if err != nil {
		logger.Printf("Error retrieving push URL: %+v", err)
		return false
	}

	// If pushing using file protocol and destination repository does
	// not alread exist, initialize it
	isFileProtocol := pushURL.Scheme == "" || strings.ToLower(pushURL.Scheme) == "file"
	if isFileProtocol {
		_, err := os.Stat(pushURL.Path)
		if err != nil && os.IsNotExist(err) {
			if err := gitInitBareRepo(pushURL.Path, logFile); err != nil {
				logger.Printf("Error initializing bare git repository: %+v", err)
				return false
			}
		}
	}

	// Push
	if err := gitPushMirror(gitDir, logFile); err != nil {
		logger.Printf("Error pushing: %+v", err)
		return false
	}

	// Update server info
	if isFileProtocol {
		if err := gitUpdateServerInfo(pushURL.Path, logFile); err != nil {
			logger.Printf("Error updating server info: %+v", err)
			return false
		}
	}

	logger.Println("Done")
	return true
}

func findGitDirs() []string {
	gitDirs := []string{}
	filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
		if err == nil &&
			info.IsDir() &&
			strings.ToLower(filepath.Base(path)) != ".git" &&
			strings.ToLower(filepath.Ext(path)) == ".git" {
			gitDirs = append(gitDirs, path)
			return filepath.SkipDir
		}
		return nil
	})
	return gitDirs
}

func ensureGitExt(str string) string {
	if strings.ToLower(filepath.Ext(str)) != ".git" {
		return fmt.Sprintf("%v.git", str)
	}
	return str
}

func getLog(gitDir, prefix string) (io.WriteCloser, *log.Logger, error) {
	logPath := fmt.Sprintf("%v.log", gitDir)
	logFile, err := os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return nil, nil, errors.Wrap(err, "Error opening log file")
	}

	logger := log.New(logFile, prefix, log.Ldate|log.Ltime|log.Lshortfile|log.LUTC)
	return logFile, logger, nil
}

type gitDirOperation func(gitDir string) bool

func performOperationAsync(op gitDirOperation) int64 {
	var wg sync.WaitGroup
	var errCount int64
	for _, gitDir := range findGitDirs() {
		wg.Add(1)
		go func(gitDir string) {
			defer wg.Done()

			if op(gitDir) {
				color.Green("[✔] %v", gitDir)
			} else {
				atomic.AddInt64(&errCount, 1)
				color.Red("[X] %v", gitDir)
			}
		}(gitDir)
	}

	wg.Wait()
	return errCount
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"

	"kubernetes-cloudwatch-exporter/util"

	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
	settingsFile = flag.String("settings-file", "./settings.json", "Path to load as the settings file")
	promMetrics  = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "k8s_cw_metric",
			Help: "Cloudwatch Metrics.",
		},
		[]string{"elb", "app", "namespace", "metric", "statistic"},
	)
)

func init() {
	prometheus.MustRegister(promMetrics)
}

func main() {
	settings, err := util.NewSettings(*settingsFile)

	if err != nil {
		log.Fatalf("settings.NewSettings %v", err)
	}

	sess := session.Must(session.NewSession(&aws.Config{
		Region: aws.String(settings.AWSRegion),
	}))

	getELBNamesFunc := util.MakeELBNamesFunc(settings.TagName, settings.TagValue, settings.AppTagName, sess)
	getMetricsFunc := util.MakeMetricsFunc(sess)

	go func() {
		for {
			elbDescriptions, err := getELBNamesFunc()

			if err != nil {
				log.Fatalf("elbFunc %v", err)
			}

			fmt.Printf("Found %d load balancers\n", len(elbDescriptions))

			for _, elbDesc := range elbDescriptions {
				for _, elbMetric := range settings.Metrics {

					log.Printf("Requesting Metrics %v", elbMetric)
					log.Printf("Requesting for ELB %v", *elbDesc.Name)

					datapoints, err := getMetricsFunc(elbDesc.Name, &elbMetric, settings)

					if err != nil {
						log.Fatalf("metricsFunc %v", err)
					}

					log.Printf("Datapoints %v", datapoints)

					observeDatapoints(datapoints, elbMetric, elbDesc)
				}
			}

			time.Sleep(60 * time.Second)
		}
	}()

	// Expose the registered metrics via HTTP.
	http.Handle("/metrics", promhttp.Handler())
	log.Fatal(http.ListenAndServe(":8080", nil))
}

func observeDatapoints(datapoints []*cloudwatch.Datapoint, elbMetric util.ELBMetric, elbDesc *util.ELBDescription) {

	for _, dp := range datapoints {
		for n, v := range getMetrics(dp) {
			promMetrics.WithLabelValues(*elbDesc.Name, *elbDesc.AppName, *elbDesc.AppNamespace, elbMetric.Name, n).Set(v)
		}
	}
}

func getMetrics(dp *cloudwatch.Datapoint) map[string]float64 {

	metrics := make(map[string]float64)

	if dp.Average != nil {
		metrics["Average"] = *dp.Average
	}

	if dp.Maximum != nil {
		metrics["Maximum"] = *dp.Maximum
	}

	if dp.Minimum != nil {
		metrics["Minimum"] = *dp.Minimum
	}

	if dp.SampleCount != nil {
		metrics["SampleCount"] = *dp.SampleCount
	}

	if dp.Sum != nil {
		metrics["Sum"] = *dp.Sum
	}

	if dp.ExtendedStatistics != nil {
		for p, v := range dp.ExtendedStatistics {
			metrics[p] = *v
		}
	}

	return metrics
}
<commit_msg>Minor var name refactoring<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"

	"kubernetes-cloudwatch-exporter/util"

	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
	_settingsFile = flag.String("settings-file", "./settings.json", "Path to load as the settings file")
	_elbMetrics   = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "k8s_cw_metric",
			Help: "Kubernetes ELB metrics",
		},
		[]string{"elb", "app", "namespace", "metric", "statistic"},
	)
)

func init() {
	prometheus.MustRegister(_elbMetrics)
}

func main() {
	settings, err := util.NewSettings(*_settingsFile)

	if err != nil {
		log.Fatalf("settings.NewSettings %v", err)
	}

	sess := session.Must(session.NewSession(&aws.Config{
		Region: aws.String(settings.AWSRegion),
	}))

	getELBNamesFunc := util.MakeELBNamesFunc(settings.TagName, settings.TagValue, settings.AppTagName, sess)
	getMetricsFunc := util.MakeMetricsFunc(sess)

	go func() {
		for {
			elbDescriptions, err := getELBNamesFunc()

			if err != nil {
				log.Fatalf("elbFunc %v", err)
			}

			fmt.Printf("Found %d load balancers\n", len(elbDescriptions))

			for _, elbDesc := range elbDescriptions {
				for _, elbMetric := range settings.Metrics {

					log.Printf("Requesting Metrics %v", elbMetric)
					log.Printf("Requesting for ELB %v", *elbDesc.Name)

					datapoints, err := getMetricsFunc(elbDesc.Name, &elbMetric, settings)

					if err != nil {
						log.Fatalf("metricsFunc %v", err)
					}

					log.Printf("Datapoints %v", datapoints)

					observeDatapoints(datapoints, elbMetric, elbDesc)
				}
			}

			time.Sleep(60 * time.Second)
		}
	}()

	// Expose the registered metrics via HTTP.
	http.Handle("/metrics", promhttp.Handler())
	log.Fatal(http.ListenAndServe(":8080", nil))
}

func observeDatapoints(datapoints []*cloudwatch.Datapoint, elbMetric util.ELBMetric, elbDesc *util.ELBDescription) {

	for _, dp := range datapoints {
		for n, v := range getMetrics(dp) {
			_elbMetrics.WithLabelValues(*elbDesc.Name, *elbDesc.AppName, *elbDesc.AppNamespace, elbMetric.Name, n).Set(v)
		}
	}
}

func getMetrics(dp *cloudwatch.Datapoint) map[string]float64 {

	metrics := make(map[string]float64)

	if dp.Average != nil {
		metrics["Average"] = *dp.Average
	}

	if dp.Maximum != nil {
		metrics["Maximum"] = *dp.Maximum
	}

	if dp.Minimum != nil {
		metrics["Minimum"] = *dp.Minimum
	}

	if dp.SampleCount != nil {
		metrics["SampleCount"] = *dp.SampleCount
	}

	if dp.Sum != nil {
		metrics["Sum"] = *dp.Sum
	}

	if dp.ExtendedStatistics != nil {
		for p, v := range dp.ExtendedStatistics {
			metrics[p] = *v
		}
	}

	return metrics
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
)

func main() {
	if len(os.Args) != 2 {
		fmt.Fprintln(os.Stderr, "usage: px86 filename")
		return
	}

	fileName := os.Args[1]

	f, err := os.Open(fileName)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%sファイルが開けません: %v", fileName, err)
		return
	}

	emu := newEmulator()
	if err := emu.load(f); err != nil {
		fmt.Fprintf(os.Stderr, "%sファイルが読み込めません: %v", fileName, err)
	}

	emu.eval()
}
<commit_msg>close the file<commit_after>package main

import (
	"fmt"
	"os"
)

func main() {
	if len(os.Args) != 2 {
		fmt.Fprintln(os.Stderr, "usage: px86 filename")
		return
	}

	fileName := os.Args[1]

	f, err := os.Open(fileName)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%sファイルが開けません: %v", fileName, err)
		return
	}
	defer f.Close()

	emu := newEmulator()
	if err := emu.load(f); err != nil {
		fmt.Fprintf(os.Stderr, "%sファイルが読み込めません: %v", fileName, err)
	}

	emu.eval()
}
<|endoftext|>


<commit_before>// Reading and writing files are basic tasks needed for
// many Go programs. First we'll look at some examples of
// reading files.

package main

import (
	"bufio"
	"fmt"
	// "io"
	//"io/ioutil"
	"os"
	"strconv"
	"strings"
)

// Reading files requires checking most calls for errors.
// This helper will streamline our error checks below.
func check(e error) {
	if e != nil {
		panic(e)
	}
}

type Accessory struct {
	ItemNumber string
	Price      float64
}

func main() {
	fmt.Println("App started")
	// slow version from internet, but maybe correct one?
	file, err := os.Open("P05D150701.TXT")
	check(err)
	//
	defer file.Close()
	//
	var lines []Accessory
	var line string
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line = scanner.Text()
		itemNumber := line[1 : len(line)-64]
		price, err := strconv.ParseFloat(strings.Replace(line[47:len(line)-23], ",", ".", -1), 2)
		check(err)
		l := Accessory{itemNumber, price}
		lines = append(lines, l)
	}
	//fmt.Print(lines)
	fmt.Println("done")

	// Perhaps the most basic file reading task is
	// slurping a file's entire contents into memory.
	//dat, err := ioutil.ReadFile("P05D150701.TXT")
	//check(err)
	//fmt.Print(string(dat))
	//fmt.Print("done")

	// You'll often want more control over how and what
	// parts of a file are read. For these tasks, start
	// by `Open`ing a file to obtain an `os.File` value.
	// f, err := os.Open("P05D150701.TXT")
	// check(err)

	// Read some bytes from the beginning of the file.
	// Allow up to 5 to be read but also note how many
	// actually were read.
	// b1 := make([]byte, 50)
	// n1, err := f.Read(b1)
	// check(err)
	// fmt.Printf("%d bytes: %s\n", n1, string(b1))

	// You can also `Seek` to a known location in the file
	// and `Read` from there.
	// o2, err := f.Seek(6, 0)
	// check(err)
	// b2 := make([]byte, 2)
	// n2, err := f.Read(b2)
	// check(err)
	// fmt.Printf("%d bytes @ %d: %s\n", n2, o2, string(b2))

	// The `io` package provides some functions that may
	// be helpful for file reading. For example, reads
	// like the ones above can be more robustly
	// implemented with `ReadAtLeast`.
	// o3, err := f.Seek(6, 0)
	// check(err)
	// b3 := make([]byte, 2)
	// n3, err := io.ReadAtLeast(f, b3, 2)
	// check(err)
	// fmt.Printf("%d bytes @ %d: %s\n", n3, o3, string(b3))

	// There is no built-in rewind, but `Seek(0, 0)`
	// accomplishes this.
	// _, err = f.Seek(0, 0)
	// check(err)

	// The `bufio` package implements a buffered
	// reader that may be useful both for its efficiency
	// with many small reads and because of the additional
	// reading methods it provides.
	// r4 := bufio.NewReader(f)
	// b4, err := r4.Peek(50)
	// check(err)
	// fmt.Printf("50 bytes: %s\n", string(b4))

	// Close the file when you're done (usually this would
	// be scheduled immediately after `Open`ing with
	// `defer`).
	//f.Close()

}
<commit_msg>add concurrent funtion to compare accessory lists<commit_after>// Reading and writing files are basic tasks needed for
// many Go programs. First we'll look at some examples of
// reading files.

package main

import (
	"bufio"
	"fmt"
	// "io"
	//"io/ioutil"
	"os"
	"strconv"
	"strings"
	"sync"
)

// Reading files requires checking most calls for errors.
// This helper will streamline our error checks below.
func check(e error) {
	if e != nil {
		panic(e)
	}
}

type Accessory struct {
	ItemNumber string
	Price      float64
}

func main() {
	fmt.Println("App started")
	// slow version from internet, but maybe correct one?
	file, err := os.Open("P05D150701.TXT")
	check(err)
	//
	defer file.Close()
	//
	var lines []Accessory
	var line string
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line = scanner.Text()
		itemNumber := line[1 : len(line)-64]
		price, err := strconv.ParseFloat(strings.Replace(line[47:len(line)-23], ",", ".", -1), 2)
		check(err)
		l := Accessory{itemNumber, price}
		lines = append(lines, l)
	}
	//fmt.Print(lines)

	otherAccessories := []Accessory{
		Accessory{"a", 12.4},
		Accessory{"55110-87L00-0EP", 12.4},
		Accessory{"b", 12.4},
		Accessory{"55110-97J01-0EP", 12.4},
		Accessory{"c", 12.4},
		Accessory{"d", 12.4},
	}

	var wg sync.WaitGroup

	wg.Add(len(lines))

	for _, v := range otherAccessories {
		go func(a1 Accessory, file []Accessory) {
			for _, l := range file {
				defer wg.Done()
				if a1.ItemNumber == l.ItemNumber {
					fmt.Println(a1.ItemNumber)
					//f, err := os.Create("output.txt")
					//check(err)
					//defer f.Close()
					//d1 := []byte(a1.ItemNumber)
					//_, err = f.WriteString(string(d1))
					//err := ioutil.WriteFile("output.txt", d1, 0644)
					//check(err)
				}
			}
		}(v, lines)
	}

	wg.Wait()
	//fmt.Print(otherAccessories)
	fmt.Println("done")

	// Perhaps the most basic file reading task is
	// slurping a file's entire contents into memory.
	//dat, err := ioutil.ReadFile("P05D150701.TXT")
	//check(err)
	//fmt.Print(string(dat))
	//fmt.Print("done")

	// You'll often want more control over how and what
	// parts of a file are read. For these tasks, start
	// by `Open`ing a file to obtain an `os.File` value.
	// f, err := os.Open("P05D150701.TXT")
	// check(err)

	// Read some bytes from the beginning of the file.
	// Allow up to 5 to be read but also note how many
	// actually were read.
	// b1 := make([]byte, 50)
	// n1, err := f.Read(b1)
	// check(err)
	// fmt.Printf("%d bytes: %s\n", n1, string(b1))

	// You can also `Seek` to a known location in the file
	// and `Read` from there.
	// o2, err := f.Seek(6, 0)
	// check(err)
	// b2 := make([]byte, 2)
	// n2, err := f.Read(b2)
	// check(err)
	// fmt.Printf("%d bytes @ %d: %s\n", n2, o2, string(b2))

	// The `io` package provides some functions that may
	// be helpful for file reading. For example, reads
	// like the ones above can be more robustly
	// implemented with `ReadAtLeast`.
	// o3, err := f.Seek(6, 0)
	// check(err)
	// b3 := make([]byte, 2)
	// n3, err := io.ReadAtLeast(f, b3, 2)
	// check(err)
	// fmt.Printf("%d bytes @ %d: %s\n", n3, o3, string(b3))

	// There is no built-in rewind, but `Seek(0, 0)`
	// accomplishes this.
	// _, err = f.Seek(0, 0)
	// check(err)

	// The `bufio` package implements a buffered
	// reader that may be useful both for its efficiency
	// with many small reads and because of the additional
	// reading methods it provides.
	// r4 := bufio.NewReader(f)
	// b4, err := r4.Peek(50)
	// check(err)
	// fmt.Printf("50 bytes: %s\n", string(b4))

	// Close the file when you're done (usually this would
	// be scheduled immediately after `Open`ing with
	// `defer`).
	//f.Close()

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os/exec"
	"regexp"
	"strings"

	"github.com/russross/blackfriday"
)

// ChangeLog represents a collection of changes between versions
type ChangeLog struct {
	NameTags string
	Commits  []CommitMessage
}

// CommitMessage contains the commit information
type CommitMessage struct {
	ShortCommit string
	Commit      string
	Author      string
	Email       string
	Date        string
	Message     string
}

func fetchTags(releasePattern string) []string {
	out, _ := exec.Command("git", "log", "--tags", "--simplify-by-decoration", "--pretty=\"%ai @%d\"").Output()
	uncuratedTags := strings.Split(string(out), "\n")

	var tags []string
	validTag := regexp.MustCompile(fmt.Sprintf(`(.*\s)@\s.*tag:\s(%v)`, releasePattern))

	var match [][]string
	for _, tag := range uncuratedTags {
		match = validTag.FindAllStringSubmatch(tag, -1)
		if len(match) > 0 && len(match[0]) > 2 {
			// fmt.Printf("%v => %v \n", match[0][1], match[0][2])
			tags = append(tags, strings.Replace(match[0][2], ",", "", -1))
		}
	}
	return tags
}

func fetchChanges(releasePattern string) []ChangeLog {
	var changeLogs []ChangeLog
	tags := fetchTags(releasePattern)

	var diffs []string
	for i := 0; i < len(tags)-1; i++ {
		diffs = append(diffs, fmt.Sprintf("%v..%v", tags[i+1], tags[i]))
	}

	format := "--pretty=format:{\"shortcommit\":\"%h\", \"commit\":\"%H\", \"author\":\"%an\", \"email\":\"%ae\", \"date\":\"%ad\", \"message\":\"%f\"},"

	var out []byte
	var outs string
	var data []CommitMessage
	re := regexp.MustCompile(`,$`)
	for _, diff := range diffs {
		out, _ = exec.Command("git", "log", format, diff).Output()
		outs = fmt.Sprintf("[%v]", re.ReplaceAllString(string(out), ""))
		// fmt.Printf("%v\n", outs)
		if err := json.Unmarshal([]byte(outs), &data); err != nil {
			panic(err)
		}
		changeLogs = append(changeLogs, ChangeLog{NameTags: diff, Commits: data})
	}

	return changeLogs
}

func (c *CommitMessage) toMarkdown(originURL string) string {
	space := regexp.MustCompile(`-`)
	commitURL := fmt.Sprintf("%vcommit/%v", originURL, c.Commit)
	return fmt.Sprintf("* [%v](%v) %v [%v](mailto:%v)\n", c.ShortCommit, commitURL, space.ReplaceAllString(c.Message, " "), c.Author, c.Email)
}

func main() {
	originURL, _ := exec.Command("git", "config", "--get", "remote.origin.url").Output()
	re := regexp.MustCompile(`:`)
	originURL = []byte(re.ReplaceAllString(string(originURL), `/`))
	re = regexp.MustCompile(`\.git\n$`)
	originURL = []byte(re.ReplaceAllString(string(originURL), `/`))
	re = regexp.MustCompile(`^git@`)
	originURL = []byte(re.ReplaceAllString(string(originURL), `https://`))

	releasePattern := `v[\d{1,4}\.]{1,}`
	// releasePattern := `release-v?[\d{1,4}\.]{1,}`
	changeLog := fetchChanges(releasePattern)
	fmt.Println(changeLog) // json format

	var mdBuffer bytes.Buffer
	mdBuffer.WriteString("# Changelog\n")

	for _, change := range changeLog {
		mdBuffer.WriteString(fmt.Sprintf("\n## %v \n\n", change.NameTags))
		for _, commit := range change.Commits {
			mdBuffer.WriteString(commit.toMarkdown(string(originURL)))
		}
	}
	fmt.Println(mdBuffer.String())

	html := blackfriday.MarkdownCommon([]byte(mdBuffer.String()))

	fmt.Println(string(html))
}
<commit_msg>organize and structure formats into diferent functions<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os/exec"
	"regexp"
	"strings"
	"text/template"

	"github.com/russross/blackfriday"
)

const layout = `
<!DOCTYPE html>
<html lang="en">
  <head>
    <title>Changelog</title>
  </head>
  <body>
	  {{.Content}}
	</body>
</html>
`

// ChangeLog represents a collection of changes between versions
type ChangeLog struct {
	NameTags string
	Commits  []CommitMessage
}

// CommitMessage contains the commit information
type CommitMessage struct {
	ShortCommit string `json:"shortcommit"`
	Commit      string `json:"commit"`
	Author      string `json:"author"`
	Email       string `json:"email"`
	Date        string `json:"date"`
	Message     string `json:"message"`
}

// Template for html format
type Template struct {
	Content string
}

// Report full report structure
type Report struct {
	pattern   string
	originURL string
	ChangeLog []ChangeLog
	JSON      string
	Markdown  string
	HTML      string
}

func fetchTags(releasePattern string) []string {
	out, _ := exec.Command("git", "log", "--tags", "--simplify-by-decoration", "--pretty=\"%ai @%d\"").Output()
	uncuratedTags := strings.Split(string(out), "\n")

	var tags []string
	validTag := regexp.MustCompile(fmt.Sprintf(`(.*\s)@\s.*tag:\s(%v)`, releasePattern))

	var match [][]string
	for _, tag := range uncuratedTags {
		match = validTag.FindAllStringSubmatch(tag, -1)
		if len(match) > 0 && len(match[0]) > 2 {
			// fmt.Printf("%v => %v \n", match[0][1], match[0][2])
			tags = append(tags, strings.Replace(match[0][2], ",", "", -1))
		}
	}
	return tags
}

func fetchChanges(releasePattern string) []ChangeLog {
	var changeLogs []ChangeLog
	tags := fetchTags(releasePattern)

	var diffs []string
	for i := 0; i < len(tags)-1; i++ {
		diffs = append(diffs, fmt.Sprintf("%v..%v", tags[i+1], tags[i]))
	}

	format := "--pretty=format:{\"shortcommit\":\"%h\", \"commit\":\"%H\", \"author\":\"%an\", \"email\":\"%ae\", \"date\":\"%ad\", \"message\":\"%f\"},"

	var out []byte
	var outs string
	var data []CommitMessage
	re := regexp.MustCompile(`,$`)
	for _, diff := range diffs {
		out, _ = exec.Command("git", "log", format, diff).Output()
		outs = fmt.Sprintf("[%v]", re.ReplaceAllString(string(out), ""))
		// fmt.Printf("%v\n", outs)
		if err := json.Unmarshal([]byte(outs), &data); err != nil {
			panic(err)
		}
		changeLogs = append(changeLogs, ChangeLog{NameTags: diff, Commits: data})
	}

	return changeLogs
}

func (c *CommitMessage) toMarkdown(originURL string) string {
	space := regexp.MustCompile(`-`)
	commitURL := fmt.Sprintf("%vcommit/%v", originURL, c.Commit)
	return fmt.Sprintf("* [%v](%v) %v [%v](mailto:%v)\n", c.ShortCommit, commitURL, space.ReplaceAllString(c.Message, " "), c.Author, c.Email)
}

func fetchProjectURL() string {
	originURL, _ := exec.Command("git", "config", "--get", "remote.origin.url").Output()
	re := regexp.MustCompile(`:`)
	originURL = []byte(re.ReplaceAllString(string(originURL), `/`))
	re = regexp.MustCompile(`\.git\n$`)
	originURL = []byte(re.ReplaceAllString(string(originURL), `/`))
	re = regexp.MustCompile(`^git@`)
	originURL = []byte(re.ReplaceAllString(string(originURL), `https://`))

	return string(originURL)
}

func buildReport(releasePattern string) Report {
	report := Report{pattern: releasePattern, originURL: fetchProjectURL()}
	report.ChangeLog = fetchChanges(releasePattern)

	byteJSON, _ := json.Marshal(report.ChangeLog)
	report.JSON = string(byteJSON)

	return report
}

func (r *Report) getMarkdown() string {
	if len(r.Markdown) > 0 {
		return r.Markdown
	}
	var mdBuffer bytes.Buffer
	mdBuffer.WriteString("# Changelog\n")

	for _, change := range r.ChangeLog {
		mdBuffer.WriteString(fmt.Sprintf("\n## %v \n\n", change.NameTags))
		for _, commit := range change.Commits {
			mdBuffer.WriteString(commit.toMarkdown(string(r.originURL)))
		}
	}
	r.Markdown = mdBuffer.String()
	return r.Markdown
}

func (r *Report) getHTML() string {
	if len(r.HTML) > 0 {
		return r.HTML
	}
	out := bytes.NewBuffer(nil)

	html := blackfriday.MarkdownCommon([]byte(r.getMarkdown()))

	t := template.Must(template.New("layout").Parse(layout))
	err := t.Execute(out, Template{Content: string(html)})
	if err != nil {
		panic(err)
	}
	r.HTML = out.String()
	return r.HTML
}

func main() {
	releasePattern := `v[\d{1,4}\.]{1,}`
	// releasePattern := `release-v?[\d{1,4}\.]{1,}`
	report := buildReport(releasePattern)
	fmt.Println(report.JSON)
	fmt.Println(report.getMarkdown())
	fmt.Print(report.getHTML())
}
<|endoftext|>


<commit_before>package main

import (
  "./mega"
  "./task"
  "os"
  "fmt"
  "log"
  "io/ioutil"
)

const MaxWorker = 10

func main() {
  if len(os.Args) == 0 {
    log.Fatal("Please offer json file.")
  }
  pool, err := task.CreatePool()
  if err != nil {
    log.Fatal(err)
  }

  pool.Registry(task.MegaType, mega.MegaRoot{})

  file, err := os.Open(os.Args[1])
  if err != nil {
    log.Fatal(err)
  }
  defer file.Close()

  data, err := ioutil.ReadAll(file)
  if err != nil {
    log.Fatal(err)
  }

  _, err = pool.Add(task.MegaType, file.Name(), data)
  if err != nil {
    log.Fatal(err)
  }

  go pool.Start()

  for i := 0; i < MaxWorker; i++{
    go func(i int) {
      for {
        fmt.Println("Process", i, "asking...")
        t, ok := pool.Ask()
        if ok {
          fmt.Println("Process", i, "downloading...")
          t.Download()
          fmt.Println("Process", i, "reporting...")
          pool.Report(t)
          fmt.Println("Process", i, "done")
        }
      }
    }(i)
  }

  select{}
}
<commit_msg>Create workers base on cpu number.<commit_after>package main

import (
  "./mega"
  "./task"
  "os"
  "fmt"
  "log"
  "runtime"
  "io/ioutil"
)

var MaxWorker int

func init() {
  MaxWorker = runtime.NumCPU() << 1
}

func main() {
  if len(os.Args) == 0 {
    log.Fatal("Please offer json file.")
  }
  pool, err := task.CreatePool()
  if err != nil {
    log.Fatal(err)
  }

  pool.Registry(task.MegaType, mega.MegaRoot{})

  file, err := os.Open(os.Args[1])
  if err != nil {
    log.Fatal(err)
  }
  defer file.Close()

  data, err := ioutil.ReadAll(file)
  if err != nil {
    log.Fatal(err)
  }

  _, err = pool.Add(task.MegaType, file.Name(), data)
  if err != nil {
    log.Fatal(err)
  }

  go pool.Start()

  for i := 0; i < MaxWorker; i++{
    go func(i int) {
      for {
        fmt.Println("Process", i, "asking...")
        t, ok := pool.Ask()
        if ok {
          fmt.Println("Process", i, "downloading...")
          t.Download()
          fmt.Println("Process", i, "reporting...")
          pool.Report(t)
          fmt.Println("Process", i, "done")
        }
      }
    }(i)
  }

  select{}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Hanoverd Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/signal"
	"sync"

	"github.com/fsouza/go-dockerclient"
	"github.com/pwaller/barrier"
)

// DockerErrorStatus returns the HTTP status code represented by `err` or Status
// OK if no error or 0 if err != nil and is not a docker error.
func DockerErrorStatus(err error) int {
	if err, ok := err.(*docker.Error); ok {
		return err.Status
	}
	if err == nil {
		return http.StatusOK
	}
	return 0
}

func main() {
	log.Println("Hanoverd")

	client, err := docker.NewClient("unix:///run/docker.sock")
	if err != nil {
		log.Fatal(err)
	}

	var wg sync.WaitGroup
	defer wg.Wait()

	listener := make(chan *docker.APIEvents)
	client.AddEventListener(listener)
	defer close(listener)
	defer client.RemoveEventListener(listener)

	wg.Add(1)
	go func() {
		defer wg.Done()

		// TODO(pwaller): Unsure if this loop serves any purpose yet. We're
		// probably better off doing synchronization elsewhere.

		for ev := range listener {
			switch ev.Status {
			case "create":
			case "start":
				log.Println("Container started:", ev.ID)
			case "die":
				log.Println("Container finished:", ev.ID)
			default:
				log.Printf("Ev: %#+v", ev)
			}
		}
	}()

	// Fired when we're signalled to exit
	var dying barrier.Barrier
	defer dying.Fall()

	go func() {
		defer dying.Fall()
		// Await Stdin closure
		io.Copy(ioutil.Discard, os.Stdin)
	}()

	Go := func(c *Container) {
		defer wg.Done()
		defer dying.Fall()

		go func() {
			for err := range c.Errors {
				log.Println("BUG: Async container error:", err)
			}
		}()

		status, err := c.Run()
		if err != nil {
			log.Println(err)
			dying.Fall()
			return
		}
		log.Println("exit:", status)
	}

	c := NewContainer(client, "hello", &wg, &dying)
	wg.Add(1)
	go Go(c)

	go func() {
		sig := make(chan os.Signal)
		signal.Notify(sig, os.Interrupt)
		for {
			<-sig
			log.Println("Signalled!")

			d := NewContainer(client, "other", &wg, &dying)
			wg.Add(1)
			go Go(d)
		}
	}()

	<-dying.Barrier()
}
<commit_msg>Name containers incrementally<commit_after>// Copyright 2014 The Hanoverd Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/signal"
	"path"
	"sync"
	"sync/atomic"

	"github.com/fsouza/go-dockerclient"
	"github.com/pwaller/barrier"
)

// DockerErrorStatus returns the HTTP status code represented by `err` or Status
// OK if no error or 0 if err != nil and is not a docker error.
func DockerErrorStatus(err error) int {
	if err, ok := err.(*docker.Error); ok {
		return err.Status
	}
	if err == nil {
		return http.StatusOK
	}
	return 0
}

func main() {
	log.Println("Hanoverd")

	client, err := docker.NewClient("unix:///run/docker.sock")
	if err != nil {
		log.Fatal(err)
	}

	var wg sync.WaitGroup
	defer wg.Wait()

	listener := make(chan *docker.APIEvents)
	client.AddEventListener(listener)
	defer close(listener)
	defer client.RemoveEventListener(listener)

	wg.Add(1)
	go func() {
		defer wg.Done()

		// TODO(pwaller): Unsure if this loop serves any purpose yet. We're
		// probably better off doing synchronization elsewhere.

		for ev := range listener {
			switch ev.Status {
			case "create":
			case "start":
				log.Println("Container started:", ev.ID)
			case "die":
				log.Println("Container finished:", ev.ID)
			default:
				log.Printf("Ev: %#+v", ev)
			}
		}
	}()

	// Fired when we're signalled to exit
	var dying barrier.Barrier
	defer dying.Fall()

	go func() {
		defer dying.Fall()
		// Await Stdin closure
		io.Copy(ioutil.Discard, os.Stdin)
	}()

	Go := func(c *Container) {
		defer wg.Done()
		defer dying.Fall()

		go func() {
			for err := range c.Errors {
				log.Println("BUG: Async container error:", err)
			}
		}()

		status, err := c.Run()
		if err != nil {
			log.Println(err)
			dying.Fall()
			return
		}
		log.Println("exit:", status)
	}

	wd, err := os.Getwd()
	if err != nil {
		panic(err)
	}
	baseName := path.Base(wd)

	var i uint64
	getName := func() string {
		n := atomic.AddUint64(&i, 1)
		return fmt.Sprint(baseName, "_", n)
	}

	c := NewContainer(client, getName(), &wg, &dying)
	wg.Add(1)
	go Go(c)

	go func() {
		sig := make(chan os.Signal)
		signal.Notify(sig, os.Interrupt)
		for {
			<-sig
			log.Println("Signalled!")

			d := NewContainer(client, getName(), &wg, &dying)
			wg.Add(1)
			go Go(d)
		}
	}()

	<-dying.Barrier()
}
<|endoftext|>