<commit_before>// Copyright 2015-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// bbramfs builds a simple initramfs given an existing built bb; see bb.go
// You have to run bb first, which creates cmds/bb/bbsh. cd to that directory,
// and run bbramfs, and you have a single binary which does all u-root commands.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/u-root/u-root/pkg/cpio"
	_ "github.com/u-root/u-root/pkg/cpio/newc"
	"github.com/u-root/u-root/pkg/ldd"
)

var (
	// Paths contains the paths to put into the initramfs.
	// The index is a root directory, and the value is the place from which to
	// walk. The only required root is the bbsh dir itself, and the starting
	// walk is init -- i.e. we grab only one file. Should you wish to bring in,
	// e.g., /lib/modules/4.04, you would do add the root as / and the
	// starting point for the walk as lib/modules/4.04. That way we only preserve
	// as much of the path as we need, but we can preserve it all.
	paths = map[string][]string{}
	/*extraPaths = flag.String("extra", "", `Extra paths to add in the form root:start, e.g. /:etc/hosts.
	The path before the : is used as a starting point for a walk; the path after the : selects what things to put
	into the initramfs starting at /. E.g., /tmp/prototype:/ will install the prototype file system into / of the initramfs`)*/
	extraCmds = flag.String("cmds", "", "Extra commands to add (full path, comma-separated string)")
	extraCpio = flag.String("cpio", "", "A list of cpio archives to include in the output")
)

func sanity() {
	goBinGo := filepath.Join(config.Goroot, "bin/go")
	_, err := os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	// but does the one in go/bin/OS_ARCH exist too?
	goBinGo = filepath.Join(config.Goroot, fmt.Sprintf("bin/%s_%s/go", config.Goos, config.Arch))
	_, err = os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	if config.Go == "" {
		log.Fatalf("Can't find a go binary! Is GOROOT set correctly?")
	}
}

// dirComponents takes a string and returns an array of strings,
// such that we can create the directory records for intermediate
// directories.
func dirComponents(dir string) []string {
	var dirlist []string
	if filepath.Dir(dir) == "." {
		return []string{}
	}
	for d := filepath.Dir(dir); d != "/"; d = filepath.Dir(d) {
		dirlist = append([]string{d}, dirlist...)
	}
	dirlist = append(dirlist, dir)
	return dirlist
}

// copyCommands takes a list of commands, generates the list of libs,
// and creates cpio records, including directory records.
func copyCommands(w cpio.Writer, cmd string) {
	debug("copyCommands: start with %v", cmd)
	var recs []cpio.Record
	//introduced the tmpSlice variable to be the dependency slice for the cmd string
	tmpSlice := []string{cmd}
	libs, err := uroot.LddList(tmpSlice)
	if err != nil {
		log.Fatalf("%v\n", err)
	}
	tmpSlice = append(tmpSlice, libs...)
	for _, n := range tmpSlice {
		debug("\ncopyCommands: file %v", n)
		for _, n := range dirComponents(n) {
			debug("\ncopyCommands: %v", n)
			r, err := cpio.GetRecord(n)
			if err != nil {
				log.Fatalf("%v: %v", n, err)
			}
			recs = append(recs, r)
		}
	}
	cpio.MakeReproducible(recs)
	if err := w.WriteRecords(recs); err != nil {
		log.Fatalf("%v\n", err)
	}
}

func ramfs() {
	archiver, err := cpio.Format("newc")
	if err != nil {
		log.Fatalf("Creating newc archiver: %v", err)
	}

	oname := fmt.Sprintf("/tmp/initramfs.%v_%v.cpio", config.Goos, config.Arch)
	f, err := os.Create(oname)
	if err != nil {
		log.Fatalf("%v\n", err)
	}

	w := archiver.Writer(f)
	cpio.MakeReproducible(devCPIO[:])
	if err := w.WriteRecords(devCPIO[:]); err != nil {
		log.Fatalf("%v\n", err)
	}

	paths[filepath.Join(config.Gopath, "src/github.com/u-root/u-root/bb/bbsh")] = []string{"init", "ubin"}

	if *extraCmds != "" {
		copyc := strings.Fields(*extraCmds)
		//check if the path is a file or directory
		for _, eachPath := range copyc {
			//must not include ~ in path
			//replace only the first : with //
			modPath := strings.Replace(eachPath, ":", "", 1)
			fmt.Println("this is the path", modPath)
			statval, err := os.Stat(modPath)
			if err != nil {
				log.Fatalf("%v\n", err)
			}
			if statval.IsDir() {
				// If the file is a directory, append all the files in the directory to the path
				p := strings.Split(eachPath, ":")
				if len(p) != 2 {
					p = append([]string{"/"}, p...)
				}
				paths[p[0]] = append(paths[p[0]], p[1])
				debug("putps")
			} else {
				copyCommands(w, eachPath)
			}
		}
	}

	if *extraCpio != "" {
		extras := strings.Fields(*extraCpio)
		for _, x := range extras {
			a, err := cpio.Format("newc")
			if err != nil {
				log.Fatalf("Creating archiver: %v", err)
			}
			f, err := os.Open(x)
			if err != nil {
				log.Fatalf("%v: %v", x, err)
			}
			defer f.Close()
			rr := a.Reader(f)
			recs, err := rr.ReadRecords()
			if err != nil {
				log.Fatalf("read records: %v", err)
			}
			cpio.MakeReproducible(recs)
			if err := w.WriteRecords(recs); err != nil {
				log.Fatalf("%v\n", err)
			}
		}
	}

	// For all the 'roots' in paths, start walking at the name.
	fmt.Printf("PATHS: %v", paths)
	for r, list := range paths {
		debug("PATHS: root %v", r)
		// we need to make all the path prefix directories.
		for _, n := range list {
			debug("\troot %v, name %v", r, n)
			for _, d := range dirComponents(n) {
				debug("\t\troot %v, name %v, component %v", r, n, d)
				rec, err := cpio.GetRecord(d)
				if err != nil {
					log.Fatalf("Getting record of %q failed: %v", d, err)
				}
				recs := []cpio.Record{rec}
				cpio.MakeReproducible(recs)
				if err := w.WriteRecords(recs); err != nil {
					log.Fatalf("%v\n", err)
				}
			}
		}
		for _, n := range list {
			if err := filepath.Walk(filepath.Join(r, n), func(name string, fi os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				cn, err := filepath.Rel(r, name)
				if err != nil {
					log.Fatalf("filepath.Rel(%v, %v): %v", r, name, err)
				}
				debug("%v\n", cn)
				rec, err := cpio.GetRecord(name)
				if err != nil {
					log.Fatalf("Getting record of %q failed: %v", cn, err)
				}
				// the name in the cpio is relative to our starting point.
				rec.Name = cn
				recs := []cpio.Record{rec}
				cpio.MakeReproducible(recs)
				if err := w.WriteRecords(recs); err != nil {
					log.Fatalf("%v\n", err)
				}
				return nil
			}); err != nil {
				log.Fatalf("bbsh walk failed: %v", err)
			}
		}
	}

	if err := w.WriteTrailer(); err != nil {
		log.Fatalf("Error writing trailer record: %v", err)
	}
	fmt.Printf("Output file is in %v\n", oname)
}
<commit_msg> removed unnecessary comments<commit_after>// Copyright 2015-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// bbramfs builds a simple initramfs given an existing built bb; see bb.go
// You have to run bb first, which creates cmds/bb/bbsh. cd to that directory,
// and run bbramfs, and you have a single binary which does all u-root commands.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/u-root/u-root/pkg/cpio"
	_ "github.com/u-root/u-root/pkg/cpio/newc"
	"github.com/u-root/u-root/pkg/ldd"
)

var (
	// Paths contains the paths to put into the initramfs.
	// The index is a root directory, and the value is the place from which to
	// walk. The only required root is the bbsh dir itself, and the starting
	// walk is init -- i.e. we grab only one file. Should you wish to bring in,
	// e.g., /lib/modules/4.04, you would do add the root as / and the
	// starting point for the walk as lib/modules/4.04. That way we only preserve
	// as much of the path as we need, but we can preserve it all.
	paths = map[string][]string{}
	extraCmds = flag.String("cmds", "", "Extra commands to add (full path, comma-separated string)")
	extraCpio = flag.String("cpio", "", "A list of cpio archives to include in the output")
)

func sanity() {
	goBinGo := filepath.Join(config.Goroot, "bin/go")
	_, err := os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	// but does the one in go/bin/OS_ARCH exist too?
	goBinGo = filepath.Join(config.Goroot, fmt.Sprintf("bin/%s_%s/go", config.Goos, config.Arch))
	_, err = os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	if config.Go == "" {
		log.Fatalf("Can't find a go binary! Is GOROOT set correctly?")
	}
}

// dirComponents takes a string and returns an array of strings,
// such that we can create the directory records for intermediate
// directories.
func dirComponents(dir string) []string {
	var dirlist []string
	if filepath.Dir(dir) == "." {
		return []string{}
	}
	for d := filepath.Dir(dir); d != "/"; d = filepath.Dir(d) {
		dirlist = append([]string{d}, dirlist...)
	}
	dirlist = append(dirlist, dir)
	return dirlist
}

// copyCommands takes a list of commands, generates the list of libs,
// and creates cpio records, including directory records.
func copyCommands(w cpio.Writer, cmd string) {
	debug("copyCommands: start with %v", cmd)
	var recs []cpio.Record
	//introduced the tmpSlice variable to be the dependency slice for the cmd string
	tmpSlice := []string{cmd}
	libs, err := uroot.LddList(tmpSlice)
	if err != nil {
		log.Fatalf("%v\n", err)
	}
	tmpSlice = append(tmpSlice, libs...)
	for _, n := range tmpSlice {
		debug("\ncopyCommands: file %v", n)
		for _, n := range dirComponents(n) {
			debug("\ncopyCommands: %v", n)
			r, err := cpio.GetRecord(n)
			if err != nil {
				log.Fatalf("%v: %v", n, err)
			}
			recs = append(recs, r)
		}
	}
	cpio.MakeReproducible(recs)
	if err := w.WriteRecords(recs); err != nil {
		log.Fatalf("%v\n", err)
	}
}

func ramfs() {
	archiver, err := cpio.Format("newc")
	if err != nil {
		log.Fatalf("Creating newc archiver: %v", err)
	}

	oname := fmt.Sprintf("/tmp/initramfs.%v_%v.cpio", config.Goos, config.Arch)
	f, err := os.Create(oname)
	if err != nil {
		log.Fatalf("%v\n", err)
	}

	w := archiver.Writer(f)
	cpio.MakeReproducible(devCPIO[:])
	if err := w.WriteRecords(devCPIO[:]); err != nil {
		log.Fatalf("%v\n", err)
	}

	paths[filepath.Join(config.Gopath, "src/github.com/u-root/u-root/bb/bbsh")] = []string{"init", "ubin"}

	if *extraCmds != "" {
		copyc := strings.Fields(*extraCmds)
		//check if the path is a file or directory
		for _, eachPath := range copyc {
			//must not include ~ in path
			//replace only the first : with //
			modPath := strings.Replace(eachPath, ":", "", 1)
			fmt.Println("this is the path", modPath)
			statval, err := os.Stat(modPath)
			if err != nil {
				log.Fatalf("%v\n", err)
			}
			if statval.IsDir() {
				// If the file is a directory, append all the files in the directory to the path
				p := strings.Split(eachPath, ":")
				if len(p) != 2 {
					p = append([]string{"/"}, p...)
				}
				paths[p[0]] = append(paths[p[0]], p[1])
				debug("putps")
			} else {
				copyCommands(w, eachPath)
			}
		}
	}

	if *extraCpio != "" {
		extras := strings.Fields(*extraCpio)
		for _, x := range extras {
			a, err := cpio.Format("newc")
			if err != nil {
				log.Fatalf("Creating archiver: %v", err)
			}
			f, err := os.Open(x)
			if err != nil {
				log.Fatalf("%v: %v", x, err)
			}
			defer f.Close()
			rr := a.Reader(f)
			recs, err := rr.ReadRecords()
			if err != nil {
				log.Fatalf("read records: %v", err)
			}
			cpio.MakeReproducible(recs)
			if err := w.WriteRecords(recs); err != nil {
				log.Fatalf("%v\n", err)
			}
		}
	}

	// For all the 'roots' in paths, start walking at the name.
	fmt.Printf("PATHS: %v", paths)
	for r, list := range paths {
		debug("PATHS: root %v", r)
		// we need to make all the path prefix directories.
		for _, n := range list {
			debug("\troot %v, name %v", r, n)
			for _, d := range dirComponents(n) {
				debug("\t\troot %v, name %v, component %v", r, n, d)
				rec, err := cpio.GetRecord(d)
				if err != nil {
					log.Fatalf("Getting record of %q failed: %v", d, err)
				}
				recs := []cpio.Record{rec}
				cpio.MakeReproducible(recs)
				if err := w.WriteRecords(recs); err != nil {
					log.Fatalf("%v\n", err)
				}
			}
		}
		for _, n := range list {
			if err := filepath.Walk(filepath.Join(r, n), func(name string, fi os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				cn, err := filepath.Rel(r, name)
				if err != nil {
					log.Fatalf("filepath.Rel(%v, %v): %v", r, name, err)
				}
				debug("%v\n", cn)
				rec, err := cpio.GetRecord(name)
				if err != nil {
					log.Fatalf("Getting record of %q failed: %v", cn, err)
				}
				// the name in the cpio is relative to our starting point.
				rec.Name = cn
				recs := []cpio.Record{rec}
				cpio.MakeReproducible(recs)
				if err := w.WriteRecords(recs); err != nil {
					log.Fatalf("%v\n", err)
				}
				return nil
			}); err != nil {
				log.Fatalf("bbsh walk failed: %v", err)
			}
		}
	}

	if err := w.WriteTrailer(); err != nil {
		log.Fatalf("Error writing trailer record: %v", err)
	}
	fmt.Printf("Output file is in %v\n", oname)
}
<|endoftext|>


<commit_before>// Copyright 2016 Kinvolk
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bcc

import (
	"encoding/binary"
	"fmt"
	"sync"
	"unsafe"
)

/*
#cgo CFLAGS: -I/usr/include/bcc/compat
#cgo LDFLAGS: -lbcc
#include <bcc/bpf_common.h>
#include <bcc/libbpf.h>
#include <bcc/perf_reader.h>

// perf_reader_raw_cb as defined in bcc libbpf.h
// typedef void (*perf_reader_raw_cb)(void *cb_cookie, void *raw, int raw_size);
extern void callback_to_go(void*, void*, int);
*/
import "C"

type PerfMap struct {
	table   *Table
	readers []*C.struct_perf_reader
	stop    chan bool
}

type callbackData struct {
	receiverChan chan []byte
}

const BPF_PERF_READER_PAGE_CNT = 8

var byteOrder binary.ByteOrder
var callbackRegister = make(map[uint64]*callbackData)
var callbackIndex uint64
var mu sync.Mutex

// In lack of binary.HostEndian ...
func init() {
	var i int32 = 0x01020304
	u := unsafe.Pointer(&i)
	pb := (*byte)(u)
	b := *pb
	if b == 0x04 {
		byteOrder = binary.LittleEndian
	} else {
		byteOrder = binary.BigEndian
	}
}

func registerCallback(data *callbackData) uint64 {
	mu.Lock()
	defer mu.Unlock()
	callbackIndex++
	for callbackRegister[callbackIndex] != nil {
		callbackIndex++
	}
	callbackRegister[callbackIndex] = data
	return callbackIndex
}

func unregisterCallback(i uint64) {
	mu.Lock()
	defer mu.Unlock()
	delete(callbackRegister, i)
}

func lookupCallback(i uint64) *callbackData {
	return callbackRegister[i]
}

// Gateway function as required with CGO Go >= 1.6
// "If a C-program wants a function pointer, a gateway function has to
// be written. This is because we can't take the address of a Go
// function and give that to C-code since the cgo tool will generate a
// stub in C that should be called."
//export callback_to_go
func callback_to_go(cbCookie unsafe.Pointer, raw unsafe.Pointer, rawSize C.int) {
	callbackData := lookupCallback(uint64(uintptr(cbCookie)))
	receiverChan := callbackData.receiverChan
	go func() {
		receiverChan <- C.GoBytes(raw, rawSize)
	}()
}

// InitPerfMap initializes a perf map with a receiver channel.
func InitPerfMap(table *Table, receiverChan chan []byte) (*PerfMap, error) {
	fd := table.Config()["fd"].(int)
	keySize := table.Config()["key_size"].(uint64)
	leafSize := table.Config()["leaf_size"].(uint64)

	if keySize != 4 || leafSize != 4 {
		return nil, fmt.Errorf("passed table has wrong size")
	}

	callbackDataIndex := registerCallback(&callbackData{
		receiverChan,
	})

	key := make([]byte, keySize)
	leaf := make([]byte, leafSize)
	keyP := unsafe.Pointer(&key[0])
	leafP := unsafe.Pointer(&leaf[0])

	readers := []*C.struct_perf_reader{}

	cpu := 0
	res := 0
	for res == 0 {
		reader, err := C.bpf_open_perf_buffer((C.perf_reader_raw_cb)(unsafe.Pointer(C.callback_to_go)), unsafe.Pointer(uintptr(callbackDataIndex)), -1, C.int(cpu), BPF_PERF_READER_PAGE_CNT)
		if reader == nil {
			return nil, fmt.Errorf("failed to open perf buffer: %v", err)
		}

		perfFd := C.perf_reader_fd(reader)

		readers = append(readers, (*C.struct_perf_reader)(reader))

		byteOrder.PutUint32(leaf, uint32(perfFd))

		r, err := C.bpf_update_elem(C.int(fd), keyP, leafP, 0)
		if r != 0 {
			return nil, fmt.Errorf("unable to initialize perf map: %v", err)
		}

		res = int(C.bpf_get_next_key(C.int(fd), keyP, keyP))
		cpu++
	}
	return &PerfMap{
		table,
		readers,
		make(chan bool),
	}, nil
}

// Start to poll the perf map reader and send back event data
// over the connected channel.
func (pm *PerfMap) Start() {
	go pm.poll(500)
}

// Stop to poll the perf map readers after a maximum of 500ms
// (the timeout we use for perf_reader_poll). Ideally we would
// have a way to cancel the poll, but perf_reader_poll doesn't
// support that yet.
func (pm *PerfMap) Stop() {
	pm.stop <- true
}

func (pm *PerfMap) poll(timeout int) {
	for {
		select {
		case <-pm.stop:
			return
		default:
			C.perf_reader_poll(C.int(len(pm.readers)), &pm.readers[0], C.int(timeout))
		}
	}
}
<commit_msg>bcc: cast reader explicitly<commit_after>// Copyright 2016 Kinvolk
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bcc

import (
	"encoding/binary"
	"fmt"
	"sync"
	"unsafe"
)

/*
#cgo CFLAGS: -I/usr/include/bcc/compat
#cgo LDFLAGS: -lbcc
#include <bcc/bpf_common.h>
#include <bcc/libbpf.h>
#include <bcc/perf_reader.h>

// perf_reader_raw_cb as defined in bcc libbpf.h
// typedef void (*perf_reader_raw_cb)(void *cb_cookie, void *raw, int raw_size);
extern void callback_to_go(void*, void*, int);
*/
import "C"

type PerfMap struct {
	table   *Table
	readers []*C.struct_perf_reader
	stop    chan bool
}

type callbackData struct {
	receiverChan chan []byte
}

const BPF_PERF_READER_PAGE_CNT = 8

var byteOrder binary.ByteOrder
var callbackRegister = make(map[uint64]*callbackData)
var callbackIndex uint64
var mu sync.Mutex

// In lack of binary.HostEndian ...
func init() {
	var i int32 = 0x01020304
	u := unsafe.Pointer(&i)
	pb := (*byte)(u)
	b := *pb
	if b == 0x04 {
		byteOrder = binary.LittleEndian
	} else {
		byteOrder = binary.BigEndian
	}
}

func registerCallback(data *callbackData) uint64 {
	mu.Lock()
	defer mu.Unlock()
	callbackIndex++
	for callbackRegister[callbackIndex] != nil {
		callbackIndex++
	}
	callbackRegister[callbackIndex] = data
	return callbackIndex
}

func unregisterCallback(i uint64) {
	mu.Lock()
	defer mu.Unlock()
	delete(callbackRegister, i)
}

func lookupCallback(i uint64) *callbackData {
	return callbackRegister[i]
}

// Gateway function as required with CGO Go >= 1.6
// "If a C-program wants a function pointer, a gateway function has to
// be written. This is because we can't take the address of a Go
// function and give that to C-code since the cgo tool will generate a
// stub in C that should be called."
//export callback_to_go
func callback_to_go(cbCookie unsafe.Pointer, raw unsafe.Pointer, rawSize C.int) {
	callbackData := lookupCallback(uint64(uintptr(cbCookie)))
	receiverChan := callbackData.receiverChan
	go func() {
		receiverChan <- C.GoBytes(raw, rawSize)
	}()
}

// InitPerfMap initializes a perf map with a receiver channel.
func InitPerfMap(table *Table, receiverChan chan []byte) (*PerfMap, error) {
	fd := table.Config()["fd"].(int)
	keySize := table.Config()["key_size"].(uint64)
	leafSize := table.Config()["leaf_size"].(uint64)

	if keySize != 4 || leafSize != 4 {
		return nil, fmt.Errorf("passed table has wrong size")
	}

	callbackDataIndex := registerCallback(&callbackData{
		receiverChan,
	})

	key := make([]byte, keySize)
	leaf := make([]byte, leafSize)
	keyP := unsafe.Pointer(&key[0])
	leafP := unsafe.Pointer(&leaf[0])

	readers := []*C.struct_perf_reader{}

	cpu := 0
	res := 0
	for res == 0 {
		reader, err := C.bpf_open_perf_buffer((C.perf_reader_raw_cb)(unsafe.Pointer(C.callback_to_go)), unsafe.Pointer(uintptr(callbackDataIndex)), -1, C.int(cpu), BPF_PERF_READER_PAGE_CNT)
		if reader == nil {
			return nil, fmt.Errorf("failed to open perf buffer: %v", err)
		}

		perfFd := C.perf_reader_fd((*C.struct_perf_reader)(reader))

		readers = append(readers, (*C.struct_perf_reader)(reader))

		byteOrder.PutUint32(leaf, uint32(perfFd))

		r, err := C.bpf_update_elem(C.int(fd), keyP, leafP, 0)
		if r != 0 {
			return nil, fmt.Errorf("unable to initialize perf map: %v", err)
		}

		res = int(C.bpf_get_next_key(C.int(fd), keyP, keyP))
		cpu++
	}
	return &PerfMap{
		table,
		readers,
		make(chan bool),
	}, nil
}

// Start to poll the perf map reader and send back event data
// over the connected channel.
func (pm *PerfMap) Start() {
	go pm.poll(500)
}

// Stop to poll the perf map readers after a maximum of 500ms
// (the timeout we use for perf_reader_poll). Ideally we would
// have a way to cancel the poll, but perf_reader_poll doesn't
// support that yet.
func (pm *PerfMap) Stop() {
	pm.stop <- true
}

func (pm *PerfMap) poll(timeout int) {
	for {
		select {
		case <-pm.stop:
			return
		default:
			C.perf_reader_poll(C.int(len(pm.readers)), &pm.readers[0], C.int(timeout))
		}
	}
}
<|endoftext|>


<commit_before>package chart

import (
	"sync"

	"github.com/golang/freetype/truetype"
	"github.com/wcharczuk/go-chart/drawing"
)

const (
	// DefaultChartHeight is the default chart height.
	DefaultChartHeight = 400
	// DefaultChartWidth is the default chart width.
	DefaultChartWidth = 200
	// DefaultStrokeWidth is the default chart line/stroke width.
	DefaultStrokeWidth = 1.0
	// DefaultAxisLineWidth is the line width of the axis lines.
	DefaultAxisLineWidth = 1.0
	//DefaultDPI is the default dots per inch for the chart.
	DefaultDPI = 92.0
	// DefaultMinimumFontSize is the default minimum font size.
	DefaultMinimumFontSize = 8.0
	// DefaultFontSize is the default font size.
	DefaultFontSize = 10.0
	// DefaultTitleFontSize is the default title font size.
	DefaultTitleFontSize = 18.0
	// DefaultAnnotationDeltaWidth is the width of the left triangle out of annotations.
	DefaultAnnotationDeltaWidth = 10
	// DefaultAnnotationFontSize is the font size of annotations.
	DefaultAnnotationFontSize = 10.0
	// DefaultAxisFontSize is the font size of the axis labels.
	DefaultAxisFontSize = 10.0
	// DefaultTitleTop is the default distance from the top of the chart to put the title.
	DefaultTitleTop = 10

	// DefaultYAxisMargin is the default distance from the right of the canvas to the y axis labels.
	DefaultYAxisMargin = 10
	// DefaultXAxisMargin is the default distance from bottom of the canvas to the x axis labels.
	DefaultXAxisMargin = 10

	//DefaultVerticalTickWidth is half the margin.
	DefaultVerticalTickWidth = DefaultYAxisMargin >> 1

	//DefaultHorizontalTickWidth is half the margin.
	DefaultHorizontalTickWidth = DefaultXAxisMargin >> 1

	// DefaultMinimumTickHorizontalSpacing is the minimum distance between horizontal ticks.
	DefaultMinimumTickHorizontalSpacing = 20
	// DefaultMinimumTickVerticalSpacing is the minimum distance between vertical ticks.
	DefaultMinimumTickVerticalSpacing = 20

	// DefaultDateFormat is the default date format.
	DefaultDateFormat = "2006-01-02"
)

var (
	// DefaultBackgroundColor is the default chart background color.
	// It is equivalent to css color:white.
	DefaultBackgroundColor = drawing.Color{R: 255, G: 255, B: 255, A: 255}
	// DefaultBackgroundStrokeColor is the default chart border color.
	// It is equivalent to color:white.
	DefaultBackgroundStrokeColor = drawing.Color{R: 255, G: 255, B: 255, A: 255}
	// DefaultCanvasColor is the default chart canvas color.
	// It is equivalent to css color:white.
	DefaultCanvasColor = drawing.Color{R: 255, G: 255, B: 255, A: 255}
	// DefaultCanvasStrokColor is the default chart canvas stroke color.
	// It is equivalent to css color:white.
	DefaultCanvasStrokColor = drawing.Color{R: 255, G: 255, B: 255, A: 255}
	// DefaultTextColor is the default chart text color.
	// It is equivalent to #333333.
	DefaultTextColor = drawing.Color{R: 51, G: 51, B: 51, A: 255}
	// DefaultAxisColor is the default chart axis line color.
	// It is equivalent to #333333.
	DefaultAxisColor = drawing.Color{R: 51, G: 51, B: 51, A: 255}
	// DefaultStrokeColor is the default chart border color.
	// It is equivalent to #efefef.
	DefaultStrokeColor = drawing.Color{R: 239, G: 239, B: 239, A: 255}
	// DefaultFillColor is the default fill color.
	// It is equivalent to #0074d9.
	DefaultFillColor = drawing.Color{R: 0, G: 217, B: 116, A: 255}
	// DefaultAnnotationFillColor is the default annotation background color.
	DefaultAnnotationFillColor = drawing.Color{R: 255, G: 255, B: 255, A: 255}
)

var (
	// DefaultSeriesStrokeColors are a couple default series colors.
	DefaultSeriesStrokeColors = []drawing.Color{
		drawing.Color{R: 0, G: 116, B: 217, A: 255},
		drawing.Color{R: 0, G: 217, B: 116, A: 255},
		drawing.Color{R: 217, G: 0, B: 116, A: 255},
	}
)

// GetDefaultSeriesStrokeColor returns a color from the default list by index.
// NOTE: the index will wrap around (using a modulo).g
func GetDefaultSeriesStrokeColor(index int) drawing.Color {
	finalIndex := index % len(DefaultSeriesStrokeColors)
	return DefaultSeriesStrokeColors[finalIndex]
}

var (
	// DefaultAnnotationPadding is the padding around an annotation.
	DefaultAnnotationPadding = Box{Top: 5, Left: 0, Right: 7, Bottom: 5}
	// DefaultBackgroundPadding is the default canvas padding config.
	DefaultBackgroundPadding = Box{Top: 5, Left: 5, Right: 5, Bottom: 5}
)

var (
	_defaultFontLock sync.Mutex
	_defaultFont     *truetype.Font
)

// GetDefaultFont returns the default font (Roboto-Medium).
func GetDefaultFont() (*truetype.Font, error) {
	if _defaultFont == nil {
		_defaultFontLock.Lock()
		defer _defaultFontLock.Unlock()
		if _defaultFont == nil {
			font, err := truetype.Parse(roboto)
			if err != nil {
				return nil, err
			}
			_defaultFont = font
		}
	}
	return _defaultFont, nil
}
<commit_msg>tweaks<commit_after>package chart

import (
	"sync"

	"github.com/golang/freetype/truetype"
	"github.com/wcharczuk/go-chart/drawing"
)

const (
	// DefaultChartHeight is the default chart height.
	DefaultChartHeight = 400
	// DefaultChartWidth is the default chart width.
	DefaultChartWidth = 200
	// DefaultStrokeWidth is the default chart line/stroke width.
	DefaultStrokeWidth = 1.0
	// DefaultAxisLineWidth is the line width of the axis lines.
	DefaultAxisLineWidth = 1.0
	//DefaultDPI is the default dots per inch for the chart.
	DefaultDPI = 92.0
	// DefaultMinimumFontSize is the default minimum font size.
	DefaultMinimumFontSize = 8.0
	// DefaultFontSize is the default font size.
	DefaultFontSize = 10.0
	// DefaultTitleFontSize is the default title font size.
	DefaultTitleFontSize = 18.0
	// DefaultAnnotationDeltaWidth is the width of the left triangle out of annotations.
	DefaultAnnotationDeltaWidth = 10
	// DefaultAnnotationFontSize is the font size of annotations.
	DefaultAnnotationFontSize = 10.0
	// DefaultAxisFontSize is the font size of the axis labels.
	DefaultAxisFontSize = 10.0
	// DefaultTitleTop is the default distance from the top of the chart to put the title.
	DefaultTitleTop = 10

	// DefaultYAxisMargin is the default distance from the right of the canvas to the y axis labels.
	DefaultYAxisMargin = 10
	// DefaultXAxisMargin is the default distance from bottom of the canvas to the x axis labels.
	DefaultXAxisMargin = 10

	//DefaultVerticalTickWidth is half the margin.
	DefaultVerticalTickWidth = DefaultYAxisMargin >> 1

	//DefaultHorizontalTickWidth is half the margin.
	DefaultHorizontalTickWidth = DefaultXAxisMargin >> 1

	// DefaultMinimumTickHorizontalSpacing is the minimum distance between horizontal ticks.
	DefaultMinimumTickHorizontalSpacing = 20
	// DefaultMinimumTickVerticalSpacing is the minimum distance between vertical ticks.
	DefaultMinimumTickVerticalSpacing = 20

	// DefaultDateFormat is the default date format.
	DefaultDateFormat = "2006-01-02"
)

var (
	// DefaultBackgroundColor is the default chart background color.
	// It is equivalent to css color:white.
	DefaultBackgroundColor = drawing.Color{R: 255, G: 255, B: 255, A: 255}
	// DefaultBackgroundStrokeColor is the default chart border color.
	// It is equivalent to color:white.
	DefaultBackgroundStrokeColor = drawing.Color{R: 255, G: 255, B: 255, A: 255}
	// DefaultCanvasColor is the default chart canvas color.
	// It is equivalent to css color:white.
	DefaultCanvasColor = drawing.Color{R: 255, G: 255, B: 255, A: 255}
	// DefaultCanvasStrokColor is the default chart canvas stroke color.
	// It is equivalent to css color:white.
	DefaultCanvasStrokColor = drawing.Color{R: 255, G: 255, B: 255, A: 255}
	// DefaultTextColor is the default chart text color.
	// It is equivalent to #333333.
	DefaultTextColor = drawing.Color{R: 51, G: 51, B: 51, A: 255}
	// DefaultAxisColor is the default chart axis line color.
	// It is equivalent to #333333.
	DefaultAxisColor = drawing.Color{R: 51, G: 51, B: 51, A: 255}
	// DefaultStrokeColor is the default chart border color.
	// It is equivalent to #efefef.
	DefaultStrokeColor = drawing.Color{R: 239, G: 239, B: 239, A: 255}
	// DefaultFillColor is the default fill color.
	// It is equivalent to #0074d9.
	DefaultFillColor = drawing.Color{R: 0, G: 217, B: 116, A: 255}
	// DefaultAnnotationFillColor is the default annotation background color.
	DefaultAnnotationFillColor = drawing.Color{R: 255, G: 255, B: 255, A: 255}
)

var (
	// DefaultSeriesStrokeColors are a couple default series colors.
	DefaultSeriesStrokeColors = []drawing.Color{
		drawing.Color{R: 0, G: 116, B: 217, A: 255},
		drawing.Color{R: 0, G: 217, B: 116, A: 255},
		drawing.Color{R: 217, G: 0, B: 116, A: 255},
	}
)

// GetDefaultSeriesStrokeColor returns a color from the default list by index.
// NOTE: the index will wrap around (using a modulo).g
func GetDefaultSeriesStrokeColor(index int) drawing.Color {
	finalIndex := index % len(DefaultSeriesStrokeColors)
	return DefaultSeriesStrokeColors[finalIndex]
}

var (
	// DefaultAnnotationPadding is the padding around an annotation.
	DefaultAnnotationPadding = Box{Top: 5, Left: 5, Right: 5, Bottom: 5}
	// DefaultBackgroundPadding is the default canvas padding config.
	DefaultBackgroundPadding = Box{Top: 5, Left: 5, Right: 5, Bottom: 5}
)

var (
	_defaultFontLock sync.Mutex
	_defaultFont     *truetype.Font
)

// GetDefaultFont returns the default font (Roboto-Medium).
func GetDefaultFont() (*truetype.Font, error) {
	if _defaultFont == nil {
		_defaultFontLock.Lock()
		defer _defaultFontLock.Unlock()
		if _defaultFont == nil {
			font, err := truetype.Parse(roboto)
			if err != nil {
				return nil, err
			}
			_defaultFont = font
		}
	}
	return _defaultFont, nil
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package webrisk

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	pb "github.com/GoogleCloudPlatform/golang-samples/webrisk/internal/webrisk_proto"
	"github.com/golang/protobuf/proto"
)

const (
	findHashPath    = "/$rpc/google.cloud.webrisk.v1beta1.WebRiskServiceV1Beta1/SearchHashes"
	fetchUpdatePath = "/$rpc/google.cloud.webrisk.v1beta1.WebRiskServiceV1Beta1/ComputeThreatListDiff"
)

// The api interface specifies wrappers around the Web Risk API.
type api interface {
	ListUpdate(ctx context.Context, req *pb.ComputeThreatListDiffRequest) (*pb.ComputeThreatListDiffResponse, error)
	HashLookup(ctx context.Context, req *pb.SearchHashesRequest) (*pb.SearchHashesResponse, error)
}

// netAPI is an api object that talks to the server over HTTP.
type netAPI struct {
	client *http.Client
	url    *url.URL
}

// newNetAPI creates a new netAPI object pointed at the provided root URL.
// For every request, it will use the provided API key.
// If a proxy URL is given, it will be used in place of the default $HTTP_PROXY.
// If the protocol is not specified in root, then this defaults to using HTTPS.
func newNetAPI(root string, key string, proxy string) (*netAPI, error) {
	if !strings.Contains(root, "://") {
		root = "https://" + root
	}
	u, err := url.Parse(root)
	if err != nil {
		return nil, err
	}

	httpClient := &http.Client{}

	if proxy != "" {
		proxyUrl, err := url.Parse(proxy)
		if err != nil {
			return nil, err
		}
		httpClient = &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
	}

	q := u.Query()
	q.Set("key", key)
	q.Set("alt", "proto")
	u.RawQuery = q.Encode()
	return &netAPI{url: u, client: httpClient}, nil
}

// doRequests performs a POST to requestPath. It uses the marshaled form of req
// as the request body payload, and automatically unmarshals the response body
// payload as resp.
func (a *netAPI) doRequest(ctx context.Context, requestPath string, req proto.Message, resp proto.Message) error {
	p, err := proto.Marshal(req)
	if err != nil {
		return err
	}

	u := *a.url // Make a copy of URL
	u.Path = requestPath
	httpReq, err := http.NewRequest("POST", u.String(), bytes.NewReader(p))
	httpReq.Header.Add("Content-Type", "application/x-protobuf")
	httpReq = httpReq.WithContext(ctx)
	httpResp, err := a.client.Do(httpReq)
	if err != nil {
		return err
	}
	defer httpResp.Body.Close()
	if httpResp.StatusCode != 200 {
		return fmt.Errorf("webrisk: unexpected server response code: %d", httpResp.StatusCode)
	}
	body, err := ioutil.ReadAll(httpResp.Body)
	if err != nil {
		return err
	}
	return proto.Unmarshal(body, resp)
}

// ListUpdate issues a ComputeThreatListDiff API call and returns the response.
func (a *netAPI) ListUpdate(ctx context.Context, req *pb.ComputeThreatListDiffRequest) (*pb.ComputeThreatListDiffResponse, error) {
	resp := new(pb.ComputeThreatListDiffResponse)
	return resp, a.doRequest(ctx, fetchUpdatePath, req, resp)
}

// HashLookup issues a SearchHashes API call and returns the response.
func (a *netAPI) HashLookup(ctx context.Context, req *pb.SearchHashesRequest) (*pb.SearchHashesResponse, error) {
	resp := new(pb.SearchHashesResponse)
	return resp, a.doRequest(ctx, findHashPath, req, resp)
}
<commit_msg>webrisk: add user agent header (#984)<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package webrisk

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	pb "github.com/GoogleCloudPlatform/golang-samples/webrisk/internal/webrisk_proto"
	"github.com/golang/protobuf/proto"
)

const (
	findHashPath    = "/$rpc/google.cloud.webrisk.v1beta1.WebRiskServiceV1Beta1/SearchHashes"
	fetchUpdatePath = "/$rpc/google.cloud.webrisk.v1beta1.WebRiskServiceV1Beta1/ComputeThreatListDiff"
)

// The api interface specifies wrappers around the Web Risk API.
type api interface {
	ListUpdate(ctx context.Context, req *pb.ComputeThreatListDiffRequest) (*pb.ComputeThreatListDiffResponse, error)
	HashLookup(ctx context.Context, req *pb.SearchHashesRequest) (*pb.SearchHashesResponse, error)
}

// netAPI is an api object that talks to the server over HTTP.
type netAPI struct {
	client *http.Client
	url    *url.URL
}

// newNetAPI creates a new netAPI object pointed at the provided root URL.
// For every request, it will use the provided API key.
// If a proxy URL is given, it will be used in place of the default $HTTP_PROXY.
// If the protocol is not specified in root, then this defaults to using HTTPS.
func newNetAPI(root string, key string, proxy string) (*netAPI, error) {
	if !strings.Contains(root, "://") {
		root = "https://" + root
	}
	u, err := url.Parse(root)
	if err != nil {
		return nil, err
	}

	httpClient := &http.Client{}

	if proxy != "" {
		proxyUrl, err := url.Parse(proxy)
		if err != nil {
			return nil, err
		}
		httpClient = &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
	}

	q := u.Query()
	q.Set("key", key)
	q.Set("alt", "proto")
	u.RawQuery = q.Encode()
	return &netAPI{url: u, client: httpClient}, nil
}

// doRequests performs a POST to requestPath. It uses the marshaled form of req
// as the request body payload, and automatically unmarshals the response body
// payload as resp.
func (a *netAPI) doRequest(ctx context.Context, requestPath string, req proto.Message, resp proto.Message) error {
	p, err := proto.Marshal(req)
	if err != nil {
		return err
	}

	u := *a.url // Make a copy of URL
	u.Path = requestPath
	httpReq, err := http.NewRequest("POST", u.String(), bytes.NewReader(p))
	httpReq.Header.Add("Content-Type", "application/x-protobuf")
	httpReq.Header.Add("User-Agnet", "Webrisk-Client/0.1.2")
	httpReq = httpReq.WithContext(ctx)
	httpResp, err := a.client.Do(httpReq)
	if err != nil {
		return err
	}
	defer httpResp.Body.Close()
	if httpResp.StatusCode != 200 {
		return fmt.Errorf("webrisk: unexpected server response code: %d", httpResp.StatusCode)
	}
	body, err := ioutil.ReadAll(httpResp.Body)
	if err != nil {
		return err
	}
	return proto.Unmarshal(body, resp)
}

// ListUpdate issues a ComputeThreatListDiff API call and returns the response.
func (a *netAPI) ListUpdate(ctx context.Context, req *pb.ComputeThreatListDiffRequest) (*pb.ComputeThreatListDiffResponse, error) {
	resp := new(pb.ComputeThreatListDiffResponse)
	return resp, a.doRequest(ctx, fetchUpdatePath, req, resp)
}

// HashLookup issues a SearchHashes API call and returns the response.
func (a *netAPI) HashLookup(ctx context.Context, req *pb.SearchHashesRequest) (*pb.SearchHashesResponse, error) {
	resp := new(pb.SearchHashesResponse)
	return resp, a.doRequest(ctx, findHashPath, req, resp)
}
<|endoftext|>


<commit_before>// Package wkhtmltopdf contains wrappers around the wkhtmltopdf commandline tool
package wkhtmltopdf

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
)

//the cached mutexed path as used by findPath()
type stringStore struct {
	val string
	sync.Mutex
}

func (ss *stringStore) Get() string {
	ss.Lock()
	defer ss.Unlock()
	return ss.val
}

func (ss *stringStore) Set(s string) {
	ss.Lock()
	ss.val = s
	ss.Unlock()
}

var binPath stringStore

// SetPath sets the path to wkhtmltopdf
func SetPath(path string) {
	binPath.Set(path)
}

// GetPath gets the path to wkhtmltopdf
func GetPath() string {
	return binPath.Get()
}

// Page is the input struct for each page
type Page struct {
	Input string
	PageOptions
}

// InputFile returns the input string and is part of the page interface
func (p *Page) InputFile() string {
	return p.Input
}

// Args returns the argument slice and is part of the page interface
func (p *Page) Args() []string {
	return p.PageOptions.Args()
}

// Reader returns the io.Reader and is part of the page interface
func (p *Page) Reader() io.Reader {
	return nil
}

// NewPage creates a new input page from a local or web resource (filepath or URL)
func NewPage(input string) *Page {
	return &Page{
		Input:       input,
		PageOptions: NewPageOptions(),
	}
}

// PageReader is one input page (a HTML document) that is read from an io.Reader
// You can add only one Page from a reader
type PageReader struct {
	Input io.Reader
	PageOptions
}

// InputFile returns the input string and is part of the page interface
func (pr *PageReader) InputFile() string {
	return "-"
}

// Args returns the argument slice and is part of the page interface
func (pr *PageReader) Args() []string {
	return pr.PageOptions.Args()
}

//Reader returns the io.Reader and is part of the page interface
func (pr *PageReader) Reader() io.Reader {
	return pr.Input
}

// NewPageReader creates a new PageReader from an io.Reader
func NewPageReader(input io.Reader) *PageReader {
	return &PageReader{
		Input:       input,
		PageOptions: NewPageOptions(),
	}
}

type page interface {
	Args() []string
	InputFile() string
	Reader() io.Reader
}

// PageOptions are options for each input page
type PageOptions struct {
	pageOptions
	headerAndFooterOptions
}

// Args returns the argument slice
func (po *PageOptions) Args() []string {
	return append(append([]string{}, po.pageOptions.Args()...), po.headerAndFooterOptions.Args()...)
}

// NewPageOptions returns a new PageOptions struct with all options
func NewPageOptions() PageOptions {
	return PageOptions{
		pageOptions:            newPageOptions(),
		headerAndFooterOptions: newHeaderAndFooterOptions(),
	}
}

// cover page
type cover struct {
	Input string
	pageOptions
}

// table of contents
type toc struct {
	Include bool
	allTocOptions
}

type allTocOptions struct {
	pageOptions
	tocOptions
}

// PDFGenerator is the main wkhtmltopdf struct, always use NewPDFGenerator to obtain a new PDFGenerator struct
type PDFGenerator struct {
	globalOptions
	outlineOptions

	Cover      cover
	TOC        toc
	OutputFile string //filename to write to, default empty (writes to internal buffer)

	binPath   string
	outbuf    bytes.Buffer
	outWriter io.Writer
	pages     []page
}

//Args returns the commandline arguments as a string slice
func (pdfg *PDFGenerator) Args() []string {
	args := append([]string{}, pdfg.globalOptions.Args()...)
	args = append(args, pdfg.outlineOptions.Args()...)
	if pdfg.Cover.Input != "" {
		args = append(args, "cover")
		args = append(args, pdfg.Cover.Input)
		args = append(args, pdfg.Cover.pageOptions.Args()...)
	}
	if pdfg.TOC.Include {
		args = append(args, "toc")
		args = append(args, pdfg.TOC.pageOptions.Args()...)
		args = append(args, pdfg.TOC.tocOptions.Args()...)
	}
	for _, page := range pdfg.pages {
		args = append(args, "page")
		args = append(args, page.InputFile())
		args = append(args, page.Args()...)
	}
	if pdfg.OutputFile != "" {
		args = append(args, pdfg.OutputFile)
	} else {
		args = append(args, "-")
	}
	return args
}

// ArgString returns Args as a single string
func (pdfg *PDFGenerator) ArgString() string {
	return strings.Join(pdfg.Args(), " ")
}

// AddPage adds a new input page to the document.
// A page is an input HTML page, it can span multiple pages in the output document.
// It is a Page when read from file or URL or a PageReader when read from memory.
func (pdfg *PDFGenerator) AddPage(p page) {
	pdfg.pages = append(pdfg.pages, p)
}

// SetPages resets all pages
func (pdfg *PDFGenerator) SetPages(p []page) {
	pdfg.pages = p
}

// ResetPages drops all pages previously added by AddPage or SetPages.
// This allows reuse of current instance of PDFGenerator with all of it's configuration preserved.
func (pdfg *PDFGenerator) ResetPages() {
	pdfg.pages = []page{}
}

// Buffer returns the embedded output buffer used if OutputFile is empty
func (pdfg *PDFGenerator) Buffer() *bytes.Buffer {
	return &pdfg.outbuf
}

// Bytes returns the output byte slice from the output buffer used if OutputFile is empty
func (pdfg *PDFGenerator) Bytes() []byte {
	return pdfg.outbuf.Bytes()
}

// SetOutput sets the output to write the PDF to, when this method is called, the internal buffer will not be used,
// so the Bytes(), Buffer() and WriteFile() methods will not work.
func (pdfg *PDFGenerator) SetOutput(w io.Writer) {
	pdfg.outWriter = w
}

// WriteFile writes the contents of the output buffer to a file
func (pdfg *PDFGenerator) WriteFile(filename string) error {
	return ioutil.WriteFile(filename, pdfg.Bytes(), 0666)
}

//findPath finds the path to wkhtmltopdf by
//- first looking in the current dir
//- looking in the PATH and PATHEXT environment dirs
//- using the WKHTMLTOPDF_PATH environment dir
//The path is cached, meaning you can not change the location of wkhtmltopdf in
//a running program once it has been found
func (pdfg *PDFGenerator) findPath() error {
	const exe = "wkhtmltopdf"
	pdfg.binPath = GetPath()
	if pdfg.binPath != "" {
		// wkhtmltopdf has already already found, return
		return nil
	}
	exeDir, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err != nil {
		return err
	}
	path, err := exec.LookPath(filepath.Join(exeDir, exe))
	if err == nil && path != "" {
		binPath.Set(path)
		pdfg.binPath = path
		return nil
	}
	path, err = exec.LookPath(exe)
	if err == nil && path != "" {
		binPath.Set(path)
		pdfg.binPath = path
		return nil
	}
	dir := os.Getenv("WKHTMLTOPDF_PATH")
	if dir == "" {
		return fmt.Errorf("%s not found", exe)
	}
	path, err = exec.LookPath(filepath.Join(dir, exe))
	if err == nil && path != "" {
		binPath.Set(path)
		pdfg.binPath = path
		return nil
	}
	return fmt.Errorf("%s not found", exe)
}

// Create creates the PDF document and stores it in the internal buffer if no error is returned
func (pdfg *PDFGenerator) Create() error {
	return pdfg.run()
}

func (pdfg *PDFGenerator) run() error {

	errbuf := &bytes.Buffer{}

	cmd := exec.Command(pdfg.binPath, pdfg.Args()...)
	cmd.Stderr = errbuf

	// set output to the desired writer or the internal buffer
	if pdfg.outWriter != nil {
		cmd.Stdout = pdfg.outWriter
	} else {
		cmd.Stdout = &pdfg.outbuf
	}

	// if there is a pageReader page (from Stdin) we set Stdin to that reader
	for _, page := range pdfg.pages {
		if page.Reader() != nil {
			cmd.Stdin = page.Reader()
			break
		}
	}

	err := cmd.Run()
	if err != nil {
		errStr := errbuf.String()
		if strings.TrimSpace(errStr) == "" {
			errStr = err.Error()
		}
		return errors.New(errStr)
	}
	return nil
}

// NewPDFGenerator returns a new PDFGenerator struct with all options created and
// checks if wkhtmltopdf can be found on the system
func NewPDFGenerator() (*PDFGenerator, error) {
	pdfg := NewPDFPreparer()
	err := pdfg.findPath()
	return pdfg, err
}

// NewPDFPreparer returns a PDFGenerator object without looking for the wkhtmltopdf executable file.
// This is useful to prepare a PDF file that is generated elsewhere and you just want to save the config as JSON.
// Note that Create() can not be called on this object unless you call SetPath yourself.
func NewPDFPreparer() *PDFGenerator {
	return &PDFGenerator{
		globalOptions:  newGlobalOptions(),
		outlineOptions: newOutlineOptions(),
		Cover: cover{
			pageOptions: newPageOptions(),
		},
		TOC: toc{
			allTocOptions: allTocOptions{
				tocOptions:  newTocOptions(),
				pageOptions: newPageOptions(),
			},
		},
	}
}
<commit_msg>minor code cleanup<commit_after>// Package wkhtmltopdf contains wrappers around the wkhtmltopdf commandline tool
package wkhtmltopdf

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
)

//the cached mutexed path as used by findPath()
type stringStore struct {
	val string
	sync.Mutex
}

func (ss *stringStore) Get() string {
	ss.Lock()
	defer ss.Unlock()
	return ss.val
}

func (ss *stringStore) Set(s string) {
	ss.Lock()
	ss.val = s
	ss.Unlock()
}

var binPath stringStore

// SetPath sets the path to wkhtmltopdf
func SetPath(path string) {
	binPath.Set(path)
}

// GetPath gets the path to wkhtmltopdf
func GetPath() string {
	return binPath.Get()
}

// Page is the input struct for each page
type Page struct {
	Input string
	PageOptions
}

// InputFile returns the input string and is part of the page interface
func (p *Page) InputFile() string {
	return p.Input
}

// Args returns the argument slice and is part of the page interface
func (p *Page) Args() []string {
	return p.PageOptions.Args()
}

// Reader returns the io.Reader and is part of the page interface
func (p *Page) Reader() io.Reader {
	return nil
}

// NewPage creates a new input page from a local or web resource (filepath or URL)
func NewPage(input string) *Page {
	return &Page{
		Input:       input,
		PageOptions: NewPageOptions(),
	}
}

// PageReader is one input page (a HTML document) that is read from an io.Reader
// You can add only one Page from a reader
type PageReader struct {
	Input io.Reader
	PageOptions
}

// InputFile returns the input string and is part of the page interface
func (pr *PageReader) InputFile() string {
	return "-"
}

// Args returns the argument slice and is part of the page interface
func (pr *PageReader) Args() []string {
	return pr.PageOptions.Args()
}

//Reader returns the io.Reader and is part of the page interface
func (pr *PageReader) Reader() io.Reader {
	return pr.Input
}

// NewPageReader creates a new PageReader from an io.Reader
func NewPageReader(input io.Reader) *PageReader {
	return &PageReader{
		Input:       input,
		PageOptions: NewPageOptions(),
	}
}

type page interface {
	Args() []string
	InputFile() string
	Reader() io.Reader
}

// PageOptions are options for each input page
type PageOptions struct {
	pageOptions
	headerAndFooterOptions
}

// Args returns the argument slice
func (po *PageOptions) Args() []string {
	return append(append([]string{}, po.pageOptions.Args()...), po.headerAndFooterOptions.Args()...)
}

// NewPageOptions returns a new PageOptions struct with all options
func NewPageOptions() PageOptions {
	return PageOptions{
		pageOptions:            newPageOptions(),
		headerAndFooterOptions: newHeaderAndFooterOptions(),
	}
}

// cover page
type cover struct {
	Input string
	pageOptions
}

// table of contents
type toc struct {
	Include bool
	allTocOptions
}

type allTocOptions struct {
	pageOptions
	tocOptions
}

// PDFGenerator is the main wkhtmltopdf struct, always use NewPDFGenerator to obtain a new PDFGenerator struct
type PDFGenerator struct {
	globalOptions
	outlineOptions

	Cover      cover
	TOC        toc
	OutputFile string //filename to write to, default empty (writes to internal buffer)

	binPath   string
	outbuf    bytes.Buffer
	outWriter io.Writer
	pages     []page
}

//Args returns the commandline arguments as a string slice
func (pdfg *PDFGenerator) Args() []string {
	args := append([]string{}, pdfg.globalOptions.Args()...)
	args = append(args, pdfg.outlineOptions.Args()...)
	if pdfg.Cover.Input != "" {
		args = append(args, "cover")
		args = append(args, pdfg.Cover.Input)
		args = append(args, pdfg.Cover.pageOptions.Args()...)
	}
	if pdfg.TOC.Include {
		args = append(args, "toc")
		args = append(args, pdfg.TOC.pageOptions.Args()...)
		args = append(args, pdfg.TOC.tocOptions.Args()...)
	}
	for _, page := range pdfg.pages {
		args = append(args, "page")
		args = append(args, page.InputFile())
		args = append(args, page.Args()...)
	}
	if pdfg.OutputFile != "" {
		args = append(args, pdfg.OutputFile)
	} else {
		args = append(args, "-")
	}
	return args
}

// ArgString returns Args as a single string
func (pdfg *PDFGenerator) ArgString() string {
	return strings.Join(pdfg.Args(), " ")
}

// AddPage adds a new input page to the document.
// A page is an input HTML page, it can span multiple pages in the output document.
// It is a Page when read from file or URL or a PageReader when read from memory.
func (pdfg *PDFGenerator) AddPage(p page) {
	pdfg.pages = append(pdfg.pages, p)
}

// SetPages resets all pages
func (pdfg *PDFGenerator) SetPages(p []page) {
	pdfg.pages = p
}

// ResetPages drops all pages previously added by AddPage or SetPages.
// This allows reuse of current instance of PDFGenerator with all of it's configuration preserved.
func (pdfg *PDFGenerator) ResetPages() {
	pdfg.pages = []page{}
}

// Buffer returns the embedded output buffer used if OutputFile is empty
func (pdfg *PDFGenerator) Buffer() *bytes.Buffer {
	return &pdfg.outbuf
}

// Bytes returns the output byte slice from the output buffer used if OutputFile is empty
func (pdfg *PDFGenerator) Bytes() []byte {
	return pdfg.outbuf.Bytes()
}

// SetOutput sets the output to write the PDF to, when this method is called, the internal buffer will not be used,
// so the Bytes(), Buffer() and WriteFile() methods will not work.
func (pdfg *PDFGenerator) SetOutput(w io.Writer) {
	pdfg.outWriter = w
}

// WriteFile writes the contents of the output buffer to a file
func (pdfg *PDFGenerator) WriteFile(filename string) error {
	return ioutil.WriteFile(filename, pdfg.Bytes(), 0666)
}

//findPath finds the path to wkhtmltopdf by
//- first looking in the current dir
//- looking in the PATH and PATHEXT environment dirs
//- using the WKHTMLTOPDF_PATH environment dir
//The path is cached, meaning you can not change the location of wkhtmltopdf in
//a running program once it has been found
func (pdfg *PDFGenerator) findPath() error {
	const exe = "wkhtmltopdf"
	pdfg.binPath = GetPath()
	if pdfg.binPath != "" {
		// wkhtmltopdf has already already found, return
		return nil
	}
	exeDir, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err != nil {
		return err
	}
	path, err := exec.LookPath(filepath.Join(exeDir, exe))
	if err == nil && path != "" {
		binPath.Set(path)
		pdfg.binPath = path
		return nil
	}
	path, err = exec.LookPath(exe)
	if err == nil && path != "" {
		binPath.Set(path)
		pdfg.binPath = path
		return nil
	}
	dir := os.Getenv("WKHTMLTOPDF_PATH")
	if dir == "" {
		return fmt.Errorf("%s not found", exe)
	}
	path, err = exec.LookPath(filepath.Join(dir, exe))
	if err == nil && path != "" {
		binPath.Set(path)
		pdfg.binPath = path
		return nil
	}
	return fmt.Errorf("%s not found", exe)
}

// Create creates the PDF document and stores it in the internal buffer if no error is returned
func (pdfg *PDFGenerator) Create() error {
	return pdfg.run()
}

func (pdfg *PDFGenerator) run() error {
	errbuf := &bytes.Buffer{}

	cmd := exec.Command(pdfg.binPath, pdfg.Args()...)
	cmd.Stderr = errbuf

	// set output to the desired writer or the internal buffer
	if pdfg.outWriter != nil {
		cmd.Stdout = pdfg.outWriter
	} else {
		cmd.Stdout = &pdfg.outbuf
	}

	// if there is a pageReader page (from Stdin) we set Stdin to that reader
	for _, page := range pdfg.pages {
		if page.Reader() != nil {
			cmd.Stdin = page.Reader()
			break
		}
	}

	err := cmd.Run()
	if err != nil {
		errStr := errbuf.String()
		if strings.TrimSpace(errStr) == "" {
			errStr = err.Error()
		}
		return errors.New(errStr)
	}
	return nil
}

// NewPDFGenerator returns a new PDFGenerator struct with all options created and
// checks if wkhtmltopdf can be found on the system
func NewPDFGenerator() (*PDFGenerator, error) {
	pdfg := NewPDFPreparer()
	return pdfg, pdfg.findPath()
}

// NewPDFPreparer returns a PDFGenerator object without looking for the wkhtmltopdf executable file.
// This is useful to prepare a PDF file that is generated elsewhere and you just want to save the config as JSON.
// Note that Create() can not be called on this object unless you call SetPath yourself.
func NewPDFPreparer() *PDFGenerator {
	return &PDFGenerator{
		globalOptions:  newGlobalOptions(),
		outlineOptions: newOutlineOptions(),
		Cover: cover{
			pageOptions: newPageOptions(),
		},
		TOC: toc{
			allTocOptions: allTocOptions{
				tocOptions:  newTocOptions(),
				pageOptions: newPageOptions(),
			},
		},
	}
}
<|endoftext|>


<commit_before>package sub

import (
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/objectcache"
	"github.com/Symantec/Dominator/lib/triggers"
	"github.com/Symantec/Dominator/proto/common"
	"time"
)

type Configuration struct {
	ScanSpeedPercent    uint
	NetworkSpeedPercent uint
	ScanExclusionList   []string
}

type FetchRequest struct {
	ServerAddress string
	Hashes        []hash.Hash
}

type FetchResponse common.StatusResponse

type GetConfigurationRequest struct {
}

type GetConfigurationResponse Configuration

type PollRequest struct {
	HaveGeneration uint64
}

type PollResponse struct {
	NetworkSpeed                 uint64
	FetchInProgress              bool // Fetch() and Update() mutually exclusive
	UpdateInProgress             bool
	LastFetchError               error
	LastUpdateError              error
	LastUpdateHadTriggerFailures bool
	StartTime                    time.Time
	PollTime                     time.Time
	GenerationCount              uint64
	FileSystem                   *filesystem.FileSystem // Streamed separately.
	FileSystemFollows            bool
	ObjectCache                  objectcache.ObjectCache // Streamed separately.
} // FileSystem is encoded afterwards, followed by ObjectCache.

type SetConfigurationRequest Configuration

type SetConfigurationResponse common.StatusResponse

type FileToCopyToCache struct {
	Name string
	Hash hash.Hash
}

type Hardlink struct {
	NewLink string
	Target  string
}

type Inode struct {
	Name string
	filesystem.GenericInode
}

type UpdateRequest struct {
	// The ordering here reflects the ordering that the sub is expected to use.
	FilesToCopyToCache  []FileToCopyToCache
	DirectoriesToMake   []Inode
	InodesToMake        []Inode
	HardlinksToMake     []Hardlink
	PathsToDelete       []string
	InodesToChange      []Inode
	MultiplyUsedObjects map[hash.Hash]uint64
	Triggers            *triggers.Triggers
}

type UpdateResponse struct{}

type CleanupRequest struct {
	Hashes []hash.Hash
}

type CleanupResponse struct{}
<commit_msg>Defined the Sub.GetFiles() RPC protocol.<commit_after>package sub

import (
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/objectcache"
	"github.com/Symantec/Dominator/lib/triggers"
	"github.com/Symantec/Dominator/proto/common"
	"time"
)

type Configuration struct {
	ScanSpeedPercent    uint
	NetworkSpeedPercent uint
	ScanExclusionList   []string
}

type FetchRequest struct {
	ServerAddress string
	Hashes        []hash.Hash
}

type FetchResponse common.StatusResponse

type GetConfigurationRequest struct {
}

type GetConfigurationResponse Configuration

// The GetFiles() RPC is fully streamed.
// The client sends a stream of strings (filenames) it wants. An empty string
// signals the end of the stream.
// The server (the sub) sends a stream of GetFileResponse messages. No response
// is sent for the end-of-stream signal.

type GetFileResponse struct {
	Error error
	Size  uint64
}

type PollRequest struct {
	HaveGeneration uint64
}

type PollResponse struct {
	NetworkSpeed                 uint64
	FetchInProgress              bool // Fetch() and Update() mutually exclusive
	UpdateInProgress             bool
	LastFetchError               error
	LastUpdateError              error
	LastUpdateHadTriggerFailures bool
	StartTime                    time.Time
	PollTime                     time.Time
	GenerationCount              uint64
	FileSystem                   *filesystem.FileSystem // Streamed separately.
	FileSystemFollows            bool
	ObjectCache                  objectcache.ObjectCache // Streamed separately.
} // FileSystem is encoded afterwards, followed by ObjectCache.

type SetConfigurationRequest Configuration

type SetConfigurationResponse common.StatusResponse

type FileToCopyToCache struct {
	Name string
	Hash hash.Hash
}

type Hardlink struct {
	NewLink string
	Target  string
}

type Inode struct {
	Name string
	filesystem.GenericInode
}

type UpdateRequest struct {
	// The ordering here reflects the ordering that the sub is expected to use.
	FilesToCopyToCache  []FileToCopyToCache
	DirectoriesToMake   []Inode
	InodesToMake        []Inode
	HardlinksToMake     []Hardlink
	PathsToDelete       []string
	InodesToChange      []Inode
	MultiplyUsedObjects map[hash.Hash]uint64
	Triggers            *triggers.Triggers
}

type UpdateResponse struct{}

type CleanupRequest struct {
	Hashes []hash.Hash
}

type CleanupResponse struct{}
<|endoftext|>


<commit_before>package keybase1

import (
	"encoding/base64"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"strings"
	"time"

	jsonw "github.com/keybase/go-jsonw"
)

const (
	UID_LEN          = 16
	UID_SUFFIX       = 0x00
	UID_SUFFIX_2     = 0x19
	UID_SUFFIX_HEX   = "00"
	UID_SUFFIX_2_HEX = "19"
	PUBLIC_UID       = "ffffffffffffffffffffffffffffff00"
)

// UID for the special "public" user.
var PublicUID = UID(PUBLIC_UID)

const (
	SIG_ID_LEN         = 32
	SIG_ID_SUFFIX      = 0x0f
	SIG_SHORT_ID_BYTES = 27
)

const (
	DeviceIDLen       = 16
	DeviceIDSuffix    = 0x18
	DeviceIDSuffixHex = "18"
)

func Unquote(data []byte) string {
	return strings.Trim(string(data), "\"")
}

func Quote(s string) []byte {
	return []byte("\"" + s + "\"")
}

func KIDFromSlice(b []byte) KID {
	return KID(hex.EncodeToString(b))
}

func KIDFromString(s string) KID {
	// there are no validations for KIDs (length, suffixes)
	return KID(s)
}

func (k KID) IsValid() bool {
	return len(k) > 0
}

func (k KID) String() string {
	return string(k)
}

func (k KID) IsNil() bool {
	return len(k) == 0
}

func (k KID) Exists() bool {
	return !k.IsNil()
}

func (k KID) Equal(v KID) bool {
	return k == v
}

func (k KID) NotEqual(v KID) bool {
	return !k.Equal(v)
}

func (k KID) Match(q string, exact bool) bool {
	if k.IsNil() {
		return false
	}

	if exact {
		return strings.ToLower(k.String()) == strings.ToLower(q)
	}

	if strings.HasPrefix(k.String(), strings.ToLower(q)) {
		return true
	}
	if strings.HasPrefix(k.ToShortIDString(), q) {
		return true
	}
	return false
}

func (k KID) ToBytes() []byte {
	b, err := hex.DecodeString(string(k))
	if err != nil {
		return nil
	}
	return b
}

func (k KID) ToShortIDString() string {
	return encode(k.ToBytes()[0:12])
}

func (k KID) ToJsonw() *jsonw.Wrapper {
	if k.IsNil() {
		return jsonw.NewNil()
	}
	return jsonw.NewString(string(k))
}

func DeviceIDFromBytes(b [DeviceIDLen]byte) DeviceID {
	return DeviceID(hex.EncodeToString(b[:]))
}

func DeviceIDFromSlice(b []byte) (DeviceID, error) {
	if len(b) != DeviceIDLen {
		return "", fmt.Errorf("invalid byte slice for DeviceID: len == %d, expected %d", len(b), DeviceIDLen)
	}
	var x [DeviceIDLen]byte
	copy(x[:], b)
	return DeviceIDFromBytes(x), nil
}

func DeviceIDFromString(s string) (DeviceID, error) {
	if len(s) != hex.EncodedLen(DeviceIDLen) {
		return "", fmt.Errorf("Bad Device ID length: %d", len(s))
	}
	suffix := s[len(s)-2:]
	if suffix != DeviceIDSuffixHex {
		return "", fmt.Errorf("Bad suffix byte: %s", suffix)
	}
	return DeviceID(s), nil
}

func (d DeviceID) String() string {
	return string(d)
}

func (d DeviceID) IsNil() bool {
	return len(d) == 0
}

func (d DeviceID) Exists() bool {
	return !d.IsNil()
}

func UIDFromString(s string) (UID, error) {
	if len(s) != hex.EncodedLen(UID_LEN) {
		return "", fmt.Errorf("Bad UID '%s'; must be %d bytes long", s, UID_LEN)
	}
	suffix := s[len(s)-2:]
	if suffix != UID_SUFFIX_HEX && suffix != UID_SUFFIX_2_HEX {
		return "", fmt.Errorf("Bad UID '%s': must end in 0x%x or 0x%x", s, UID_SUFFIX, UID_SUFFIX_2)
	}
	return UID(s), nil
}

// Used by unit tests.
func MakeTestUID(n uint32) UID {
	b := make([]byte, 8)
	binary.LittleEndian.PutUint32(b, n)
	s := hex.EncodeToString(b)
	c := 2*UID_LEN - len(UID_SUFFIX_HEX) - len(s)
	s += strings.Repeat("0", c) + UID_SUFFIX_HEX
	uid, err := UIDFromString(s)
	if err != nil {
		panic(err)
	}
	return uid
}

func (u UID) String() string {
	return string(u)
}

func (u UID) IsNil() bool {
	return len(u) == 0
}

func (u UID) Exists() bool {
	return !u.IsNil()
}

func (u UID) Equal(v UID) bool {
	return u == v
}

func (u UID) NotEqual(v UID) bool {
	return !u.Equal(v)
}

func (u UID) Less(v UID) bool {
	return u < v
}

// Returns a number in [0, shardCount) which can be treated as roughly
// uniformly distributed. Used for things that need to shard by user.
func (u UID) GetShard(shardCount int) (int, error) {
	bytes, err := hex.DecodeString(string(u))
	if err != nil {
		return 0, err
	}
	n := binary.LittleEndian.Uint32(bytes)
	return int(n) % shardCount, nil
}

func (s SigID) IsNil() bool {
	return len(s) == 0
}

func (s SigID) Exists() bool {
	return !s.IsNil()
}

func (s SigID) Equal(t SigID) bool {
	return s == t
}

func (s SigID) NotEqual(t SigID) bool {
	return !s.Equal(t)
}

func (s SigID) ToDisplayString(verbose bool) string {
	if verbose {
		return string(s)
	}
	return fmt.Sprintf("%s...", s[0:6])
}

func (s SigID) ToString(suffix bool) string {
	if len(s) == 0 {
		return ""
	}
	if suffix {
		return string(s)
	}
	return string(s[0 : len(s)-2])
}

func SigIDFromString(s string, suffix bool) (SigID, error) {
	blen := SIG_ID_LEN
	if suffix {
		blen++
	}
	if len(s) != hex.EncodedLen(blen) {
		return "", fmt.Errorf("Invalid SigID string length: %d, expected %d (suffix = %v)", len(s), hex.EncodedLen(blen), suffix)
	}
	if suffix {
		return SigID(s), nil
	}
	return SigID(fmt.Sprintf("%s%02x", s, SIG_ID_SUFFIX)), nil
}

func SigIDFromBytes(b [SIG_ID_LEN]byte) SigID {
	s := hex.EncodeToString(b[:])
	return SigID(fmt.Sprintf("%s%02x", s, SIG_ID_SUFFIX))
}

func SigIDFromSlice(b []byte) (SigID, error) {
	if len(b) != SIG_ID_LEN {
		return "", fmt.Errorf("invalid byte slice for SigID: len == %d, expected %d", len(b), SIG_ID_LEN)
	}
	var x [SIG_ID_LEN]byte
	copy(x[:], b)
	return SigIDFromBytes(x), nil
}

func (s SigID) toBytes() []byte {
	b, err := hex.DecodeString(string(s))
	if err != nil {
		return nil
	}
	return b[0:SIG_ID_LEN]
}

func (s SigID) ToMediumID() string {
	return encode(s.toBytes())
}

func (s SigID) ToShortID() string {
	return encode(s.toBytes()[0:SIG_SHORT_ID_BYTES])
}

func encode(b []byte) string {
	return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=")
}

func FromTime(t Time) time.Time {
	return time.Unix(0, int64(t)*1000000)
}

func ToTime(t time.Time) Time {
	return Time(t.UnixNano() / 1000000)
}

func TimeFromSeconds(seconds int64) Time {
	return Time(seconds * 1000)
}

func FormatTime(t Time) string {
	layout := "2006-01-02 15:04:05 MST"
	return FromTime(t).Format(layout)
}
<commit_msg>protocol: don't cast a uint32 to an int and expect a positive num<commit_after>package keybase1

import (
	"encoding/base64"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"strings"
	"time"

	jsonw "github.com/keybase/go-jsonw"
)

const (
	UID_LEN          = 16
	UID_SUFFIX       = 0x00
	UID_SUFFIX_2     = 0x19
	UID_SUFFIX_HEX   = "00"
	UID_SUFFIX_2_HEX = "19"
	PUBLIC_UID       = "ffffffffffffffffffffffffffffff00"
)

// UID for the special "public" user.
var PublicUID = UID(PUBLIC_UID)

const (
	SIG_ID_LEN         = 32
	SIG_ID_SUFFIX      = 0x0f
	SIG_SHORT_ID_BYTES = 27
)

const (
	DeviceIDLen       = 16
	DeviceIDSuffix    = 0x18
	DeviceIDSuffixHex = "18"
)

func Unquote(data []byte) string {
	return strings.Trim(string(data), "\"")
}

func Quote(s string) []byte {
	return []byte("\"" + s + "\"")
}

func KIDFromSlice(b []byte) KID {
	return KID(hex.EncodeToString(b))
}

func KIDFromString(s string) KID {
	// there are no validations for KIDs (length, suffixes)
	return KID(s)
}

func (k KID) IsValid() bool {
	return len(k) > 0
}

func (k KID) String() string {
	return string(k)
}

func (k KID) IsNil() bool {
	return len(k) == 0
}

func (k KID) Exists() bool {
	return !k.IsNil()
}

func (k KID) Equal(v KID) bool {
	return k == v
}

func (k KID) NotEqual(v KID) bool {
	return !k.Equal(v)
}

func (k KID) Match(q string, exact bool) bool {
	if k.IsNil() {
		return false
	}

	if exact {
		return strings.ToLower(k.String()) == strings.ToLower(q)
	}

	if strings.HasPrefix(k.String(), strings.ToLower(q)) {
		return true
	}
	if strings.HasPrefix(k.ToShortIDString(), q) {
		return true
	}
	return false
}

func (k KID) ToBytes() []byte {
	b, err := hex.DecodeString(string(k))
	if err != nil {
		return nil
	}
	return b
}

func (k KID) ToShortIDString() string {
	return encode(k.ToBytes()[0:12])
}

func (k KID) ToJsonw() *jsonw.Wrapper {
	if k.IsNil() {
		return jsonw.NewNil()
	}
	return jsonw.NewString(string(k))
}

func DeviceIDFromBytes(b [DeviceIDLen]byte) DeviceID {
	return DeviceID(hex.EncodeToString(b[:]))
}

func DeviceIDFromSlice(b []byte) (DeviceID, error) {
	if len(b) != DeviceIDLen {
		return "", fmt.Errorf("invalid byte slice for DeviceID: len == %d, expected %d", len(b), DeviceIDLen)
	}
	var x [DeviceIDLen]byte
	copy(x[:], b)
	return DeviceIDFromBytes(x), nil
}

func DeviceIDFromString(s string) (DeviceID, error) {
	if len(s) != hex.EncodedLen(DeviceIDLen) {
		return "", fmt.Errorf("Bad Device ID length: %d", len(s))
	}
	suffix := s[len(s)-2:]
	if suffix != DeviceIDSuffixHex {
		return "", fmt.Errorf("Bad suffix byte: %s", suffix)
	}
	return DeviceID(s), nil
}

func (d DeviceID) String() string {
	return string(d)
}

func (d DeviceID) IsNil() bool {
	return len(d) == 0
}

func (d DeviceID) Exists() bool {
	return !d.IsNil()
}

func UIDFromString(s string) (UID, error) {
	if len(s) != hex.EncodedLen(UID_LEN) {
		return "", fmt.Errorf("Bad UID '%s'; must be %d bytes long", s, UID_LEN)
	}
	suffix := s[len(s)-2:]
	if suffix != UID_SUFFIX_HEX && suffix != UID_SUFFIX_2_HEX {
		return "", fmt.Errorf("Bad UID '%s': must end in 0x%x or 0x%x", s, UID_SUFFIX, UID_SUFFIX_2)
	}
	return UID(s), nil
}

// Used by unit tests.
func MakeTestUID(n uint32) UID {
	b := make([]byte, 8)
	binary.LittleEndian.PutUint32(b, n)
	s := hex.EncodeToString(b)
	c := 2*UID_LEN - len(UID_SUFFIX_HEX) - len(s)
	s += strings.Repeat("0", c) + UID_SUFFIX_HEX
	uid, err := UIDFromString(s)
	if err != nil {
		panic(err)
	}
	return uid
}

func (u UID) String() string {
	return string(u)
}

func (u UID) IsNil() bool {
	return len(u) == 0
}

func (u UID) Exists() bool {
	return !u.IsNil()
}

func (u UID) Equal(v UID) bool {
	return u == v
}

func (u UID) NotEqual(v UID) bool {
	return !u.Equal(v)
}

func (u UID) Less(v UID) bool {
	return u < v
}

// Returns a number in [0, shardCount) which can be treated as roughly
// uniformly distributed. Used for things that need to shard by user.
func (u UID) GetShard(shardCount int) (int, error) {
	bytes, err := hex.DecodeString(string(u))
	if err != nil {
		return 0, err
	}
	n := binary.LittleEndian.Uint32(bytes)
	return int(n % uint32(shardCount)), nil
}

func (s SigID) IsNil() bool {
	return len(s) == 0
}

func (s SigID) Exists() bool {
	return !s.IsNil()
}

func (s SigID) Equal(t SigID) bool {
	return s == t
}

func (s SigID) NotEqual(t SigID) bool {
	return !s.Equal(t)
}

func (s SigID) ToDisplayString(verbose bool) string {
	if verbose {
		return string(s)
	}
	return fmt.Sprintf("%s...", s[0:6])
}

func (s SigID) ToString(suffix bool) string {
	if len(s) == 0 {
		return ""
	}
	if suffix {
		return string(s)
	}
	return string(s[0 : len(s)-2])
}

func SigIDFromString(s string, suffix bool) (SigID, error) {
	blen := SIG_ID_LEN
	if suffix {
		blen++
	}
	if len(s) != hex.EncodedLen(blen) {
		return "", fmt.Errorf("Invalid SigID string length: %d, expected %d (suffix = %v)", len(s), hex.EncodedLen(blen), suffix)
	}
	if suffix {
		return SigID(s), nil
	}
	return SigID(fmt.Sprintf("%s%02x", s, SIG_ID_SUFFIX)), nil
}

func SigIDFromBytes(b [SIG_ID_LEN]byte) SigID {
	s := hex.EncodeToString(b[:])
	return SigID(fmt.Sprintf("%s%02x", s, SIG_ID_SUFFIX))
}

func SigIDFromSlice(b []byte) (SigID, error) {
	if len(b) != SIG_ID_LEN {
		return "", fmt.Errorf("invalid byte slice for SigID: len == %d, expected %d", len(b), SIG_ID_LEN)
	}
	var x [SIG_ID_LEN]byte
	copy(x[:], b)
	return SigIDFromBytes(x), nil
}

func (s SigID) toBytes() []byte {
	b, err := hex.DecodeString(string(s))
	if err != nil {
		return nil
	}
	return b[0:SIG_ID_LEN]
}

func (s SigID) ToMediumID() string {
	return encode(s.toBytes())
}

func (s SigID) ToShortID() string {
	return encode(s.toBytes()[0:SIG_SHORT_ID_BYTES])
}

func encode(b []byte) string {
	return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=")
}

func FromTime(t Time) time.Time {
	return time.Unix(0, int64(t)*1000000)
}

func ToTime(t time.Time) Time {
	return Time(t.UnixNano() / 1000000)
}

func TimeFromSeconds(seconds int64) Time {
	return Time(seconds * 1000)
}

func FormatTime(t Time) string {
	layout := "2006-01-02 15:04:05 MST"
	return FromTime(t).Format(layout)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"strconv"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/lancetw/hcfd-forecast/db"
	"github.com/lancetw/hcfd-forecast/rain"
	"github.com/line/line-bot-sdk-go/linebot"
	"github.com/robfig/cron"
)

const timeZone = "Asia/Taipei"

var bot *linebot.Client

func main() {
	c := cron.New()
	c.AddFunc("0 */3 * * * *", GoProcess)
	c.Start()

	for {
		time.Sleep(10000000000000)
		fmt.Println("sleep")
	}
}

// GoProcess is main process
func GoProcess() {
	strID := os.Getenv("ChannelID")
	numID, err := strconv.ParseInt(strID, 10, 64)
	if err != nil {
		log.Fatal("Wrong environment setting about ChannelID")
	}
	bot, err = linebot.NewClient(numID, os.Getenv("ChannelSecret"), os.Getenv("MID"))
	if err != nil {
		log.Println("Bot:", bot, " err:", err)
	}

	log.Println("=== 查詢開始 === [")

	c := db.Connect(os.Getenv("REDISTOGO_URL"))

	targets0 := []string{"新竹市"}
	msgs0, token0 := rain.GetRainingInfo(targets0, false)

	if token0 != "" {
		status0, getErr := redis.Int(c.Do("SISMEMBER", "token0", token0))
		if getErr != nil {
			log.Println("SISMEMBER err", getErr)
		}

		if status0 == 0 {
			users0, smembersErr := redis.Strings(c.Do("SMEMBERS", "user"))

			if smembersErr != nil {
				log.Println("GetRainingInfo SMEMBERS redis error", smembersErr)
			} else {
				local := time.Now()
				location, timeZoneErr := time.LoadLocation(timeZone)
				if timeZoneErr == nil {
					local = local.In(location)
				}

				if len(msgs0) > 0 {
					var text string
					for _, msg := range msgs0 {
						text = text + msg + "\n\n"
					}
					for _, contentTo := range users0 {
						_, err = bot.SendText([]string{contentTo}, text)
						if err != nil {
							log.Println(err)
						}
					}
				}
			}
		}

		n0, addErr := c.Do("SADD", "token0", token0)
		if addErr != nil {
			log.Println("GetRainingInfo SADD to redis error", addErr, n0)
		}
	}

	targets1 := []string{"新竹市", "新竹縣"}
	msgs1, token1 := rain.GetWarningInfo(targets1)

	if token1 != "" {
		status1, getErr := redis.Int(c.Do("SISMEMBER", "token1", token1))
		if getErr != nil {
			log.Println("SISMEMBER err", getErr)
		}

		if status1 == 0 {
			users1, smembersErr := redis.Strings(c.Do("SMEMBERS", "user"))

			if smembersErr != nil {
				log.Println("GetWarningInfo SMEMBERS redis error", smembersErr)
			} else {
				local := time.Now()
				location, locationErr := time.LoadLocation(timeZone)
				if locationErr == nil {
					local = local.In(location)
				}
				for _, contentTo := range users1 {
					for _, msg := range msgs1 {
						_, msgErr := bot.SendText([]string{contentTo}, msg)
						if msgErr != nil {
							log.Println(err)
						}
					}
				}
			}
		}
	}

	if token1 != "" {
		n, addErr := c.Do("SADD", "token1", token1)
		if addErr != nil {
			log.Println("GetWarningInfo SADD to redis error", addErr, n)
		}
	}

	defer c.Close()

	log.Println("=== 查詢結束 === ]")
}
<commit_msg>Update main.go<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"strconv"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/lancetw/hcfd-forecast/db"
	"github.com/lancetw/hcfd-forecast/rain"
	"github.com/line/line-bot-sdk-go/linebot"
	"github.com/robfig/cron"
)

const timeZone = "Asia/Taipei"

var bot *linebot.Client

func main() {
	c := cron.New()
	c.AddFunc("0 */2 * * * *", GoProcess)
	c.Start()

	for {
		time.Sleep(10000000000000)
		fmt.Println("sleep")
	}
}

// GoProcess is main process
func GoProcess() {
	strID := os.Getenv("ChannelID")
	numID, err := strconv.ParseInt(strID, 10, 64)
	if err != nil {
		log.Fatal("Wrong environment setting about ChannelID")
	}
	bot, err = linebot.NewClient(numID, os.Getenv("ChannelSecret"), os.Getenv("MID"))
	if err != nil {
		log.Println("Bot:", bot, " err:", err)
	}

	log.Println("=== 查詢開始 === [")

	c := db.Connect(os.Getenv("REDISTOGO_URL"))

	targets0 := []string{"新竹市"}
	msgs0, token0 := rain.GetRainingInfo(targets0, false)

	if token0 != "" {
		status0, getErr := redis.Int(c.Do("SISMEMBER", "token0", token0))
		if getErr != nil {
			log.Println("SISMEMBER err", getErr)
		}

		if status0 == 0 {
			users0, smembersErr := redis.Strings(c.Do("SMEMBERS", "user"))

			if smembersErr != nil {
				log.Println("GetRainingInfo SMEMBERS redis error", smembersErr)
			} else {
				local := time.Now()
				location, timeZoneErr := time.LoadLocation(timeZone)
				if timeZoneErr == nil {
					local = local.In(location)
				}

				if len(msgs0) > 0 {
					var text string
					for _, msg := range msgs0 {
						text = text + msg + "\n\n"
					}
					for _, contentTo := range users0 {
						_, err = bot.SendText([]string{contentTo}, text)
						if err != nil {
							log.Println(err)
						}
					}
				}
			}
		}

		n0, addErr := c.Do("SADD", "token0", token0)
		if addErr != nil {
			log.Println("GetRainingInfo SADD to redis error", addErr, n0)
		}
	}

	targets1 := []string{"新竹市", "新竹縣"}
	msgs1, token1 := rain.GetWarningInfo(targets1)

	if token1 != "" {
		status1, getErr := redis.Int(c.Do("SISMEMBER", "token1", token1))
		if getErr != nil {
			log.Println("SISMEMBER err", getErr)
		}

		if status1 == 0 {
			users1, smembersErr := redis.Strings(c.Do("SMEMBERS", "user"))

			if smembersErr != nil {
				log.Println("GetWarningInfo SMEMBERS redis error", smembersErr)
			} else {
				local := time.Now()
				location, locationErr := time.LoadLocation(timeZone)
				if locationErr == nil {
					local = local.In(location)
				}
				for _, contentTo := range users1 {
					for _, msg := range msgs1 {
						_, msgErr := bot.SendText([]string{contentTo}, msg)
						if msgErr != nil {
							log.Println(err)
						}
					}
				}
			}
		}
	}

	if token1 != "" {
		n, addErr := c.Do("SADD", "token1", token1)
		if addErr != nil {
			log.Println("GetWarningInfo SADD to redis error", addErr, n)
		}
	}

	defer c.Close()

	log.Println("=== 查詢結束 === ]")
}
<|endoftext|>


<commit_before>package main

import (
	"container/heap"
	"flag"
	"github.com/bitly/go-simplejson"
	"github.com/bitly/nsq/nsq"
	"log"
	"strconv"
	"time"
)

var (
	topic            = flag.String("topic", "", "nsq topic")
	channel          = flag.String("channel", "", "nsq topic")
	maxInFlight      = flag.Int("max-in-flight", 1000, "max number of messages to allow in flight")
	nsqTCPAddrs      = flag.String("nsqd-tcp-address", "", "nsqd TCP address")
	nsqHTTPAddrs     = flag.String("nsqd-http-address", "", "nsqd HTTP address")
	lookupdHTTPAddrs = flag.String("lookupd-http-address", "", "lookupd HTTP address")
	lag_time         = flag.Int("lag", 10, "lag before emitting in seconds")
)

// MESSAGE HANDLER FOR THE NSQ READER
type MessageHandler struct {
	msgChan  chan *nsq.Message
	stopChan chan int
}

func (self *MessageHandler) HandleMessage(message *nsq.Message, responseChannel chan *nsq.FinishedMessage) {
	self.msgChan <- message
	responseChannel <- &nsq.FinishedMessage{message.Id, 0, true}
}

type WriteMessage struct {
	val          []byte
	t            time.Time
	responseChan chan bool
}

type PQMessage struct {
	val          []byte
	t            time.Time
	index        int
	killChan     chan bool
	responseChan chan bool
}

// PRIORITY QUEUE
// A PriorityQueue implements heap.Interface and holds Items.
type PriorityQueue []*PQMessage

func (pq PriorityQueue) Len() int {
	return len(pq)
}

func (pq PriorityQueue) Less(i, j int) bool {
	// We want Pop to give us the highest, not lowest, priority so we use greater than here.
	return pq[i].t.Before(pq[j].t)
}

func (pq PriorityQueue) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
	pq[i].index = i
	pq[j].index = j
}

func (pq *PriorityQueue) Push(x interface{}) {
	n := len(*pq)
	item := x.(*PQMessage)
	item.index = n
	*pq = append(*pq, item)
}

func (pq *PriorityQueue) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	item.index = -1 // for safety
	*pq = old[0 : n-1]
	return item
}

// update modifies the priority and value of an Item in the queue.
func (pq *PriorityQueue) update(item *PQMessage, val []byte, time time.Time) {
	heap.Remove(pq, item.index)
	item.val = val
	item.t = time
	heap.Push(pq, item)
}

func emit(msg *PQMessage, lag time.Duration) {
	emit_at := msg.t.Add(lag)
	log.Println("Sleeping")
	time.Sleep(emit_at.Sub(time.Now()))
	select {
	case <-msg.killChan:
		// do nowt
		log.Printf("killed")
	default:
		log.Printf(
			"### item's timestamp: %s. emitted at %s \n",
			msg.t.Format("15:04:05"),
			time.Now().Format("15:04:05"),
		)
	}
	msg.responseChan <- true
}

func store(writeChan chan WriteMessage, pq *PriorityQueue, lag time.Duration) {

	nextMsg := &PQMessage{
		responseChan: make(chan bool),
	}
	nextMsgChan := make(chan *PQMessage, 1)
	getNextMsg := true

	for {
		// if a message is ready, pop it and make it ready for emitting
		if pq.Len() > 0 && getNextMsg {
			nextMsg = heap.Pop(pq).(*PQMessage)
			nextMsgChan <- nextMsg
			// let's not get another message until this one is done
			getNextMsg = false
		}

		select {
		case inMsg := <-writeChan:
			// if we've recieved something in the write channel, push it onto the heap
			outMsg := &PQMessage{
				val:          inMsg.val,
				t:            inMsg.t,
				killChan:     make(chan bool, 1),
				responseChan: make(chan bool),
			}
			heap.Push(pq, outMsg)

			// check it didn't arrive before the current next message
			if nextMsg.val != nil {
				if outMsg.t.Before(nextMsg.t) {
					// if it did, kill the current emitter
					nextMsg.killChan <- true
					// put the old one back in the queue
					heap.Push(pq, nextMsg)
					// make sure the new message is loaded on the next iteration
					getNextMsg = true
				}
			}
			inMsg.responseChan <- true
		case outMsg := <-nextMsgChan:
			// if something is waitingto be emitted set it going
			go emit(outMsg, lag)
		case <-nextMsg.responseChan:
			// make sure the next message is loaded when the current nextMsg is done
			getNextMsg = true

		}
	}
}

// function to read an NSQ channel and write to the key value store
func writer(mh MessageHandler, writeChan chan WriteMessage) {
	for {
		select {
		case m := <-mh.msgChan:

			blob, err := simplejson.NewJson(m.Body)

			if err != nil {
				log.Fatalf(err.Error())
			}

			val, err := blob.Get("t").String()

			if err != nil {
				log.Fatalf(err.Error())
			}

			msg_time, err := strconv.ParseInt(val, 0, 64)

			if err != nil {
				log.Fatalf(err.Error())
			}

			t := time.Unix(msg_time, 0)
			mblob, err := blob.MarshalJSON()

			if err != nil {
				log.Fatalf(err.Error())
			}

			responseChan := make(chan bool)

			msg := WriteMessage{
				t:            t,
				val:          mblob,
				responseChan: responseChan,
			}

			writeChan <- msg

			success := <-responseChan

			if !success {
				// TODO learn about err.Error()
				log.Fatalf("its broken")
			}
		}
	}
}

func main() {

	flag.Parse()

	r, err := nsq.NewReader(*topic, *channel)

	if err != nil {
		log.Fatal(err.Error())
	}

	mh := MessageHandler{
		msgChan:  make(chan *nsq.Message, 5),
		stopChan: make(chan int),
	}

	wc := make(chan WriteMessage)

	pq := &PriorityQueue{}
	heap.Init(pq)

	lag := time.Duration(time.Duration(*lag_time) * time.Second)

	go store(wc, pq, lag)
	go writer(mh, wc)

	r.AddAsyncHandler(&mh)

	err = r.ConnectToNSQ(*nsqTCPAddrs)
	if err != nil {
		log.Fatalf(err.Error())
	}
	err = r.ConnectToLookupd(*lookupdHTTPAddrs)
	if err != nil {
		log.Fatalf(err.Error())
	}

	<-mh.stopChan
}
<commit_msg>->nanoseconds<commit_after>package main

import (
	"container/heap"
	"flag"
	"github.com/bitly/go-simplejson"
	"github.com/bitly/nsq/nsq"
	"log"
	"strconv"
	"time"
)

var (
	topic            = flag.String("topic", "", "nsq topic")
	channel          = flag.String("channel", "", "nsq topic")
	maxInFlight      = flag.Int("max-in-flight", 1000, "max number of messages to allow in flight")
	nsqTCPAddrs      = flag.String("nsqd-tcp-address", "", "nsqd TCP address")
	nsqHTTPAddrs     = flag.String("nsqd-http-address", "", "nsqd HTTP address")
	lookupdHTTPAddrs = flag.String("lookupd-http-address", "", "lookupd HTTP address")
	lag_time         = flag.Int("lag", 10, "lag before emitting in seconds")
)

// MESSAGE HANDLER FOR THE NSQ READER
type MessageHandler struct {
	msgChan  chan *nsq.Message
	stopChan chan int
}

func (self *MessageHandler) HandleMessage(message *nsq.Message, responseChannel chan *nsq.FinishedMessage) {
	self.msgChan <- message
	responseChannel <- &nsq.FinishedMessage{message.Id, 0, true}
}

type WriteMessage struct {
	val          []byte
	t            time.Time
	responseChan chan bool
}

type PQMessage struct {
	val          []byte
	t            time.Time
	index        int
	killChan     chan bool
	responseChan chan bool
}

// PRIORITY QUEUE
// A PriorityQueue implements heap.Interface and holds Items.
type PriorityQueue []*PQMessage

func (pq PriorityQueue) Len() int {
	return len(pq)
}

func (pq PriorityQueue) Less(i, j int) bool {
	// We want Pop to give us the highest, not lowest, priority so we use greater than here.
	return pq[i].t.Before(pq[j].t)
}

func (pq PriorityQueue) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
	pq[i].index = i
	pq[j].index = j
}

func (pq *PriorityQueue) Push(x interface{}) {
	n := len(*pq)
	item := x.(*PQMessage)
	item.index = n
	*pq = append(*pq, item)
}

func (pq *PriorityQueue) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	item.index = -1 // for safety
	*pq = old[0 : n-1]
	return item
}

// update modifies the priority and value of an Item in the queue.
func (pq *PriorityQueue) update(item *PQMessage, val []byte, time time.Time) {
	heap.Remove(pq, item.index)
	item.val = val
	item.t = time
	heap.Push(pq, item)
}

func emit(msg *PQMessage, lag time.Duration) {
	emit_at := msg.t.Add(lag)
	log.Println("Sleeping")
	time.Sleep(emit_at.Sub(time.Now()))
	select {
	case <-msg.killChan:
		// do nowt
		log.Printf("killed")
	default:
		log.Printf(
			"### item's timestamp: %s. emitted at %s \n",
			msg.t.Format("15:04:05"),
			time.Now().Format("15:04:05"),
		)
	}
	msg.responseChan <- true
}

func store(writeChan chan WriteMessage, pq *PriorityQueue, lag time.Duration) {

	nextMsg := &PQMessage{
		responseChan: make(chan bool),
	}
	nextMsgChan := make(chan *PQMessage, 1)
	getNextMsg := true

	for {
		// if a message is ready, pop it and make it ready for emitting
		if pq.Len() > 0 && getNextMsg {
			nextMsg = heap.Pop(pq).(*PQMessage)
			nextMsgChan <- nextMsg
			// let's not get another message until this one is done
			getNextMsg = false
		}

		select {
		case inMsg := <-writeChan:
			// if we've recieved something in the write channel, push it onto the heap
			outMsg := &PQMessage{
				val:          inMsg.val,
				t:            inMsg.t,
				killChan:     make(chan bool, 1),
				responseChan: make(chan bool),
			}
			heap.Push(pq, outMsg)

			// check it didn't arrive before the current next message
			if nextMsg.val != nil {
				if outMsg.t.Before(nextMsg.t) {
					// if it did, kill the current emitter
					nextMsg.killChan <- true
					// put the old one back in the queue
					heap.Push(pq, nextMsg)
					// make sure the new message is loaded on the next iteration
					getNextMsg = true
				}
			}
			inMsg.responseChan <- true
		case outMsg := <-nextMsgChan:
			// if something is waitingto be emitted set it going
			go emit(outMsg, lag)
		case <-nextMsg.responseChan:
			// make sure the next message is loaded when the current nextMsg is done
			getNextMsg = true

		}
	}
}

// function to read an NSQ channel and write to the key value store
func writer(mh MessageHandler, writeChan chan WriteMessage) {
	for {
		select {
		case m := <-mh.msgChan:

			blob, err := simplejson.NewJson(m.Body)

			if err != nil {
				log.Fatalf(err.Error())
			}

			val, err := blob.Get("t").String()

			if err != nil {
				log.Fatalf(err.Error())
			}

			msg_time, err := strconv.ParseInt(val, 0, 64)

			if err != nil {
				log.Fatalf(err.Error())
			}

			t := time.Unix(0, msg_time)
			mblob, err := blob.MarshalJSON()

			if err != nil {
				log.Fatalf(err.Error())
			}

			responseChan := make(chan bool)

			msg := WriteMessage{
				t:            t,
				val:          mblob,
				responseChan: responseChan,
			}

			writeChan <- msg

			success := <-responseChan

			if !success {
				// TODO learn about err.Error()
				log.Fatalf("its broken")
			}
		}
	}
}

func main() {

	flag.Parse()

	r, err := nsq.NewReader(*topic, *channel)

	if err != nil {
		log.Fatal(err.Error())
	}

	mh := MessageHandler{
		msgChan:  make(chan *nsq.Message, 5),
		stopChan: make(chan int),
	}

	wc := make(chan WriteMessage)

	pq := &PriorityQueue{}
	heap.Init(pq)

	lag := time.Duration(time.Duration(*lag_time) * time.Second)

	go store(wc, pq, lag)
	go writer(mh, wc)

	r.AddAsyncHandler(&mh)

	err = r.ConnectToNSQ(*nsqTCPAddrs)
	if err != nil {
		log.Fatalf(err.Error())
	}
	err = r.ConnectToLookupd(*lookupdHTTPAddrs)
	if err != nil {
		log.Fatalf(err.Error())
	}

	<-mh.stopChan
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

type configuration struct {
	ImageName           string   `json:"image_name"`
	MountHomeTo         string   `json:"mount_home_to"`
	ContainerUsername   string   `json:"container_username"`
	Shell               string   `json:"shell"`
	BlacklistUserConfig []string `json:"blacklist_user_config"`
}

type configInterpolation struct {
	Home string
	User string
}

func loadConfig(filename string) (config *configuration, found bool, err error) {
	found = true
	config = &configuration{
		ImageName:         "ubuntu",
		MountHomeTo:       "%h",
		ContainerUsername: "%u",
		Shell:             "%s",
	}
	localConfigFile, err := os.Open(filename)
	if err != nil {
		found = false
		err = nil
		return
	}
	bytes, err := ioutil.ReadAll(localConfigFile)
	if err != nil {
		return
	}
	var localConfig map[string]interface{}
	err = json.Unmarshal(bytes, &localConfig)
	if err != nil {
		return
	}
	localConfigFile.Close()

	for k, v := range localConfig {
		switch k {
		case "image_name":
			if localImageName, ok := v.(string); !ok {
				return nil, found, errors.New("parse")
			} else {
				config.ImageName = localImageName
			}
		case "mount_home_to":
			if localMountHomeTo, ok := v.(string); !ok {
				return nil, found, errors.New("parse")
			} else {
				config.MountHomeTo = localMountHomeTo
			}
		case "container_username":
			if localContainerUsername, ok := v.(string); !ok {
				return nil, found, errors.New("parse")
			} else {
				config.ContainerUsername = localContainerUsername
			}
		case "shell":
			if localShell, ok := v.(string); !ok {
				return nil, found, errors.New("parse")
			} else {
				config.Shell = localShell
			}
		}
	}
	return config, found, nil
}

func main() {
	os.Exit(realMain())
}

func tmplConfigVar(template string, v *configInterpolation) string {
	shell := "/bin/bash"
	return strings.Replace(strings.Replace(strings.Replace(template, "%h", v.Home, -1), "%u", v.User, -1), "%s", shell, -1)
}

func realMain() int {
	_, err := nsenterdetect()
	if err != nil {
		return 1
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not load config: %v", err)
		return 1
	}
	/* Woo! We found nsenter, now to move onto more interesting things */
	username, homedir, uid, gid, err := getCurrentUser()
	config, found, err := loadConfig(fmt.Sprintf("%s/.dockersh.json", homedir))
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not load config: %v", err)
		return 1
	}
	if !found {
		config, found, err = loadConfig("/etc/dockersh.json")
		if err != nil {
			fmt.Fprintf(os.Stderr, "could not load config: %v", err)
			return 1
		}
	}
	configInterpolations := configInterpolation{homedir, username}

	realUsername := tmplConfigVar(config.ContainerUsername, &configInterpolations)
	realHomedir := tmplConfigVar(config.MountHomeTo, &configInterpolations)
	realImageName := tmplConfigVar(config.ImageName, &configInterpolations)
	realShell := tmplConfigVar(config.Shell, &configInterpolations)
	containerName := fmt.Sprintf("%s_dockersh", realUsername)

	pid, err := dockerpid(containerName)
	if err != nil {
		pid, err = dockerstart(realUsername, realHomedir, containerName, realImageName)
		if err != nil {
			fmt.Fprintf(os.Stderr, "could not start container: %s\n", err)
			return 1
		}
	}
	nsenterexec(pid, uid, gid, realHomedir, realShell)
	return 0
}
<commit_msg>Cleanup config handling switch<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

type configuration struct {
	ImageName           string   `json:"image_name"`
	MountHomeTo         string   `json:"mount_home_to"`
	ContainerUsername   string   `json:"container_username"`
	Shell               string   `json:"shell"`
	BlacklistUserConfig []string `json:"blacklist_user_config"`
}

type configInterpolation struct {
	Home string
	User string
}

func loadConfig(filename string) (config *configuration, found bool, err error) {
	found = true
	config = &configuration{
		ImageName:         "ubuntu",
		MountHomeTo:       "%h",
		ContainerUsername: "%u",
		Shell:             "%s",
	}
	localConfigFile, err := os.Open(filename)
	if err != nil {
		found = false
		err = nil
		return
	}
	bytes, err := ioutil.ReadAll(localConfigFile)
	if err != nil {
		return
	}
	var localConfig map[string]interface{}
	err = json.Unmarshal(bytes, &localConfig)
	if err != nil {
		return
	}
	localConfigFile.Close()

	for k, v := range localConfig {
		data, ok := v.(string)
		if !ok {
			return nil, found, errors.New("parse")
		}
		switch k {
		case "image_name":
			config.ImageName = data
		case "mount_home_to":
			config.MountHomeTo = data
		case "container_username":
			config.ContainerUsername = data
		case "shell":
			config.Shell = data
		}
	}
	return config, found, nil
}

func main() {
	os.Exit(realMain())
}

func tmplConfigVar(template string, v *configInterpolation) string {
	shell := "/bin/bash"
	return strings.Replace(strings.Replace(strings.Replace(template, "%h", v.Home, -1), "%u", v.User, -1), "%s", shell, -1)
}

func realMain() int {
	_, err := nsenterdetect()
	if err != nil {
		return 1
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not load config: %v", err)
		return 1
	}
	/* Woo! We found nsenter, now to move onto more interesting things */
	username, homedir, uid, gid, err := getCurrentUser()
	config, found, err := loadConfig(fmt.Sprintf("%s/.dockersh.json", homedir))
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not load config: %v", err)
		return 1
	}
	if !found {
		config, found, err = loadConfig("/etc/dockersh.json")
		if err != nil {
			fmt.Fprintf(os.Stderr, "could not load config: %v", err)
			return 1
		}
	}
	configInterpolations := configInterpolation{homedir, username}

	realUsername := tmplConfigVar(config.ContainerUsername, &configInterpolations)
	realHomedir := tmplConfigVar(config.MountHomeTo, &configInterpolations)
	realImageName := tmplConfigVar(config.ImageName, &configInterpolations)
	realShell := tmplConfigVar(config.Shell, &configInterpolations)
	containerName := fmt.Sprintf("%s_dockersh", realUsername)

	pid, err := dockerpid(containerName)
	if err != nil {
		pid, err = dockerstart(realUsername, realHomedir, containerName, realImageName)
		if err != nil {
			fmt.Fprintf(os.Stderr, "could not start container: %s\n", err)
			return 1
		}
	}
	nsenterexec(pid, uid, gid, realHomedir, realShell)
	return 0
}
<|endoftext|>


<commit_before>// Copyright 2015 Robert S. Gerus. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package bot

import (
	"math/rand"
	"strings"
	"time"

	"github.com/arachnist/gorepost/irc"
)

func pick(output func(irc.Message), msg irc.Message) {
	var args []string
	if !strings.HasPrefix(msg.Trailing, ":pick ") {
		return
	}

	a := strings.TrimPrefix(msg.Trailing, ":pick ")

	if strings.Contains(a, ",") {
		args = strings.Split(a, ",")
	} else {
		args = strings.Fields(a)
	}

	choice := args[rand.Intn(len(args))]

	output(reply(msg, choice))
}

func init() {
	rand.Seed(time.Now().UnixNano())
	addCallback("PRIVMSG", "pick", pick)
}
<commit_msg>Make splitting with commas more sensible.<commit_after>// Copyright 2015 Robert S. Gerus. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package bot

import (
	"math/rand"
	"strings"
	"time"

	"github.com/arachnist/gorepost/irc"
)

func pick(output func(irc.Message), msg irc.Message) {
	var args []string
	if !strings.HasPrefix(msg.Trailing, ":pick ") {
		return
	}

	a := strings.TrimPrefix(msg.Trailing, ":pick ")

	if strings.Contains(a, ", ") {
		args = strings.Split(a, ", ")
	} else if strings.Contains(a, ",") {
		args = strings.Split(a, ",")
	} else {
		args = strings.Fields(a)
	}

	choice := args[rand.Intn(len(args))]

	output(reply(msg, choice))
}

func init() {
	rand.Seed(time.Now().UnixNano())
	addCallback("PRIVMSG", "pick", pick)
}
<|endoftext|>