<commit_before>package main

import (
	"bufio"
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/jessevdk/go-flags"
	"github.com/mackerelio/checkers"
	"github.com/mattn/go-encoding"
	enc "golang.org/x/text/encoding"
)

type logOpts struct {
	LogFile         string  `short:"f" long:"file" value-name:"FILE" description:"Path to log file"`
	Pattern         string  `short:"p" long:"pattern" required:"true" value-name:"PAT" description:"Pattern to search for"`
	Exclude         string  `short:"E" long:"exclude" value-name:"PAT" description:"Pattern to exclude from matching"`
	WarnOver        int64   `short:"w" long:"warning-over" description:"Trigger a warning if matched lines is over a number"`
	CritOver        int64   `short:"c" long:"critical-over" description:"Trigger a critical if matched lines is over a number"`
	WarnLevel       float64 `long:"warning-level" value-name:"N" description:"Warning level if pattern has a group"`
	CritLevel       float64 `long:"critical-level" value-name:"N" description:"Critical level if pattern has a group"`
	ReturnContent   bool    `short:"r" long:"return" description:"Return matched line"`
	FilePattern     string  `short:"F" long:"file-pattern" value-name:"FILE" description:"Check a pattern of files, instead of one file"`
	CaseInsensitive bool    `short:"i" long:"icase" description:"Run a case insensitive match"`
	StateDir        string  `short:"s" long:"state-dir" value-name:"DIR" description:"Dir to keep state files under"`
	NoState         bool    `long:"no-state" description:"Don't use state file and read whole logs"`
	Encoding        string  `long:"encoding" description:"Encoding of log file"`
	Missing         string  `long:"missing" default:"UNKNOWN" value-name:"(CRITICAL|WARNING|OK|UNKNOWN)" description:"Exit status when log files missing"`
	patternReg      *regexp.Regexp
	excludeReg      *regexp.Regexp
	fileList        []string
	origArgs        []string
	decoder         *enc.Decoder
}

func (opts *logOpts) prepare() error {
	if opts.LogFile == "" && opts.FilePattern == "" {
		return fmt.Errorf("No log file specified")
	}

	var err error
	if opts.patternReg, err = regCompileWithCase(opts.Pattern, opts.CaseInsensitive); err != nil {
		return fmt.Errorf("pattern is invalid")
	}

	if opts.Exclude != "" {
		opts.excludeReg, err = regCompileWithCase(opts.Exclude, opts.CaseInsensitive)
		if err != nil {
			return fmt.Errorf("exclude pattern is invalid")
		}
	}

	if opts.LogFile != "" {
		opts.fileList = append(opts.fileList, opts.LogFile)
	}

	if opts.FilePattern != "" {
		dirStr := filepath.Dir(opts.FilePattern)
		filePat := filepath.Base(opts.FilePattern)
		reg, err := regCompileWithCase(filePat, opts.CaseInsensitive)
		if err != nil {
			return fmt.Errorf("file-pattern is invalid")
		}

		fileInfos, err := ioutil.ReadDir(dirStr)
		if err != nil {
			return fmt.Errorf("cannot read the directory:" + err.Error())
		}

		for _, fileInfo := range fileInfos {
			if fileInfo.IsDir() {
				continue
			}
			fname := fileInfo.Name()
			if reg.MatchString(fname) {
				opts.fileList = append(opts.fileList, dirStr+string(filepath.Separator)+fileInfo.Name())
			}
		}
	}
	if !validateMissing(opts.Missing) {
		return fmt.Errorf("missing option is invalid")
	}
	return nil
}

func main() {
	ckr := run(os.Args[1:])
	ckr.Name = "LOG"
	ckr.Exit()
}

func regCompileWithCase(ptn string, caseInsensitive bool) (*regexp.Regexp, error) {
	if caseInsensitive {
		ptn = "(?i)" + ptn
	}
	return regexp.Compile(ptn)
}

func validateMissing(missing string) bool {
	switch missing {
	case "CRITICAL", "WARNING", "OK", "UNKNOWN", "":
		return true
	default:
		return false
	}
}

func parseArgs(args []string) (*logOpts, error) {
	origArgs := make([]string, len(args))
	copy(origArgs, args)
	opts := &logOpts{}
	_, err := flags.ParseArgs(opts, args)
	opts.origArgs = origArgs
	if opts.StateDir == "" {
		workdir := os.Getenv("MACKEREL_PLUGIN_WORKDIR")
		if workdir == "" {
			workdir = os.TempDir()
		}
		opts.StateDir = filepath.Join(workdir, "check-log")
	}
	return opts, err
}

func run(args []string) *checkers.Checker {
	opts, err := parseArgs(args)
	if err != nil {
		os.Exit(1)
	}

	err = opts.prepare()
	if err != nil {
		return checkers.Unknown(err.Error())
	}

	warnNum := int64(0)
	critNum := int64(0)
	var missingFiles []string
	errorOverall := ""

	for _, f := range opts.fileList {
		_, err := os.Stat(f)
		if err != nil {
			missingFiles = append(missingFiles, f)
			continue
		}
		w, c, errLines, err := opts.searchLog(f)
		if err != nil {
			return checkers.Unknown(err.Error())
		}
		warnNum += w
		critNum += c
		if opts.ReturnContent {
			errorOverall += errLines
		}
	}

	msg := fmt.Sprintf("%d warnings, %d criticals for pattern /%s/.", warnNum, critNum, opts.Pattern)
	if errorOverall != "" {
		msg += "\n" + errorOverall
	}
	checkSt := checkers.OK
	if len(missingFiles) > 0 {
		switch opts.Missing {
		case "OK":
		case "WARNING":
			checkSt = checkers.WARNING
		case "CRITICAL":
			checkSt = checkers.CRITICAL
		default:
			checkSt = checkers.UNKNOWN
		}
		msg += "\n" + fmt.Sprintf("The following %d files are missing.", len(missingFiles))
		for _, f := range missingFiles {
			msg += "\n" + f
		}
	}
	if warnNum > opts.WarnOver {
		checkSt = checkers.WARNING
	}
	if critNum > opts.CritOver {
		checkSt = checkers.CRITICAL
	}
	return checkers.NewChecker(checkSt, msg)
}

func (opts *logOpts) searchLog(logFile string) (int64, int64, string, error) {
	stateFile := getStateFile(opts.StateDir, logFile, opts.origArgs)
	skipBytes := int64(0)
	if !opts.NoState {
		s, err := getBytesToSkip(stateFile)
		if err != nil {
			return 0, 0, "", err
		}
		skipBytes = s
	}

	f, err := os.Open(logFile)
	if err != nil {
		return 0, 0, "", err
	}
	defer f.Close()

	stat, err := f.Stat()
	if err != nil {
		return 0, 0, "", err
	}

	rotated := false
	if stat.Size() < skipBytes {
		rotated = true
	} else if skipBytes > 0 {
		f.Seek(skipBytes, 0)
	}

	var r io.Reader = f
	if opts.Encoding != "" {
		e := encoding.GetEncoding(opts.Encoding)
		if e == nil {
			return 0, 0, "", fmt.Errorf("unknown encoding:" + opts.Encoding)
		}
		opts.decoder = e.NewDecoder()
	}

	warnNum, critNum, readBytes, errLines, err := opts.searchReader(r)
	if err != nil {
		return warnNum, critNum, errLines, err
	}

	if rotated {
		skipBytes = readBytes
	} else {
		skipBytes += readBytes
	}

	if !opts.NoState {
		err = writeBytesToSkip(stateFile, skipBytes)
		if err != nil {
			log.Printf("writeByteToSkip failed: %s\n", err.Error())
		}
	}
	return warnNum, critNum, errLines, nil
}

func (opts *logOpts) searchReader(rdr io.Reader) (warnNum, critNum, readBytes int64, errLines string, err error) {
	r := bufio.NewReader(rdr)
	for {
		lineBytes, rErr := r.ReadBytes('\n')
		if rErr != nil {
			if rErr != io.EOF {
				err = rErr
			}
			break
		}
		readBytes += int64(len(lineBytes))

		if opts.decoder != nil {
			lineBytes, rErr = opts.decoder.Bytes(lineBytes)
			if rErr != nil {
				if rErr != io.EOF {
					err = rErr
				}
				break
			}
		}
		line := strings.Trim(string(lineBytes), "\r\n")
		if matched, matches := opts.match(line); matched {
			if len(matches) > 1 && (opts.WarnLevel > 0 || opts.CritLevel > 0) {
				level, err := strconv.ParseFloat(matches[1], 64)
				if err != nil {
					warnNum++
					critNum++
					errLines += line + "\n"
				} else {
					levelOver := false
					if level > opts.WarnLevel {
						levelOver = true
						warnNum++
					}
					if level > opts.CritLevel {
						levelOver = true
						critNum++
					}
					if levelOver {
						errLines += line + "\n"
					}
				}
			} else {
				warnNum++
				critNum++
				errLines += line + "\n"
			}
		}
	}
	return
}

func (opts *logOpts) match(line string) (bool, []string) {
	pReg := opts.patternReg
	eReg := opts.excludeReg

	matches := pReg.FindStringSubmatch(line)
	matched := len(matches) > 0 && (eReg == nil || !eReg.MatchString(line))
	return matched, matches
}

var stateRe = regexp.MustCompile(`^([A-Z]):[/\\]`)

func getStateFile(stateDir, f string, args []string) string {
	return filepath.Join(
		stateDir,
		fmt.Sprintf(
			"%s-%x",
			stateRe.ReplaceAllString(f, `$1`+string(filepath.Separator)),
			md5.Sum([]byte(strings.Join(args, " "))),
		),
	)
}

func getBytesToSkip(f string) (int64, error) {
	_, err := os.Stat(f)
	if err != nil {
		return 0, nil
	}
	b, err := ioutil.ReadFile(f)
	if err != nil {
		return 0, err
	}
	i, err := strconv.ParseInt(strings.Trim(string(b), " \r\n"), 10, 64)
	if err != nil {
		log.Printf("failed to getBytesToSkip (ignoring): %s", err)
	}
	return i, nil
}

func writeBytesToSkip(f string, num int64) error {
	err := os.MkdirAll(filepath.Dir(f), 0755)
	if err != nil {
		return err
	}
	return writeFileAtomically(f, []byte(fmt.Sprintf("%d", num)))
}

func writeFileAtomically(f string, contents []byte) error {
	tmpf, err := ioutil.TempFile("", "")
	if err != nil {
		return err
	}
	defer os.Remove(tmpf.Name())
	_, err = tmpf.Write(contents)
	if err != nil {
		return err
	}
	tmpf.Close()
	return os.Rename(tmpf.Name(), f)
}
<commit_msg>no need to check EOF<commit_after>package main

import (
	"bufio"
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/jessevdk/go-flags"
	"github.com/mackerelio/checkers"
	"github.com/mattn/go-encoding"
	enc "golang.org/x/text/encoding"
)

type logOpts struct {
	LogFile         string  `short:"f" long:"file" value-name:"FILE" description:"Path to log file"`
	Pattern         string  `short:"p" long:"pattern" required:"true" value-name:"PAT" description:"Pattern to search for"`
	Exclude         string  `short:"E" long:"exclude" value-name:"PAT" description:"Pattern to exclude from matching"`
	WarnOver        int64   `short:"w" long:"warning-over" description:"Trigger a warning if matched lines is over a number"`
	CritOver        int64   `short:"c" long:"critical-over" description:"Trigger a critical if matched lines is over a number"`
	WarnLevel       float64 `long:"warning-level" value-name:"N" description:"Warning level if pattern has a group"`
	CritLevel       float64 `long:"critical-level" value-name:"N" description:"Critical level if pattern has a group"`
	ReturnContent   bool    `short:"r" long:"return" description:"Return matched line"`
	FilePattern     string  `short:"F" long:"file-pattern" value-name:"FILE" description:"Check a pattern of files, instead of one file"`
	CaseInsensitive bool    `short:"i" long:"icase" description:"Run a case insensitive match"`
	StateDir        string  `short:"s" long:"state-dir" value-name:"DIR" description:"Dir to keep state files under"`
	NoState         bool    `long:"no-state" description:"Don't use state file and read whole logs"`
	Encoding        string  `long:"encoding" description:"Encoding of log file"`
	Missing         string  `long:"missing" default:"UNKNOWN" value-name:"(CRITICAL|WARNING|OK|UNKNOWN)" description:"Exit status when log files missing"`
	patternReg      *regexp.Regexp
	excludeReg      *regexp.Regexp
	fileList        []string
	origArgs        []string
	decoder         *enc.Decoder
}

func (opts *logOpts) prepare() error {
	if opts.LogFile == "" && opts.FilePattern == "" {
		return fmt.Errorf("No log file specified")
	}

	var err error
	if opts.patternReg, err = regCompileWithCase(opts.Pattern, opts.CaseInsensitive); err != nil {
		return fmt.Errorf("pattern is invalid")
	}

	if opts.Exclude != "" {
		opts.excludeReg, err = regCompileWithCase(opts.Exclude, opts.CaseInsensitive)
		if err != nil {
			return fmt.Errorf("exclude pattern is invalid")
		}
	}

	if opts.LogFile != "" {
		opts.fileList = append(opts.fileList, opts.LogFile)
	}

	if opts.FilePattern != "" {
		dirStr := filepath.Dir(opts.FilePattern)
		filePat := filepath.Base(opts.FilePattern)
		reg, err := regCompileWithCase(filePat, opts.CaseInsensitive)
		if err != nil {
			return fmt.Errorf("file-pattern is invalid")
		}

		fileInfos, err := ioutil.ReadDir(dirStr)
		if err != nil {
			return fmt.Errorf("cannot read the directory:" + err.Error())
		}

		for _, fileInfo := range fileInfos {
			if fileInfo.IsDir() {
				continue
			}
			fname := fileInfo.Name()
			if reg.MatchString(fname) {
				opts.fileList = append(opts.fileList, dirStr+string(filepath.Separator)+fileInfo.Name())
			}
		}
	}
	if !validateMissing(opts.Missing) {
		return fmt.Errorf("missing option is invalid")
	}
	return nil
}

func main() {
	ckr := run(os.Args[1:])
	ckr.Name = "LOG"
	ckr.Exit()
}

func regCompileWithCase(ptn string, caseInsensitive bool) (*regexp.Regexp, error) {
	if caseInsensitive {
		ptn = "(?i)" + ptn
	}
	return regexp.Compile(ptn)
}

func validateMissing(missing string) bool {
	switch missing {
	case "CRITICAL", "WARNING", "OK", "UNKNOWN", "":
		return true
	default:
		return false
	}
}

func parseArgs(args []string) (*logOpts, error) {
	origArgs := make([]string, len(args))
	copy(origArgs, args)
	opts := &logOpts{}
	_, err := flags.ParseArgs(opts, args)
	opts.origArgs = origArgs
	if opts.StateDir == "" {
		workdir := os.Getenv("MACKEREL_PLUGIN_WORKDIR")
		if workdir == "" {
			workdir = os.TempDir()
		}
		opts.StateDir = filepath.Join(workdir, "check-log")
	}
	return opts, err
}

func run(args []string) *checkers.Checker {
	opts, err := parseArgs(args)
	if err != nil {
		os.Exit(1)
	}

	err = opts.prepare()
	if err != nil {
		return checkers.Unknown(err.Error())
	}

	warnNum := int64(0)
	critNum := int64(0)
	var missingFiles []string
	errorOverall := ""

	for _, f := range opts.fileList {
		_, err := os.Stat(f)
		if err != nil {
			missingFiles = append(missingFiles, f)
			continue
		}
		w, c, errLines, err := opts.searchLog(f)
		if err != nil {
			return checkers.Unknown(err.Error())
		}
		warnNum += w
		critNum += c
		if opts.ReturnContent {
			errorOverall += errLines
		}
	}

	msg := fmt.Sprintf("%d warnings, %d criticals for pattern /%s/.", warnNum, critNum, opts.Pattern)
	if errorOverall != "" {
		msg += "\n" + errorOverall
	}
	checkSt := checkers.OK
	if len(missingFiles) > 0 {
		switch opts.Missing {
		case "OK":
		case "WARNING":
			checkSt = checkers.WARNING
		case "CRITICAL":
			checkSt = checkers.CRITICAL
		default:
			checkSt = checkers.UNKNOWN
		}
		msg += "\n" + fmt.Sprintf("The following %d files are missing.", len(missingFiles))
		for _, f := range missingFiles {
			msg += "\n" + f
		}
	}
	if warnNum > opts.WarnOver {
		checkSt = checkers.WARNING
	}
	if critNum > opts.CritOver {
		checkSt = checkers.CRITICAL
	}
	return checkers.NewChecker(checkSt, msg)
}

func (opts *logOpts) searchLog(logFile string) (int64, int64, string, error) {
	stateFile := getStateFile(opts.StateDir, logFile, opts.origArgs)
	skipBytes := int64(0)
	if !opts.NoState {
		s, err := getBytesToSkip(stateFile)
		if err != nil {
			return 0, 0, "", err
		}
		skipBytes = s
	}

	f, err := os.Open(logFile)
	if err != nil {
		return 0, 0, "", err
	}
	defer f.Close()

	stat, err := f.Stat()
	if err != nil {
		return 0, 0, "", err
	}

	rotated := false
	if stat.Size() < skipBytes {
		rotated = true
	} else if skipBytes > 0 {
		f.Seek(skipBytes, 0)
	}

	var r io.Reader = f
	if opts.Encoding != "" {
		e := encoding.GetEncoding(opts.Encoding)
		if e == nil {
			return 0, 0, "", fmt.Errorf("unknown encoding:" + opts.Encoding)
		}
		opts.decoder = e.NewDecoder()
	}

	warnNum, critNum, readBytes, errLines, err := opts.searchReader(r)
	if err != nil {
		return warnNum, critNum, errLines, err
	}

	if rotated {
		skipBytes = readBytes
	} else {
		skipBytes += readBytes
	}

	if !opts.NoState {
		err = writeBytesToSkip(stateFile, skipBytes)
		if err != nil {
			log.Printf("writeByteToSkip failed: %s\n", err.Error())
		}
	}
	return warnNum, critNum, errLines, nil
}

func (opts *logOpts) searchReader(rdr io.Reader) (warnNum, critNum, readBytes int64, errLines string, err error) {
	r := bufio.NewReader(rdr)
	for {
		lineBytes, rErr := r.ReadBytes('\n')
		if rErr != nil {
			if rErr != io.EOF {
				err = rErr
			}
			break
		}
		readBytes += int64(len(lineBytes))

		if opts.decoder != nil {
			lineBytes, err = opts.decoder.Bytes(lineBytes)
			if err != nil {
				break
			}
		}
		line := strings.Trim(string(lineBytes), "\r\n")
		if matched, matches := opts.match(line); matched {
			if len(matches) > 1 && (opts.WarnLevel > 0 || opts.CritLevel > 0) {
				level, err := strconv.ParseFloat(matches[1], 64)
				if err != nil {
					warnNum++
					critNum++
					errLines += line + "\n"
				} else {
					levelOver := false
					if level > opts.WarnLevel {
						levelOver = true
						warnNum++
					}
					if level > opts.CritLevel {
						levelOver = true
						critNum++
					}
					if levelOver {
						errLines += line + "\n"
					}
				}
			} else {
				warnNum++
				critNum++
				errLines += line + "\n"
			}
		}
	}
	return
}

func (opts *logOpts) match(line string) (bool, []string) {
	pReg := opts.patternReg
	eReg := opts.excludeReg

	matches := pReg.FindStringSubmatch(line)
	matched := len(matches) > 0 && (eReg == nil || !eReg.MatchString(line))
	return matched, matches
}

var stateRe = regexp.MustCompile(`^([A-Z]):[/\\]`)

func getStateFile(stateDir, f string, args []string) string {
	return filepath.Join(
		stateDir,
		fmt.Sprintf(
			"%s-%x",
			stateRe.ReplaceAllString(f, `$1`+string(filepath.Separator)),
			md5.Sum([]byte(strings.Join(args, " "))),
		),
	)
}

func getBytesToSkip(f string) (int64, error) {
	_, err := os.Stat(f)
	if err != nil {
		return 0, nil
	}
	b, err := ioutil.ReadFile(f)
	if err != nil {
		return 0, err
	}
	i, err := strconv.ParseInt(strings.Trim(string(b), " \r\n"), 10, 64)
	if err != nil {
		log.Printf("failed to getBytesToSkip (ignoring): %s", err)
	}
	return i, nil
}

func writeBytesToSkip(f string, num int64) error {
	err := os.MkdirAll(filepath.Dir(f), 0755)
	if err != nil {
		return err
	}
	return writeFileAtomically(f, []byte(fmt.Sprintf("%d", num)))
}

func writeFileAtomically(f string, contents []byte) error {
	tmpf, err := ioutil.TempFile("", "")
	if err != nil {
		return err
	}
	defer os.Remove(tmpf.Name())
	_, err = tmpf.Write(contents)
	if err != nil {
		return err
	}
	tmpf.Close()
	return os.Rename(tmpf.Name(), f)
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"bytes"
	"crypto/sha256"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"

	"github.com/mvdan/fdroidcl"
)

var cmdUpdate = &Command{
	UsageLine: "update",
	Short:     "Update the index",
}

func init() {
	cmdUpdate.Run = runUpdate
}

func runUpdate(args []string) {
	r := mustOneRepo()
	if err := updateRepo(r); err != nil {
		log.Fatalf("Could not update index: %v", err)
	}
}

func updateRepo(r *repo) error {
	url := fmt.Sprintf("%s/%s", r.URL, "index.jar")
	if err := downloadEtag(url, indexPath(r.ID), nil); err != nil {
		return err
	}
	return nil
}

func respEtag(resp *http.Response) string {
	etags, e := resp.Header["Etag"]
	if !e || len(etags) == 0 {
		return ""
	}
	return etags[0]
}

func downloadEtag(url, path string, sum []byte) error {
	fmt.Printf("Downloading %s... ", url)
	defer fmt.Println()
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)

	etagPath := path + "-etag"
	if _, err := os.Stat(path); err == nil {
		etag, _ := ioutil.ReadFile(etagPath)
		req.Header.Add("If-None-Match", string(etag))
	}

	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusNotModified {
		fmt.Printf("not modified")
		return nil
	}
	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	defer f.Close()
	if sum == nil {
		_, err := io.Copy(f, resp.Body)
		if err != nil {
			return err
		}
	} else {
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		got := sha256.Sum256(data)
		if !bytes.Equal(sum, got[:]) {
			return errors.New("sha256 mismatch")
		}
		if _, err := f.Write(data); err != nil {
			return err
		}
	}
	if err := ioutil.WriteFile(etagPath, []byte(respEtag(resp)), 0644); err != nil {
		return err
	}
	fmt.Printf("done")
	return nil
}

func indexPath(name string) string {
	return filepath.Join(mustConfig(), name+".jar")
}

func mustLoadIndex() *fdroidcl.Index {
	r := mustOneRepo()
	p := indexPath(r.ID)
	f, err := os.Open(p)
	if err != nil {
		log.Fatalf("Could not open index file: %v", err)
	}
	stat, err := f.Stat()
	if err != nil {
		log.Fatalf("Could not stat index file: %v", err)
	}
	//pubkey, err := hex.DecodeString(repoPubkey)
	//if err != nil {
	//	log.Fatalf("Could not decode public key: %v", err)
	//}
	index, err := fdroidcl.LoadIndexJar(f, stat.Size(), nil)
	if err != nil {
		log.Fatalf("Could not load index: %v", err)
	}
	return index
}
<commit_msg>Make update use all configured repos<commit_after>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"bytes"
	"crypto/sha256"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"

	"github.com/mvdan/fdroidcl"
)

var cmdUpdate = &Command{
	UsageLine: "update",
	Short:     "Update the index",
}

func init() {
	cmdUpdate.Run = runUpdate
}

func runUpdate(args []string) {
	for _, r := range config.Repos {
		if err := r.updateIndex(); err != nil {
			log.Fatalf("Could not update index: %v", err)
		}
	}
}

func (r *repo) updateIndex() error {
	url := fmt.Sprintf("%s/%s", r.URL, "index.jar")
	if err := downloadEtag(url, indexPath(r.ID), nil); err != nil {
		return err
	}
	return nil
}

func respEtag(resp *http.Response) string {
	etags, e := resp.Header["Etag"]
	if !e || len(etags) == 0 {
		return ""
	}
	return etags[0]
}

func downloadEtag(url, path string, sum []byte) error {
	fmt.Printf("Downloading %s... ", url)
	defer fmt.Println()
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)

	etagPath := path + "-etag"
	if _, err := os.Stat(path); err == nil {
		etag, _ := ioutil.ReadFile(etagPath)
		req.Header.Add("If-None-Match", string(etag))
	}

	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusNotModified {
		fmt.Printf("not modified")
		return nil
	}
	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	defer f.Close()
	if sum == nil {
		_, err := io.Copy(f, resp.Body)
		if err != nil {
			return err
		}
	} else {
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		got := sha256.Sum256(data)
		if !bytes.Equal(sum, got[:]) {
			return errors.New("sha256 mismatch")
		}
		if _, err := f.Write(data); err != nil {
			return err
		}
	}
	if err := ioutil.WriteFile(etagPath, []byte(respEtag(resp)), 0644); err != nil {
		return err
	}
	fmt.Printf("done")
	return nil
}

func indexPath(name string) string {
	return filepath.Join(mustConfig(), name+".jar")
}

func mustLoadIndex() *fdroidcl.Index {
	r := mustOneRepo()
	p := indexPath(r.ID)
	f, err := os.Open(p)
	if err != nil {
		log.Fatalf("Could not open index file: %v", err)
	}
	stat, err := f.Stat()
	if err != nil {
		log.Fatalf("Could not stat index file: %v", err)
	}
	//pubkey, err := hex.DecodeString(repoPubkey)
	//if err != nil {
	//	log.Fatalf("Could not decode public key: %v", err)
	//}
	index, err := fdroidcl.LoadIndexJar(f, stat.Size(), nil)
	if err != nil {
		log.Fatalf("Could not load index: %v", err)
	}
	return index
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
)

type SingleEpisode struct {
	commonEpisode

	VideoURL     string `json:"video_url"`
	ShowName     string `json:"show_name"`
	SeasonNumber int    `json:"season_number"`
}

func episodeExists(seasonDir string, episode TvMazeEpisode) bool {
	if episodeVideoFile(seasonDir, episode) == "" {
		return false
	}
	return true
}

func urlify(name string) string {
	re := regexp.MustCompile("[^a-zA-Z0-9]")
	return re.ReplaceAllString(name, "-")
}

func writeEpisodes(show *TvMazeShow) {
	for _, seasonNumber := range seasons(show) {
		if _, err := os.Stat(path.Join(show.Name, strconv.Itoa(seasonNumber))); err != nil {
			log.WithFields(log.Fields{
				"err":    err,
				"season": seasonNumber,
				"show":   show.Name,
			}).Warn("season not found on disk, skipping")
			continue
		}

		for _, episode := range episodes(seasonNumber, show) {
			writeEpisodeJSON(episode)
			writeEpisodeApp(episode)
		}
	}

}

func writeEpisodeApp(episode SingleEpisode) {
	episodeDir := path.Join(
		episode.ShowName,
		strconv.Itoa(episode.SeasonNumber),
		urlify(episode.Name),
	)

	app, err := os.Create(path.Join(episodeDir, "index.html"))
	if err != nil {
		log.WithField("err", err).Error("Error creating index.html in shows root")
		return
	}
	_, err = app.Write(episodeApp)
	if err != nil {
		log.WithField("err", err).Error("Error writing index.html in shows root")
		return
	}
}

func writeEpisodeJSON(episode SingleEpisode) {
	episodeDir := path.Join(
		episode.ShowName,
		strconv.Itoa(episode.SeasonNumber),
		urlify(episode.Name),
	)

	if _, err := os.Stat(episodeDir); err != nil {
		err := os.Mkdir(episodeDir, 0766)
		if err != nil {
			log.WithField("err", err).Error("failed to create episode directory")
			return
		}
	}

	file, err := os.Create(path.Join(
		episodeDir,
		"episode.json"),
	)
	if err != nil {
		log.WithFields(log.Fields{
			"err": err,
			"dir": episodeDir,
		}).Warn("failed to create episode.json")
		return
	}
	defer file.Close()

	if err := json.NewEncoder(file).Encode(episode); err != nil {
		log.WithFields(log.Fields{
			"err": err,
			"dir": episodeDir,
		}).Warn("failed to encode to episode.json")
		return
	}

	log.WithFields(log.Fields{
		"file": file.Name(),
	}).Debug("episode written to disk")
}

func episodes(seasonNumber int, show *TvMazeShow) []SingleEpisode {
	episodes := []SingleEpisode{}

	for _, episode := range show.Embedded.Episodes {
		if int(episode.Season) != seasonNumber {
			log.WithFields(log.Fields{
				"actual_season":   episode.Season,
				"required_season": seasonNumber,
			}).Debug("wrong season, skipping")
			continue
		}

		// Check if episode exists on disk
		if !episodeExists(path.Join(show.Name, strconv.Itoa(seasonNumber)), episode) {
			log.WithFields(log.Fields{
				"episode": episode.Episode,
				"name":    episode.Name,
				"path":    path.Join(show.Name, strconv.Itoa(seasonNumber)),
			}).Warn("episode doesn't exists on disk, skipping")
			continue
		}

		episodes = append(episodes, SingleEpisode{
			commonEpisode: commonEpisode{
				Number:  int(episode.Episode),
				Name:    episode.Name,
				Summary: episode.Summary,
				Image:   episode.Image,
			},

			VideoURL: path.Join(show.Name, strconv.Itoa(seasonNumber), episodeVideoFile(
				path.Join(show.Name, strconv.Itoa(seasonNumber)), episode)),
			ShowName:     show.Name,
			SeasonNumber: seasonNumber,
		})
	}

	return episodes
}

func episodeVideoFile(seasonDir string, episode TvMazeEpisode) string {
	files, err := ioutil.ReadDir(seasonDir)
	if err != nil {
		log.WithFields(log.Fields{
			"season": episode.Season,
			"err":    err,
		}).Error("Error reading season directory")
	}

	for _, file := range files {
		if file.IsDir() {
			log.WithField("file", file.Name()).Debug("is dir, skipping")
			continue
		}

		if strings.Contains(file.Name(), fmt.Sprintf("S%02dE%02d", episode.Season, episode.Episode)) &&
			strings.HasSuffix(file.Name(), "webm") {
			log.WithFields(log.Fields{
				"file":    file.Name(),
				"episode": episode.Episode,
				"season":  episode.Season,
			}).Debug("matched video file with episode")
			return file.Name()
		}
	}

	return ""
}
<commit_msg>episode: "other" need execute access, now proper<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
)

type SingleEpisode struct {
	commonEpisode

	VideoURL     string `json:"video_url"`
	ShowName     string `json:"show_name"`
	SeasonNumber int    `json:"season_number"`
}

func episodeExists(seasonDir string, episode TvMazeEpisode) bool {
	if episodeVideoFile(seasonDir, episode) == "" {
		return false
	}
	return true
}

func urlify(name string) string {
	re := regexp.MustCompile("[^a-zA-Z0-9]")
	return re.ReplaceAllString(name, "-")
}

func writeEpisodes(show *TvMazeShow) {
	for _, seasonNumber := range seasons(show) {
		if _, err := os.Stat(path.Join(show.Name, strconv.Itoa(seasonNumber))); err != nil {
			log.WithFields(log.Fields{
				"err":    err,
				"season": seasonNumber,
				"show":   show.Name,
			}).Warn("season not found on disk, skipping")
			continue
		}

		for _, episode := range episodes(seasonNumber, show) {
			writeEpisodeJSON(episode)
			writeEpisodeApp(episode)
		}
	}

}

func writeEpisodeApp(episode SingleEpisode) {
	episodeDir := path.Join(
		episode.ShowName,
		strconv.Itoa(episode.SeasonNumber),
		urlify(episode.Name),
	)

	app, err := os.Create(path.Join(episodeDir, "index.html"))
	if err != nil {
		log.WithField("err", err).Error("Error creating index.html in shows root")
		return
	}
	_, err = app.Write(episodeApp)
	if err != nil {
		log.WithField("err", err).Error("Error writing index.html in shows root")
		return
	}
}

func writeEpisodeJSON(episode SingleEpisode) {
	episodeDir := path.Join(
		episode.ShowName,
		strconv.Itoa(episode.SeasonNumber),
		urlify(episode.Name),
	)

	if _, err := os.Stat(episodeDir); err != nil {
		err := os.Mkdir(episodeDir, 0755)
		if err != nil {
			log.WithField("err", err).Error("failed to create episode directory")
			return
		}
	}

	file, err := os.Create(path.Join(
		episodeDir,
		"episode.json"),
	)
	if err != nil {
		log.WithFields(log.Fields{
			"err": err,
			"dir": episodeDir,
		}).Warn("failed to create episode.json")
		return
	}
	defer file.Close()

	if err := json.NewEncoder(file).Encode(episode); err != nil {
		log.WithFields(log.Fields{
			"err": err,
			"dir": episodeDir,
		}).Warn("failed to encode to episode.json")
		return
	}

	log.WithFields(log.Fields{
		"file": file.Name(),
	}).Debug("episode written to disk")
}

func episodes(seasonNumber int, show *TvMazeShow) []SingleEpisode {
	episodes := []SingleEpisode{}

	for _, episode := range show.Embedded.Episodes {
		if int(episode.Season) != seasonNumber {
			log.WithFields(log.Fields{
				"actual_season":   episode.Season,
				"required_season": seasonNumber,
			}).Debug("wrong season, skipping")
			continue
		}

		// Check if episode exists on disk
		if !episodeExists(path.Join(show.Name, strconv.Itoa(seasonNumber)), episode) {
			log.WithFields(log.Fields{
				"episode": episode.Episode,
				"name":    episode.Name,
				"path":    path.Join(show.Name, strconv.Itoa(seasonNumber)),
			}).Warn("episode doesn't exists on disk, skipping")
			continue
		}

		episodes = append(episodes, SingleEpisode{
			commonEpisode: commonEpisode{
				Number:  int(episode.Episode),
				Name:    episode.Name,
				Summary: episode.Summary,
				Image:   episode.Image,
			},

			VideoURL: path.Join(show.Name, strconv.Itoa(seasonNumber), episodeVideoFile(
				path.Join(show.Name, strconv.Itoa(seasonNumber)), episode)),
			ShowName:     show.Name,
			SeasonNumber: seasonNumber,
		})
	}

	return episodes
}

func episodeVideoFile(seasonDir string, episode TvMazeEpisode) string {
	files, err := ioutil.ReadDir(seasonDir)
	if err != nil {
		log.WithFields(log.Fields{
			"season": episode.Season,
			"err":    err,
		}).Error("Error reading season directory")
	}

	for _, file := range files {
		if file.IsDir() {
			log.WithField("file", file.Name()).Debug("is dir, skipping")
			continue
		}

		if strings.Contains(file.Name(), fmt.Sprintf("S%02dE%02d", episode.Season, episode.Episode)) &&
			strings.HasSuffix(file.Name(), "webm") {
			log.WithFields(log.Fields{
				"file":    file.Name(),
				"episode": episode.Episode,
				"season":  episode.Season,
			}).Debug("matched video file with episode")
			return file.Name()
		}
	}

	return ""
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"go/format"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	typeconv "github.com/haya14busa/go-typeconv"

	"golang.org/x/tools/go/loader"
)

type option struct {
	write  bool
	doDiff bool
	rules  strslice
}

func main() {
	opt := &option{}
	flag.BoolVar(&opt.write, "w", false, "write result to (source) file instead of stdout")
	flag.BoolVar(&opt.doDiff, "d", false, "display diffs instead of rewriting files")
	flag.Var(&opt.rules, "r", "type conversion rules currently just for type conversion of binary expression (e.g., 'int -> uint32')")
	flag.Parse()
	out := bufio.NewWriter(os.Stdout)
	defer out.Flush()
	if err := run(out, flag.Args(), opt); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func run(w io.Writer, args []string, opt *option) error {
	if err := addRules(opt.rules); err != nil {
		return err
	}
	prog, typeErrs, err := typeconv.Load(loader.Config{}, args)
	if err != nil {
		return err
	}
	for _, pkg := range prog.InitialPackages() {
		for _, f := range pkg.Files {
			filename := prog.Fset.File(f.Pos()).Name()
			if err := typeconv.RewriteFile(prog.Fset, f, pkg, typeErrs); err != nil {
				return err
			}
			buf := new(bytes.Buffer)
			if err := format.Node(buf, prog.Fset, f); err != nil {
				return err
			}
			res := buf.Bytes()
			in, err := os.Open(filename)
			if err != nil {
				return err
			}
			src, err := ioutil.ReadAll(in)
			if err != nil {
				return err
			}
			if !bytes.Equal(src, res) {
				if opt.write {
					fh, err := os.Create(filename)
					if err != nil {
						return err
					}
					fh.Write(res)
					fh.Close()
				}
				if opt.doDiff {
					data, err := diff(src, res)
					if err != nil {
						return fmt.Errorf("computing diff: %s", err)
					}
					fmt.Fprintf(w, "diff %s gotypeconv/%s\n", filename, filename)
					w.Write(data)
				}
			}
			if !opt.write && !opt.doDiff {
				w.Write(res)
			}
		}
	}
	return nil
}

func addRules(rules []string) error {
	for _, r := range rules {
		f := strings.Split(r, "->")
		if len(f) != 2 {
			return fmt.Errorf("type conversion rule must be the form 'from -> to': %v", r)
		}
		from, to := strings.TrimSpace(f[0]), strings.TrimSpace(f[1])
		typeconv.DefaultRule.Add(from, to)
	}
	return nil
}

// copied and modified from $GOPATH/src/github.com/golang/go/src/cmd/gofmt/gofmt.go
//
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
func diff(b1, b2 []byte) (data []byte, err error) {
	f1, err := ioutil.TempFile("", "gotypeconv")
	if err != nil {
		return
	}
	defer os.Remove(f1.Name())
	defer f1.Close()

	f2, err := ioutil.TempFile("", "gotypeconv")
	if err != nil {
		return
	}
	defer os.Remove(f2.Name())
	defer f2.Close()

	f1.Write(b1)
	f2.Write(b2)

	data, err = exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput()
	if len(data) > 0 {
		// diff exits with a non-zero status when the files don't match.
		// Ignore that failure as long as we get output.
		err = nil
	}
	return

}

type strslice []string

func (ss *strslice) String() string {
	return fmt.Sprintf("%v", *ss)
}

func (ss *strslice) Set(value string) error {
	*ss = append(*ss, value)
	return nil
}
<commit_msg>cmd: minor refactoring<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/format"
	"go/types"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	typeconv "github.com/haya14busa/go-typeconv"

	"golang.org/x/tools/go/loader"
)

type option struct {
	write  bool
	doDiff bool
	rules  strslice
}

func main() {
	opt := &option{}
	flag.BoolVar(&opt.write, "w", false, "write result to (source) file instead of stdout")
	flag.BoolVar(&opt.doDiff, "d", false, "display diffs instead of rewriting files")
	flag.Var(&opt.rules, "r", "type conversion rules currently just for type conversion of binary expression (e.g., 'int -> uint32')")
	flag.Parse()
	out := bufio.NewWriter(os.Stdout)
	defer out.Flush()
	if err := run(out, flag.Args(), opt); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func run(w io.Writer, args []string, opt *option) error {
	if err := addRules(opt.rules); err != nil {
		return err
	}
	prog, typeErrs, err := typeconv.Load(loader.Config{}, args)
	if err != nil {
		return err
	}
	for _, pkg := range prog.InitialPackages() {
		for _, f := range pkg.Files {
			if err := runFile(w, opt, prog, pkg, f, typeErrs); err != nil {
				return err
			}
		}
	}
	return nil
}

func runFile(w io.Writer, opt *option, prog *loader.Program, pkg *loader.PackageInfo, f *ast.File, typeErrs []types.Error) error {
	filename := prog.Fset.File(f.Pos()).Name()
	if err := typeconv.RewriteFile(prog.Fset, f, pkg, typeErrs); err != nil {
		return err
	}
	buf := new(bytes.Buffer)
	if err := format.Node(buf, prog.Fset, f); err != nil {
		return err
	}
	res := buf.Bytes()
	in, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer in.Close()
	src, err := ioutil.ReadAll(in)
	if err != nil {
		return err
	}
	if !bytes.Equal(src, res) {
		if opt.write {
			fh, err := os.Create(filename)
			if err != nil {
				return err
			}
			fh.Write(res)
			fh.Close()
		}
		if opt.doDiff {
			data, err := diff(src, res)
			if err != nil {
				return fmt.Errorf("computing diff: %s", err)
			}
			fmt.Fprintf(w, "diff %s gotypeconv/%s\n", filename, filename)
			w.Write(data)
		}
	}
	if !opt.write && !opt.doDiff {
		w.Write(res)
	}
	return nil
}

func addRules(rules []string) error {
	for _, r := range rules {
		f := strings.Split(r, "->")
		if len(f) != 2 {
			return fmt.Errorf("type conversion rule must be the form 'from -> to': %v", r)
		}
		from, to := strings.TrimSpace(f[0]), strings.TrimSpace(f[1])
		typeconv.DefaultRule.Add(from, to)
	}
	return nil
}

// copied and modified from $GOPATH/src/github.com/golang/go/src/cmd/gofmt/gofmt.go
//
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
func diff(b1, b2 []byte) (data []byte, err error) {
	f1, err := ioutil.TempFile("", "gotypeconv")
	if err != nil {
		return
	}
	defer os.Remove(f1.Name())
	defer f1.Close()

	f2, err := ioutil.TempFile("", "gotypeconv")
	if err != nil {
		return
	}
	defer os.Remove(f2.Name())
	defer f2.Close()

	f1.Write(b1)
	f2.Write(b2)

	data, err = exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput()
	if len(data) > 0 {
		// diff exits with a non-zero status when the files don't match.
		// Ignore that failure as long as we get output.
		err = nil
	}
	return

}

type strslice []string

func (ss *strslice) String() string {
	return fmt.Sprintf("%v", *ss)
}

func (ss *strslice) Set(value string) error {
	*ss = append(*ss, value)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/gobs/args"
	"github.com/gobs/cmd"
	"github.com/gobs/httpclient"
	"github.com/gobs/jsonpath"
	"github.com/gobs/simplejson"

	"fmt"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	completion_words = []string{}
	env              = map[string]string{}

	reFieldValue = regexp.MustCompile(`(\w[\d\w-]*)(=(.*))?`) // field-name=value
)

func CompletionFunction(text, line string) (matches []string) {
	// for the "ls" command we let readline show real file names
	if strings.HasPrefix(line, "ls ") {
		return
	}

	// for all other commands, we pick from our list of completion words
	for _, w := range completion_words {
		if strings.HasPrefix(w, text) {
			matches = append(matches, w)
		}
	}

	return
}

func request(client *httpclient.HttpClient, method, params string, print bool) *httpclient.HttpResponse {
	env["error"] = ""
	env["body"] = ""

	options := []httpclient.RequestOption{client.Method(method)}
	args := args.ParseArgs(params)

	if len(args.Arguments) > 0 {
		options = append(options, client.Path(args.Arguments[0]))
	}

	if len(args.Arguments) > 1 {
		data := strings.Join(args.Arguments[1:], " ")
		options = append(options, client.Body(strings.NewReader(data)))
	}

	if len(args.Options) > 0 {
		options = append(options, client.StringParams(args.Options))
	}

	res, err := client.SendRequest(options...)
	if err == nil {
		err = res.ResponseError()
	}
	if err != nil {
		fmt.Println("ERROR:", err)
		env["error"] = err.Error()
	}

	body := res.Content()
	if len(body) > 0 && print {
		if strings.Contains(res.Header.Get("Content-Type"), "json") {
			jbody, err := simplejson.LoadBytes(body)
			if err != nil {
				fmt.Println(err)
			} else {
				printJson(jbody.Data())
			}
		} else {
			fmt.Println(string(body))
		}
	}

	env["body"] = string(body)
	return res
}

func headerName(s string) string {
	s = strings.ToLower(s)
	parts := strings.Split(s, "-")
	for i, p := range parts {
		if len(p) > 0 {
			parts[i] = strings.ToUpper(p[0:1]) + p[1:]
		}
	}
	return strings.Join(parts, "-")
}

func unquote(s string) string {
	if res, err := strconv.Unquote(strings.TrimSpace(s)); err == nil {
		return res
	}

	return s
}

func printJson(v interface{}) {
	fmt.Println(simplejson.MustDumpString(v, simplejson.Indent("  ")))
}

func main() {
	var interrupted bool
	var client = httpclient.NewHttpClient("")

	client.UserAgent = "httpclient/0.1"

	commander := &cmd.Cmd{
		HistoryFile: ".httpclient_history",
		Complete:    CompletionFunction,
		EnableShell: true,
		Interrupt:   func(sig os.Signal) bool { interrupted = true; return false },
	}

	commander.Init()

	commander.Vars = env
	commander.SetVar("print", true)

	commander.Add(cmd.Command{
		"base",
		`base [url]`,
		func(line string) (stop bool) {
			if line != "" {
				val, err := url.Parse(line)
				if err != nil {
					fmt.Println(err)
					return
				}

				client.BaseURL = val
				commander.SetPrompt(fmt.Sprintf("%v> ", client.BaseURL), 40)
				if !commander.GetBoolVar("print") {
					return
				}
			}

			fmt.Println("base", client.BaseURL)
			return
		},
		nil})

	commander.Add(cmd.Command{
		"insecure",
		`insecure [true|false]`,
		func(line string) (stop bool) {
			if line != "" {
				val, err := strconv.ParseBool(line)
				if err != nil {
					fmt.Println(err)
					return
				}

				client.AllowInsecure(val)
			}

			// assume if there is a transport, it's because we set AllowInsecure
			fmt.Println("insecure", client.GetTransport() != nil)

			return
		},
		nil})

	commander.Add(cmd.Command{
		"timeout",
		`timeout [duration]`,
		func(line string) (stop bool) {
			if line != "" {
				val, err := time.ParseDuration(line)
				if err != nil {
					fmt.Println(err)
					return
				}

				client.SetTimeout(val)
			}

			fmt.Println("timeout", client.GetTimeout())
			return
		},
		nil})

	commander.Add(cmd.Command{
		"verbose",
		`verbose [true|false]`,
		func(line string) (stop bool) {
			if line != "" {
				val, err := strconv.ParseBool(line)
				if err != nil {
					fmt.Println(err)
					return
				}

				client.Verbose = val
			}

			fmt.Println("Verbose", client.Verbose)
			return
		},
		nil})

	commander.Add(cmd.Command{
		"agent",
		`agent user-agent-string`,
		func(line string) (stop bool) {
			if line != "" {
				client.UserAgent = line
			}

			fmt.Println("User-Agent:", client.UserAgent)
			return
		},
		nil})

	commander.Add(cmd.Command{
		"header",
		`header [name [value]]`,
		func(line string) (stop bool) {
			if line == "" {
				if len(client.Headers) == 0 {
					fmt.Println("No headers")
				} else {
					fmt.Println("Headers:")
					for k, v := range client.Headers {
						fmt.Printf("  %v: %v\n", k, v)
					}
				}

				return
			}

			parts := args.GetArgsN(line, 2)
			name := headerName(parts[0])

			if len(parts) == 2 {
				client.Headers[name] = unquote(parts[1])
				if !commander.GetBoolVar("print") {
					return
				}
			}

			fmt.Printf("%v: %v\n", name, client.Headers[name])
			return
		},
		nil})

	commander.Add(cmd.Command{"head",
		`
                head [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			res := request(client, "head", line, false)
			if res != nil {
				printJson(res.Header)
			}
			return
		},
		nil})

	commander.Add(cmd.Command{"get",
		`
                get [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			request(client, "get", line, commander.GetBoolVar("print"))
			return
		},
		nil})

	commander.Add(cmd.Command{"post",
		`
                post [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			request(client, "post", line, commander.GetBoolVar("print"))
			return
		},
		nil})

	commander.Add(cmd.Command{"put",
		`
                put [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			request(client, "put", line, commander.GetBoolVar("print"))
			return
		},
		nil})

	commander.Add(cmd.Command{"delete",
		`
                delete [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			request(client, "delete", line, commander.GetBoolVar("print"))
			return
		},
		nil})

	commander.Add(cmd.Command{"json",
		`
                json field1=value1 field2=value... // json object
                json [value, value...]             // json array
                `,
		func(line string) (stop bool) {
			var res interface{}
			jsonmap := true

			if strings.HasPrefix(line, "[") {
				jsonmap = false

				line = line[1:]
				if strings.HasSuffix(line, "]") {
					line = line[:len(line)-1]
				}
				line = strings.TrimSpace(line)
			}

			fields := args.GetArgs(line)

			parseValue := func(v string) (interface{}, error) {
				switch {
				case strings.HasPrefix(v, "{") || strings.HasPrefix(v, "["):
					j, err := simplejson.LoadString(v)
					if err != nil {
						return nil, fmt.Errorf("error parsing %q", v)
					} else {
						return j.Data(), nil
					}

				case v == "true":
					return true, nil

				case v == "false":
					return false, nil

				case v == "null":
					return nil, nil

				default:
					return v, nil
				}
			}

			if jsonmap {
				var err error
				mres := map[string]interface{}{}

				for _, f := range fields {
					matches := reFieldValue.FindStringSubmatch(f)
					if len(matches) > 0 { // [field=value field =value value]
						name, value := matches[1], matches[3]
						mres[name], err = parseValue(value)

						if err != nil {
							fmt.Println(err)
							env["error"] = err.Error()
							return
						}
					} else {
						fmt.Println("invalid name=value pair:", f)
						env["error"] = "invalid name=value pair"
						return
					}
				}

				res = mres
			} else {
				var ares []interface{}

				for _, f := range fields {
					v, err := parseValue(f)
					if err != nil {
						fmt.Println(err)
						env["error"] = err.Error()
						return
					}

					ares = append(ares, v)
				}
			}

			if commander.GetBoolVar("print") {
				printJson(res)
			}

			env["error"] = ""
			env["json"] = unquote(simplejson.MustDumpString(res))
			return
		},
		nil})

	commander.Add(cmd.Command{
		"jsonpath",
		`jsonpath [-e] [-c] path {json}`,
		func(line string) (stop bool) {
			var joptions jsonpath.ProcessOptions

			options, line := args.GetOptions(line)
			for _, o := range options {
				if o == "-e" || o == "--enhanced" {
					joptions |= jsonpath.Enhanced
				} else if o == "-c" || o == "--collapse" {
					joptions |= jsonpath.Collapse
				} else {
					line = "" // to force an error
					break
				}
			}

			parts := args.GetArgsN(line, 2)
			if len(parts) != 2 {
				fmt.Println("use: jsonpath [-e|--enhanced] path {json}")
				env["error"] = "invalid-usage"
				return
			}

			path := parts[0]
			if !strings.HasPrefix(path, "$.") {
				path = "$." + path
			}

			jbody, err := simplejson.LoadString(parts[1])
			if err != nil {
				fmt.Println("json:", err)
				env["error"] = err.Error()
				return
			}

			jp := jsonpath.NewProcessor()
			if !jp.Parse(path) {
				env["error"] = fmt.Sprintf("failed to parse %q", path)
				return // syntax error
			}

			res := jp.Process(jbody, joptions)
			if commander.GetBoolVar("print") {
				printJson(res)
			}
			env["error"] = ""
			env["json"] = unquote(simplejson.MustDumpString(res))
			return
		},
		nil})

	commander.Add(cmd.Command{
		"format",
		`format object`,
		func(line string) (stop bool) {
			jbody, err := simplejson.LoadString(line)
			if err != nil {
				fmt.Println("json:", err)
				env["error"] = err.Error()
				return
			}

			printJson(jbody.Data())
			return
		},
		nil})
	commander.Add(cmd.Command{
		"exit",
		`exit script`,
		func(line string) (stop bool) {
			fmt.Println("goodbye!")
			return true
		},
		nil})

	commander.Commands["set"] = commander.Commands["var"]

	switch len(os.Args) {
	case 1: // program name only
		break

	case 2: // one arg - expect URL or @filename
		cmd := os.Args[1]
		if !strings.HasPrefix(cmd, "@") {
			cmd = "base " + cmd
		}

		commander.OneCmd(cmd)

	default:
		fmt.Println("usage:", os.Args[0], "[base-url]")
		return
	}

	commander.CmdLoop()
}
<commit_msg>parse more types in json command (quoted string, integers and floats) and make "print" option local<commit_after>package main

import (
	"github.com/gobs/args"
	"github.com/gobs/cmd"
	"github.com/gobs/httpclient"
	"github.com/gobs/jsonpath"
	"github.com/gobs/simplejson"

	"fmt"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	completion_words = []string{}
	env              = map[string]string{}

	reFieldValue = regexp.MustCompile(`(\w[\d\w-]*)(=(.*))?`) // field-name=value
)

func CompletionFunction(text, line string) (matches []string) {
	// for the "ls" command we let readline show real file names
	if strings.HasPrefix(line, "ls ") {
		return
	}

	// for all other commands, we pick from our list of completion words
	for _, w := range completion_words {
		if strings.HasPrefix(w, text) {
			matches = append(matches, w)
		}
	}

	return
}

func request(client *httpclient.HttpClient, method, params string, print bool) *httpclient.HttpResponse {
	env["error"] = ""
	env["body"] = ""

	options := []httpclient.RequestOption{client.Method(method)}
	args := args.ParseArgs(params)

	if len(args.Arguments) > 0 {
		options = append(options, client.Path(args.Arguments[0]))
	}

	if len(args.Arguments) > 1 {
		data := strings.Join(args.Arguments[1:], " ")
		options = append(options, client.Body(strings.NewReader(data)))
	}

	if len(args.Options) > 0 {
		options = append(options, client.StringParams(args.Options))
	}

	res, err := client.SendRequest(options...)
	if err == nil {
		err = res.ResponseError()
	}
	if err != nil {
		fmt.Println("ERROR:", err)
		env["error"] = err.Error()
	}

	body := res.Content()
	if len(body) > 0 && print {
		if strings.Contains(res.Header.Get("Content-Type"), "json") {
			jbody, err := simplejson.LoadBytes(body)
			if err != nil {
				fmt.Println(err)
			} else {
				printJson(jbody.Data())
			}
		} else {
			fmt.Println(string(body))
		}
	}

	env["body"] = string(body)
	return res
}

func headerName(s string) string {
	s = strings.ToLower(s)
	parts := strings.Split(s, "-")
	for i, p := range parts {
		if len(p) > 0 {
			parts[i] = strings.ToUpper(p[0:1]) + p[1:]
		}
	}
	return strings.Join(parts, "-")
}

func unquote(s string) string {
	if res, err := strconv.Unquote(strings.TrimSpace(s)); err == nil {
		return res
	}

	return s
}

func printJson(v interface{}) {
	fmt.Println(simplejson.MustDumpString(v, simplejson.Indent("  ")))
}

func main() {
	var interrupted bool
	var client = httpclient.NewHttpClient("")

	client.UserAgent = "httpclient/0.1"

	commander := &cmd.Cmd{
		HistoryFile: ".httpclient_history",
		Complete:    CompletionFunction,
		EnableShell: true,
		Interrupt:   func(sig os.Signal) bool { interrupted = true; return false },
	}

	commander.Init()

	commander.Vars = env
	commander.SetVar("print", true, false)

	commander.Add(cmd.Command{
		"base",
		`base [url]`,
		func(line string) (stop bool) {
			if line != "" {
				val, err := url.Parse(line)
				if err != nil {
					fmt.Println(err)
					return
				}

				client.BaseURL = val
				commander.SetPrompt(fmt.Sprintf("%v> ", client.BaseURL), 40)
				if !commander.GetBoolVar("print", true) {
					return
				}
			}

			fmt.Println("base", client.BaseURL)
			return
		},
		nil})

	commander.Add(cmd.Command{
		"insecure",
		`insecure [true|false]`,
		func(line string) (stop bool) {
			if line != "" {
				val, err := strconv.ParseBool(line)
				if err != nil {
					fmt.Println(err)
					return
				}

				client.AllowInsecure(val)
			}

			// assume if there is a transport, it's because we set AllowInsecure
			fmt.Println("insecure", client.GetTransport() != nil)

			return
		},
		nil})

	commander.Add(cmd.Command{
		"timeout",
		`timeout [duration]`,
		func(line string) (stop bool) {
			if line != "" {
				val, err := time.ParseDuration(line)
				if err != nil {
					fmt.Println(err)
					return
				}

				client.SetTimeout(val)
			}

			fmt.Println("timeout", client.GetTimeout())
			return
		},
		nil})

	commander.Add(cmd.Command{
		"verbose",
		`verbose [true|false]`,
		func(line string) (stop bool) {
			if line != "" {
				val, err := strconv.ParseBool(line)
				if err != nil {
					fmt.Println(err)
					return
				}

				client.Verbose = val
			}

			fmt.Println("Verbose", client.Verbose)
			return
		},
		nil})

	commander.Add(cmd.Command{
		"agent",
		`agent user-agent-string`,
		func(line string) (stop bool) {
			if line != "" {
				client.UserAgent = line
			}

			fmt.Println("User-Agent:", client.UserAgent)
			return
		},
		nil})

	commander.Add(cmd.Command{
		"header",
		`header [name [value]]`,
		func(line string) (stop bool) {
			if line == "" {
				if len(client.Headers) == 0 {
					fmt.Println("No headers")
				} else {
					fmt.Println("Headers:")
					for k, v := range client.Headers {
						fmt.Printf("  %v: %v\n", k, v)
					}
				}

				return
			}

			parts := args.GetArgsN(line, 2)
			name := headerName(parts[0])

			if len(parts) == 2 {
				client.Headers[name] = unquote(parts[1])
				if !commander.GetBoolVar("print", true) {
					return
				}
			}

			fmt.Printf("%v: %v\n", name, client.Headers[name])
			return
		},
		nil})

	commander.Add(cmd.Command{"head",
		`
                head [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			res := request(client, "head", line, false)
			if res != nil {
				printJson(res.Header)
			}
			return
		},
		nil})

	commander.Add(cmd.Command{"get",
		`
                get [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			request(client, "get", line, commander.GetBoolVar("print", true))
			return
		},
		nil})

	commander.Add(cmd.Command{"post",
		`
                post [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			request(client, "post", line, commander.GetBoolVar("print", true))
			return
		},
		nil})

	commander.Add(cmd.Command{"put",
		`
                put [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			request(client, "put", line, commander.GetBoolVar("print", true))
			return
		},
		nil})

	commander.Add(cmd.Command{"delete",
		`
                delete [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			request(client, "delete", line, commander.GetBoolVar("print", true))
			return
		},
		nil})

	commander.Add(cmd.Command{"json",
		`
                json field1=value1 field2=value... // json object
                json [value, value...]             // json array
                `,
		func(line string) (stop bool) {
			var res interface{}
			jsonmap := true

			if strings.HasPrefix(line, "[") {
				jsonmap = false

				line = line[1:]
				if strings.HasSuffix(line, "]") {
					line = line[:len(line)-1]
				}
				line = strings.TrimSpace(line)
			}

			fields := args.GetArgs(line)

			parseValue := func(v string) (interface{}, error) {
				switch {
				case strings.HasPrefix(v, "{") || strings.HasPrefix(v, "["):
					j, err := simplejson.LoadString(v)
					if err != nil {
						return nil, fmt.Errorf("error parsing %q", v)
					} else {
						return j.Data(), nil
					}

				case strings.HasPrefix(v, `"`):
					return strings.Trim(v, `"`), nil

				case strings.HasPrefix(v, `'`):
					return strings.Trim(v, `'`), nil

				case v == "":
					return v, nil

				case v == "true":
					return true, nil

				case v == "false":
					return false, nil

				case v == "null":
					return nil, nil

				default:
					if i, err := strconv.ParseInt(v, 10, 64); err == nil {
						return i, nil
					}
					if f, err := strconv.ParseFloat(v, 64); err == nil {
						return f, nil
					}

					return v, nil
				}
			}

			if jsonmap {
				var err error
				mres := map[string]interface{}{}

				for _, f := range fields {
					matches := reFieldValue.FindStringSubmatch(f)
					if len(matches) > 0 { // [field=value field =value value]
						name, value := matches[1], matches[3]
						mres[name], err = parseValue(value)

						if err != nil {
							fmt.Println(err)
							env["error"] = err.Error()
							return
						}
					} else {
						fmt.Println("invalid name=value pair:", f)
						env["error"] = "invalid name=value pair"
						return
					}
				}

				res = mres
			} else {
				var ares []interface{}

				for _, f := range fields {
					v, err := parseValue(f)
					if err != nil {
						fmt.Println(err)
						env["error"] = err.Error()
						return
					}

					ares = append(ares, v)
				}
			}

			if commander.GetBoolVar("print", true) {
				printJson(res)
			}

			env["error"] = ""
			env["json"] = unquote(simplejson.MustDumpString(res))
			return
		},
		nil})

	commander.Add(cmd.Command{
		"jsonpath",
		`jsonpath [-e] [-c] path {json}`,
		func(line string) (stop bool) {
			var joptions jsonpath.ProcessOptions

			options, line := args.GetOptions(line)
			for _, o := range options {
				if o == "-e" || o == "--enhanced" {
					joptions |= jsonpath.Enhanced
				} else if o == "-c" || o == "--collapse" {
					joptions |= jsonpath.Collapse
				} else {
					line = "" // to force an error
					break
				}
			}

			parts := args.GetArgsN(line, 2)
			if len(parts) != 2 {
				fmt.Println("use: jsonpath [-e|--enhanced] path {json}")
				env["error"] = "invalid-usage"
				return
			}

			path := parts[0]
			if !strings.HasPrefix(path, "$.") {
				path = "$." + path
			}

			jbody, err := simplejson.LoadString(parts[1])
			if err != nil {
				fmt.Println("json:", err)
				env["error"] = err.Error()
				return
			}

			jp := jsonpath.NewProcessor()
			if !jp.Parse(path) {
				env["error"] = fmt.Sprintf("failed to parse %q", path)
				return // syntax error
			}

			res := jp.Process(jbody, joptions)
			if commander.GetBoolVar("print", true) {
				printJson(res)
			}
			env["error"] = ""
			env["json"] = unquote(simplejson.MustDumpString(res))
			return
		},
		nil})

	commander.Add(cmd.Command{
		"format",
		`format object`,
		func(line string) (stop bool) {
			jbody, err := simplejson.LoadString(line)
			if err != nil {
				fmt.Println("json:", err)
				env["error"] = err.Error()
				return
			}

			printJson(jbody.Data())
			return
		},
		nil})
	commander.Add(cmd.Command{
		"exit",
		`exit script`,
		func(line string) (stop bool) {
			fmt.Println("goodbye!")
			return true
		},
		nil})

	commander.Commands["set"] = commander.Commands["var"]

	switch len(os.Args) {
	case 1: // program name only
		break

	case 2: // one arg - expect URL or @filename
		cmd := os.Args[1]
		if !strings.HasPrefix(cmd, "@") {
			cmd = "base " + cmd
		}

		commander.OneCmd(cmd)

	default:
		fmt.Println("usage:", os.Args[0], "[base-url]")
		return
	}

	commander.CmdLoop()
}
<|endoftext|>


<commit_before>package main

// #include <string.h>
import "C"

import (
	"errors"
	"fmt"
	"unsafe"

	"github.com/ready-steady/linal/matrix"
	"github.com/ready-steady/persim/power"
	"github.com/ready-steady/probability/gaussian"
	"github.com/ready-steady/tempan/expint"

	"../../pkg/solver"
)

type tempTarget struct {
	problem *problem

	ic uint32 // inputs
	oc uint32 // outputs
	sc uint32 // steps

	power       *power.Distributer
	temperature *expint.Solver
}

func newTempTarget(p *problem) (target, error) {
	c := &p.config

	if c.TempAnalysis.TimeStep <= 0 {
		return nil, errors.New("the time step is invalid")
	}

	power := power.New(p.platform, p.application, c.TempAnalysis.TimeStep)
	temperature, err := expint.New(expint.Config(c.TempAnalysis))
	if err != nil {
		return nil, err
	}

	target := &tempTarget{
		problem: p,

		ic: 1 + p.zc, // +1 for time
		oc: uint32(len(c.CoreIndex)),
		sc: uint32(p.schedule.Span / c.TempAnalysis.TimeStep),

		power:       power,
		temperature: temperature,
	}

	return target, nil
}

func (t *tempTarget) String() string {
	return fmt.Sprintf("Target{inputs: %d, outputs: %d}", t.ic, t.oc)
}

func (t *tempTarget) InputsOutputs() (uint32, uint32) {
	return t.ic, t.oc
}

func (t *tempTarget) Serve(jobs <-chan solver.Job) {
	p := t.problem
	c := &p.config

	cc, uc, zc, oc, sc := p.cc, p.uc, p.zc, t.oc, t.sc
	coreIndex := c.CoreIndex

	g := gaussian.New(0, 1)
	m := p.marginals

	P := make([]float64, cc*sc)
	S := make([]float64, t.temperature.Nodes*sc)

	z := make([]float64, zc)
	u := make([]float64, uc)
	d := make([]float64, p.tc)

	for job := range jobs {
		Q := job.Data

		if Q == nil {
			Q = make([]float64, cc*sc)

			// Independent uniform to independent Gaussian
			for i := uint32(0); i < zc; i++ {
				z[i] = g.InvCDF(job.Node[1+i]) // +1 for time
			}

			// Independent Gaussian to dependent Gaussian
			matrix.Multiply(p.transform, z, u, uc, zc, 1)

			// Dependent Gaussian to dependent uniform to dependent target
			for i, tid := range c.TaskIndex {
				d[tid] = m[i].InvCDF(g.CDF(u[i]))
			}

			// FIXME: Bad, bad, bad!
			C.memset(unsafe.Pointer(&P[0]), 0, C.size_t(8*cc*sc))

			t.power.Compute(p.time.Recompute(p.schedule, d), P, sc)
			t.temperature.ComputeTransient(P, Q, S, sc)
		}

		sid := uint32(job.Node[0] * float64(sc-1))
		for i := uint32(0); i < oc; i++ {
			job.Value[i] = Q[sid*cc+uint32(coreIndex[i])]
		}

		job.Done <- solver.Result{job.Key, Q}
	}
}
<commit_msg>Updated the usage of persim/power<commit_after>package main

// #include <string.h>
import "C"

import (
	"errors"
	"fmt"
	"unsafe"

	"github.com/ready-steady/linal/matrix"
	"github.com/ready-steady/persim/power"
	"github.com/ready-steady/probability/gaussian"
	"github.com/ready-steady/tempan/expint"

	"../../pkg/solver"
)

type tempTarget struct {
	problem *problem

	ic uint32 // inputs
	oc uint32 // outputs
	sc uint32 // steps

	power       *power.Power
	temperature *expint.Solver
}

func newTempTarget(p *problem) (target, error) {
	c := &p.config

	if c.TempAnalysis.TimeStep <= 0 {
		return nil, errors.New("the time step is invalid")
	}

	power := power.New(p.platform, p.application, c.TempAnalysis.TimeStep)
	temperature, err := expint.New(expint.Config(c.TempAnalysis))
	if err != nil {
		return nil, err
	}

	target := &tempTarget{
		problem: p,

		ic: 1 + p.zc, // +1 for time
		oc: uint32(len(c.CoreIndex)),
		sc: uint32(p.schedule.Span / c.TempAnalysis.TimeStep),

		power:       power,
		temperature: temperature,
	}

	return target, nil
}

func (t *tempTarget) String() string {
	return fmt.Sprintf("Target{inputs: %d, outputs: %d}", t.ic, t.oc)
}

func (t *tempTarget) InputsOutputs() (uint32, uint32) {
	return t.ic, t.oc
}

func (t *tempTarget) Serve(jobs <-chan solver.Job) {
	p := t.problem
	c := &p.config

	cc, uc, zc, oc, sc := p.cc, p.uc, p.zc, t.oc, t.sc
	coreIndex := c.CoreIndex

	g := gaussian.New(0, 1)
	m := p.marginals

	P := make([]float64, cc*sc)
	S := make([]float64, t.temperature.Nodes*sc)

	z := make([]float64, zc)
	u := make([]float64, uc)
	d := make([]float64, p.tc)

	for job := range jobs {
		Q := job.Data

		if Q == nil {
			Q = make([]float64, cc*sc)

			// Independent uniform to independent Gaussian
			for i := uint32(0); i < zc; i++ {
				z[i] = g.InvCDF(job.Node[1+i]) // +1 for time
			}

			// Independent Gaussian to dependent Gaussian
			matrix.Multiply(p.transform, z, u, uc, zc, 1)

			// Dependent Gaussian to dependent uniform to dependent target
			for i, tid := range c.TaskIndex {
				d[tid] = m[i].InvCDF(g.CDF(u[i]))
			}

			// FIXME: Bad, bad, bad!
			C.memset(unsafe.Pointer(&P[0]), 0, C.size_t(8*cc*sc))

			t.power.Compute(p.time.Recompute(p.schedule, d), P, sc)
			t.temperature.ComputeTransient(P, Q, S, sc)
		}

		sid := uint32(job.Node[0] * float64(sc-1))
		for i := uint32(0); i < oc; i++ {
			job.Value[i] = Q[sid*cc+uint32(coreIndex[i])]
		}

		job.Done <- solver.Result{job.Key, Q}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"path"
	"unicode"

	"github.com/DeedleFake/signage"
)

type marshalFunc func(string, []signage.Bill) (io.Reader, error)

func marshalRSS(t string, bills []signage.Bill) (io.Reader, error) {
	var buf bytes.Buffer
	err := tmpl.ExecuteTemplate(&buf, "rss", map[string]interface{}{
		"Type":  t,
		"Bills": bills,
	})
	return &buf, err
}

func marshalJSON(t string, bills []signage.Bill) (io.Reader, error) {
	buf, err := json.Marshal(map[string]interface{}{
		"type":  t,
		"bills": bills,
	})
	return bytes.NewReader(buf), err
}

type getFunc func() ([]signage.Bill, error)

func handle(rw http.ResponseWriter, req *http.Request, mode string, get getFunc, marshal marshalFunc) {
	bills, err := get()
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	buf, err := marshal(mode, bills)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	_, err = io.Copy(rw, buf)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
}

var (
	marshallers = map[string]marshalFunc{
		"": marshalRSS,

		".rss":  marshalRSS,
		".json": marshalJSON,
	}

	modes = map[string]getFunc{
		"signed":  signage.GetSigned,
		"vetoed":  signage.GetVetoed,
		"pending": signage.GetPending,
	}
)

func handleList(rw http.ResponseWriter, req *http.Request) {
	var buf bytes.Buffer
	err := tmpl.ExecuteTemplate(&buf, "list", map[string]interface{}{
		"Marshallers": marshallers,
		"Modes":       modes,
	})
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	_, err = io.Copy(rw, &buf)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
}

func mux(rw http.ResponseWriter, req *http.Request) {
	name := path.Base(req.URL.Path)
	ext := path.Ext(name)

	mode := name[:len(name)-len(ext)]
	get, ok := modes[mode]
	if !ok {
		handleList(rw, req)
		return
	}

	marshal, ok := marshallers[ext]
	if !ok {
		http.Error(rw, fmt.Sprintf("Unknown format: %q", ext), http.StatusBadRequest)
		return
	}

	mode = string(unicode.ToUpper(rune(mode[0]))) + mode[1:]
	handle(rw, req, mode, get, marshal)
}

func main() {
	log.Fatalln(http.ListenAndServe(":8080", http.HandlerFunc(mux)))
}
<commit_msg>cmd/signage: Rename handle() to getBills().<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"path"
	"unicode"

	"github.com/DeedleFake/signage"
)

type marshalFunc func(string, []signage.Bill) (io.Reader, error)

func marshalRSS(t string, bills []signage.Bill) (io.Reader, error) {
	var buf bytes.Buffer
	err := tmpl.ExecuteTemplate(&buf, "rss", map[string]interface{}{
		"Type":  t,
		"Bills": bills,
	})
	return &buf, err
}

func marshalJSON(t string, bills []signage.Bill) (io.Reader, error) {
	buf, err := json.Marshal(map[string]interface{}{
		"type":  t,
		"bills": bills,
	})
	return bytes.NewReader(buf), err
}

type getFunc func() ([]signage.Bill, error)

func getBills(rw http.ResponseWriter, req *http.Request, mode string, get getFunc, marshal marshalFunc) {
	bills, err := get()
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	buf, err := marshal(mode, bills)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	_, err = io.Copy(rw, buf)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
}

var (
	marshallers = map[string]marshalFunc{
		"": marshalRSS,

		".rss":  marshalRSS,
		".json": marshalJSON,
	}

	modes = map[string]getFunc{
		"signed":  signage.GetSigned,
		"vetoed":  signage.GetVetoed,
		"pending": signage.GetPending,
	}
)

func handleList(rw http.ResponseWriter, req *http.Request) {
	var buf bytes.Buffer
	err := tmpl.ExecuteTemplate(&buf, "list", map[string]interface{}{
		"Marshallers": marshallers,
		"Modes":       modes,
	})
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	_, err = io.Copy(rw, &buf)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
}

func mux(rw http.ResponseWriter, req *http.Request) {
	name := path.Base(req.URL.Path)
	ext := path.Ext(name)

	mode := name[:len(name)-len(ext)]
	get, ok := modes[mode]
	if !ok {
		handleList(rw, req)
		return
	}

	marshal, ok := marshallers[ext]
	if !ok {
		http.Error(rw, fmt.Sprintf("Unknown format: %q", ext), http.StatusBadRequest)
		return
	}

	mode = string(unicode.ToUpper(rune(mode[0]))) + mode[1:]
	getBills(rw, req, mode, get, marshal)
}

func main() {
	log.Fatalln(http.ListenAndServe(":8080", http.HandlerFunc(mux)))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"text/tabwriter"
)

func optionTable(w io.Writer, rows [][]string) {
	tw := tabwriter.NewWriter(w, 2, 4, 2, ' ', 0)
	for _, row := range rows {
		for i, cell := range row {
			if i > 0 {
				tw.Write([]byte("\t"))
			}
			tw.Write([]byte(cell))
		}
		tw.Write([]byte("\n"))
	}
	tw.Flush()
}

func usageFor(fs *flag.FlagSet, usage string, extra string) func() {
	return func() {
		var b bytes.Buffer
		b.WriteString("Usage:\n  " + usage + "\n")

		var options [][]string
		fs.VisitAll(func(f *flag.Flag) {
			var opt = "  -" + f.Name

			if f.DefValue != "false" {
				opt += "=" + f.DefValue
			}

			options = append(options, []string{opt, f.Usage})
		})

		if len(options) > 0 {
			b.WriteString("\nOptions:\n")
			optionTable(&b, options)
		}

		fmt.Println(b.String())

		if len(extra) > 0 {
			fmt.Println(extra)
		}
	}
}
<commit_msg>Quote default flag parameters<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"text/tabwriter"
)

func optionTable(w io.Writer, rows [][]string) {
	tw := tabwriter.NewWriter(w, 2, 4, 2, ' ', 0)
	for _, row := range rows {
		for i, cell := range row {
			if i > 0 {
				tw.Write([]byte("\t"))
			}
			tw.Write([]byte(cell))
		}
		tw.Write([]byte("\n"))
	}
	tw.Flush()
}

func usageFor(fs *flag.FlagSet, usage string, extra string) func() {
	return func() {
		var b bytes.Buffer
		b.WriteString("Usage:\n  " + usage + "\n")

		var options [][]string
		fs.VisitAll(func(f *flag.Flag) {
			var opt = "  -" + f.Name

			if f.DefValue != "false" {
				opt += "=" + fmt.Sprintf("%q", f.DefValue)
			}

			options = append(options, []string{opt, f.Usage})
		})

		if len(options) > 0 {
			b.WriteString("\nOptions:\n")
			optionTable(&b, options)
		}

		fmt.Println(b.String())

		if len(extra) > 0 {
			fmt.Println(extra)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Send icmp packets to a server to test network connectivity.
//
// Synopsis:
//     ping [-hV] [-c COUNT] [-i INTERVAL] [-s PACKETSIZE] [-w DEADLINE] DESTINATION
//
// Options:
//     -6: use ipv6 (ip6:ipv6-icmp)
//     -s: data size (default: 64)
//     -c: # iterations, 0 to run forever (default)
//     -i: interval in milliseconds (default: 1000)
//     -V: version
//     -w: wait time in milliseconds (default: 100)
//     -a: Audible rings a bell when a packet is received
//     -h: help
package main

import (
	"encoding/binary"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"time"
)

var (
	net6       = flag.Bool("6", false, "use ipv4 (means ip4:icmp) or 6 (ip6:ipv6-icmp)")
	packetSize = flag.Int("s", 64, "Data size")
	iter       = flag.Uint64("c", 0, "# iterations")
	intv       = flag.Int("i", 1000, "interval in milliseconds")
	version    = flag.Bool("V", false, "version")
	wtf        = flag.Int("w", 100, "wait time in milliseconds")
	audible    = flag.Bool("a", false, "Audible rings a bell when a packet is received")
)

const (
	ICMP_TYPE_ECHO_REQUEST             = 8
	ICMP_TYPE_ECHO_REPLY               = 0
	ICMP_ECHO_REPLY_HEADER_IPV4_OFFSET = 20
)

const (
	ICMP6_TYPE_ECHO_REQUEST             = 128
	ICMP6_TYPE_ECHO_REPLY               = 129
	ICMP6_ECHO_REPLY_HEADER_IPV6_OFFSET = 40
)

func usage() {
	fmt.Fprintf(os.Stdout, "ping [-V] [-c count] [-i interval] [-s packetsize] [-w deadline] destination\n")
	os.Exit(0)
}

func showversion() {
	fmt.Fprintf(os.Stdout, "ping utility, Uroot version\n")
	os.Exit(0)
}

func optwithoutparam() {
	if *version {
		showversion()
	}
	// if we reach this point, invalid or help (-h) gets the same result
	usage()
}

func cksum(bs []byte) uint16 {
	sum := uint32(0)

	for k := 0; k < len(bs)/2; k++ {
		sum += uint32(bs[k*2]) << 8
		sum += uint32(bs[k*2+1])
	}
	if len(bs)%2 != 0 {
		sum += uint32(bs[len(bs)-1]) << 8
	}
	sum = (sum >> 16) + (sum & 0xffff)
	sum = (sum >> 16) + (sum & 0xffff)
	if sum == 0xffff {
		sum = 0
	}

	return ^uint16(sum)
}

func ping1(net6 bool, host string, i uint64, waitFor time.Duration) (string, error) {
	netname := "ip4:icmp"
	// todo: just figure out if it's an ip6 address and go from there.
	if net6 {
		netname = "ip6:ipv6-icmp"
	}
	c, derr := net.Dial(netname, host)
	if derr != nil {
		return "", fmt.Errorf("net.Dial(%v %v) failed: %v", netname, host, derr)
	}
	defer c.Close()

	if net6 {
		ipc := c.(*net.IPConn)
		if err := setupICMPv6Socket(ipc); err != nil {
			return "", fmt.Errorf("failed to set up the ICMPv6 connection: %w", err)
		}
	}

	// Send ICMP Echo Request
	c.SetDeadline(time.Now().Add(waitFor))
	msg := make([]byte, *packetSize)
	if net6 {
		msg[0] = ICMP6_TYPE_ECHO_REQUEST
	} else {
		msg[0] = ICMP_TYPE_ECHO_REQUEST
	}
	msg[1] = 0
	binary.BigEndian.PutUint16(msg[6:], uint16(i))
	binary.BigEndian.PutUint16(msg[4:], uint16(i>>16))
	binary.BigEndian.PutUint16(msg[2:], cksum(msg))
	if _, err := c.Write(msg[:]); err != nil {
		return "", fmt.Errorf("write failed: %v", err)
	}

	// Get ICMP Echo Reply
	c.SetDeadline(time.Now().Add(waitFor))
	rmsg := make([]byte, *packetSize+256)
	before := time.Now()
	amt, rerr := c.Read(rmsg[:])
	if rerr != nil {
		return "", fmt.Errorf("read failed: %v", rerr)
	}
	latency := time.Since(before)
	if !net6 {
		rmsg = rmsg[ICMP_ECHO_REPLY_HEADER_IPV4_OFFSET:]
	}
	if net6 {
		if rmsg[0] != ICMP6_TYPE_ECHO_REPLY {
			return "", fmt.Errorf("bad ICMPv6 echo reply type, got %d, want %d", rmsg[0], ICMP6_TYPE_ECHO_REPLY)
		}
	} else {
		if rmsg[0] != ICMP_TYPE_ECHO_REPLY {
			return "", fmt.Errorf("bad ICMP echo reply type, got %d, want %d", rmsg[0], ICMP_TYPE_ECHO_REPLY)
		}
	}
	cks := binary.BigEndian.Uint16(rmsg[2:])
	binary.BigEndian.PutUint16(rmsg[2:], 0)
	// only validate the checksum for IPv4. For IPv6 this *should* be done by the
	// TCP stack (and do we need to validate the checksum anyway?)
	if !net6 && cks != cksum(rmsg) {
		return "", fmt.Errorf("bad ICMP checksum: %v (expected %v)", cks, cksum(rmsg))
	}
	id := binary.BigEndian.Uint16(rmsg[4:])
	seq := binary.BigEndian.Uint16(rmsg[6:])
	rseq := uint64(id)<<16 + uint64(seq)
	if rseq != i {
		return "", fmt.Errorf("wrong sequence number %v (expected %v)", rseq, i)
	}

	return fmt.Sprintf("%d bytes from %v: icmp_seq=%v, time=%v", amt, host, i, latency), nil
}

func main() {
	flag.Parse()

	// options without parameters (right now just: -hV)
	if flag.NArg() < 1 {
		optwithoutparam()
	}
	if *packetSize < 8 {
		log.Fatalf("packet size too small (must be >= 8): %v", *packetSize)
	}

	interval := time.Duration(*intv)
	host := flag.Args()[0]

	// ping needs to run forever, except if '*iter' is not zero
	waitFor := time.Duration(*wtf) * time.Millisecond
	var i uint64
	for i = 1; *iter == 0 || i <= *iter; i++ {
		msg, err := ping1(*net6, host, i, waitFor)
		if err != nil {
			log.Fatalf("ping failed: %v", err)
		}
		if *audible {
			msg = "\a" + msg
		}
		log.Print(msg)
		time.Sleep(time.Millisecond * interval)
	}
}
<commit_msg>Update usage of ping command<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Send icmp packets to a server to test network connectivity.
//
// Synopsis:
//     ping [-hV] [-c COUNT] [-i INTERVAL] [-s PACKETSIZE] [-w DEADLINE] DESTINATION
//
// Options:
//     -6: use ipv6 (ip6:ipv6-icmp)
//     -s: data size (default: 64)
//     -c: # iterations, 0 to run forever (default)
//     -i: interval in milliseconds (default: 1000)
//     -V: version
//     -w: wait time in milliseconds (default: 100)
//     -a: Audible rings a bell when a packet is received
//     -h: help
package main

import (
	"encoding/binary"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"time"
)

var (
	net6       = flag.Bool("6", false, "use ipv4 (means ip4:icmp) or 6 (ip6:ipv6-icmp)")
	packetSize = flag.Int("s", 64, "Data size")
	iter       = flag.Uint64("c", 0, "# iterations")
	intv       = flag.Int("i", 1000, "interval in milliseconds")
	version    = flag.Bool("V", false, "version")
	wtf        = flag.Int("w", 100, "wait time in milliseconds")
	audible    = flag.Bool("a", false, "Audible rings a bell when a packet is received")
)

const (
	ICMP_TYPE_ECHO_REQUEST             = 8
	ICMP_TYPE_ECHO_REPLY               = 0
	ICMP_ECHO_REPLY_HEADER_IPV4_OFFSET = 20
)

const (
	ICMP6_TYPE_ECHO_REQUEST             = 128
	ICMP6_TYPE_ECHO_REPLY               = 129
	ICMP6_ECHO_REPLY_HEADER_IPV6_OFFSET = 40
)

func usage() {
	fmt.Fprintf(os.Stdout, "ping [-V] [-6] [-c count] [-i interval] [-s packetsize] [-w deadline] [-a audible] destination\n")
	os.Exit(0)
}

func showversion() {
	fmt.Fprintf(os.Stdout, "ping utility, Uroot version\n")
	os.Exit(0)
}

func optwithoutparam() {
	if *version {
		showversion()
	}
	// if we reach this point, invalid or help (-h) gets the same result
	usage()
}

func cksum(bs []byte) uint16 {
	sum := uint32(0)

	for k := 0; k < len(bs)/2; k++ {
		sum += uint32(bs[k*2]) << 8
		sum += uint32(bs[k*2+1])
	}
	if len(bs)%2 != 0 {
		sum += uint32(bs[len(bs)-1]) << 8
	}
	sum = (sum >> 16) + (sum & 0xffff)
	sum = (sum >> 16) + (sum & 0xffff)
	if sum == 0xffff {
		sum = 0
	}

	return ^uint16(sum)
}

func ping1(net6 bool, host string, i uint64, waitFor time.Duration) (string, error) {
	netname := "ip4:icmp"
	// todo: just figure out if it's an ip6 address and go from there.
	if net6 {
		netname = "ip6:ipv6-icmp"
	}
	c, derr := net.Dial(netname, host)
	if derr != nil {
		return "", fmt.Errorf("net.Dial(%v %v) failed: %v", netname, host, derr)
	}
	defer c.Close()

	if net6 {
		ipc := c.(*net.IPConn)
		if err := setupICMPv6Socket(ipc); err != nil {
			return "", fmt.Errorf("failed to set up the ICMPv6 connection: %w", err)
		}
	}

	// Send ICMP Echo Request
	c.SetDeadline(time.Now().Add(waitFor))
	msg := make([]byte, *packetSize)
	if net6 {
		msg[0] = ICMP6_TYPE_ECHO_REQUEST
	} else {
		msg[0] = ICMP_TYPE_ECHO_REQUEST
	}
	msg[1] = 0
	binary.BigEndian.PutUint16(msg[6:], uint16(i))
	binary.BigEndian.PutUint16(msg[4:], uint16(i>>16))
	binary.BigEndian.PutUint16(msg[2:], cksum(msg))
	if _, err := c.Write(msg[:]); err != nil {
		return "", fmt.Errorf("write failed: %v", err)
	}

	// Get ICMP Echo Reply
	c.SetDeadline(time.Now().Add(waitFor))
	rmsg := make([]byte, *packetSize+256)
	before := time.Now()
	amt, rerr := c.Read(rmsg[:])
	if rerr != nil {
		return "", fmt.Errorf("read failed: %v", rerr)
	}
	latency := time.Since(before)
	if !net6 {
		rmsg = rmsg[ICMP_ECHO_REPLY_HEADER_IPV4_OFFSET:]
	}
	if net6 {
		if rmsg[0] != ICMP6_TYPE_ECHO_REPLY {
			return "", fmt.Errorf("bad ICMPv6 echo reply type, got %d, want %d", rmsg[0], ICMP6_TYPE_ECHO_REPLY)
		}
	} else {
		if rmsg[0] != ICMP_TYPE_ECHO_REPLY {
			return "", fmt.Errorf("bad ICMP echo reply type, got %d, want %d", rmsg[0], ICMP_TYPE_ECHO_REPLY)
		}
	}
	cks := binary.BigEndian.Uint16(rmsg[2:])
	binary.BigEndian.PutUint16(rmsg[2:], 0)
	// only validate the checksum for IPv4. For IPv6 this *should* be done by the
	// TCP stack (and do we need to validate the checksum anyway?)
	if !net6 && cks != cksum(rmsg) {
		return "", fmt.Errorf("bad ICMP checksum: %v (expected %v)", cks, cksum(rmsg))
	}
	id := binary.BigEndian.Uint16(rmsg[4:])
	seq := binary.BigEndian.Uint16(rmsg[6:])
	rseq := uint64(id)<<16 + uint64(seq)
	if rseq != i {
		return "", fmt.Errorf("wrong sequence number %v (expected %v)", rseq, i)
	}

	return fmt.Sprintf("%d bytes from %v: icmp_seq=%v, time=%v", amt, host, i, latency), nil
}

func main() {
	flag.Parse()

	// options without parameters (right now just: -hV)
	if flag.NArg() < 1 {
		optwithoutparam()
	}
	if *packetSize < 8 {
		log.Fatalf("packet size too small (must be >= 8): %v", *packetSize)
	}

	interval := time.Duration(*intv)
	host := flag.Args()[0]

	// ping needs to run forever, except if '*iter' is not zero
	waitFor := time.Duration(*wtf) * time.Millisecond
	var i uint64
	for i = 1; *iter == 0 || i <= *iter; i++ {
		msg, err := ping1(*net6, host, i, waitFor)
		if err != nil {
			log.Fatalf("ping failed: %v", err)
		}
		if *audible {
			msg = "\a" + msg
		}
		log.Print(msg)
		time.Sleep(time.Millisecond * interval)
	}
}
<|endoftext|>


<commit_before>package commitlog

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"

	"github.com/pkg/errors"
)

var (
	ErrSegmentNotFound = errors.New("segment not found")
)

type CommitLog struct {
	Options
	cleaner        Cleaner
	name           string
	mu             sync.RWMutex
	segments       []*Segment
	vActiveSegment atomic.Value
}

type Options struct {
	Path            string
	MaxSegmentBytes int64
	MaxLogBytes     int64
}

func New(opts Options) (*CommitLog, error) {
	if opts.Path == "" {
		return nil, errors.New("path is empty")
	}

	if opts.MaxSegmentBytes == 0 {
		// TODO default here
	}

	path, _ := filepath.Abs(opts.Path)
	l := &CommitLog{
		Options: opts,
		name:    filepath.Base(path),
		cleaner: NewDeleteCleaner(opts.MaxLogBytes),
	}

	return l, nil
}

func (l *CommitLog) Init() error {
	err := os.MkdirAll(l.Path, 0755)
	if err != nil {
		return errors.Wrap(err, "mkdir failed")
	}
	return nil
}

func (l *CommitLog) Open() error {
	files, err := ioutil.ReadDir(l.Path)
	if err != nil {
		return errors.Wrap(err, "read dir failed")
	}
	for _, file := range files {
		if !strings.HasSuffix(file.Name(), ".log") {
			continue
		}
		offsetStr := strings.TrimSuffix(file.Name(), ".log")
		baseOffset, err := strconv.Atoi(offsetStr)
		segment, err := NewSegment(l.Path, int64(baseOffset), l.MaxSegmentBytes)
		if err != nil {
			return err
		}
		l.segments = append(l.segments, segment)
	}
	if len(l.segments) == 0 {
		segment, err := NewSegment(l.Path, 0, l.MaxSegmentBytes)
		if err != nil {
			return err
		}
		l.segments = append(l.segments, segment)
	}
	l.vActiveSegment.Store(l.segments[len(l.segments)-1])
	return nil
}

func (l *CommitLog) Append(ms MessageSet) (offset int64, err error) {
	if l.checkSplit() {
		if err := l.split(); err != nil {
			return offset, err
		}
	}
	position := l.activeSegment().Position
	offset = l.activeSegment().NextOffset
	ms.PutOffset(offset)
	if _, err := l.activeSegment().Write(ms); err != nil {
		return offset, err
	}
	e := Entry{
		Offset:   offset,
		Position: position,
	}
	if err := l.activeSegment().Index.WriteEntry(e); err != nil {
		return offset, err
	}
	return offset, nil
}

func (l *CommitLog) Read(p []byte) (n int, err error) {
	l.mu.Lock()
	defer l.mu.Unlock()
	return l.activeSegment().Read(p)
}
func (l *CommitLog) NewestOffset() int64 {
	return l.activeSegment().NextOffset
}

func (l *CommitLog) OldestOffset() int64 {
	l.mu.RLock()
	defer l.mu.RUnlock()
	return l.segments[0].BaseOffset
}

func (l *CommitLog) activeSegment() *Segment {
	return l.vActiveSegment.Load().(*Segment)
}

func (l *CommitLog) Close() error {
	l.mu.Lock()
	defer l.mu.Unlock()
	for _, segment := range l.segments {
		if err := segment.Close(); err != nil {
			return err
		}
	}
	return nil
}

func (l *CommitLog) DeleteAll() error {
	if err := l.Close(); err != nil {
		return err
	}
	return os.RemoveAll(l.Path)
}

func (l *CommitLog) TruncateTo(offset int64) error {
	l.mu.Lock()
	defer l.mu.Unlock()
	var segments []*Segment
	for _, segment := range l.segments {
		if segment.BaseOffset < offset {
			if err := segment.Delete(); err != nil {
				return err
			}
		} else {
			segments = append(segments, segment)
		}
	}
	l.segments = segments
	return nil
}

func (l *CommitLog) Segments() []*Segment {
	l.mu.Lock()
	defer l.mu.Unlock()
	return l.segments
}

func (l *CommitLog) checkSplit() bool {
	return l.activeSegment().IsFull()
}

func (l *CommitLog) split() error {
	seg, err := NewSegment(l.Path, l.NewestOffset(), l.MaxSegmentBytes)
	if err != nil {
		return err
	}
	l.mu.Lock()
	l.segments = append(l.segments, seg)
	segments, err := l.cleaner.Clean(l.segments)
	if err != nil {
		return err
	}
	l.segments = segments
	l.mu.Unlock()
	l.vActiveSegment.Store(seg)
	return nil
}
<commit_msg>rename var<commit_after>package commitlog

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"

	"github.com/pkg/errors"
)

var (
	ErrSegmentNotFound = errors.New("segment not found")
)

type CommitLog struct {
	Options
	cleaner        Cleaner
	name           string
	mu             sync.RWMutex
	segments       []*Segment
	vActiveSegment atomic.Value
}

type Options struct {
	Path            string
	MaxSegmentBytes int64
	MaxLogBytes     int64
}

func New(opts Options) (*CommitLog, error) {
	if opts.Path == "" {
		return nil, errors.New("path is empty")
	}

	if opts.MaxSegmentBytes == 0 {
		// TODO default here
	}

	path, _ := filepath.Abs(opts.Path)
	l := &CommitLog{
		Options: opts,
		name:    filepath.Base(path),
		cleaner: NewDeleteCleaner(opts.MaxLogBytes),
	}

	return l, nil
}

func (l *CommitLog) Init() error {
	err := os.MkdirAll(l.Path, 0755)
	if err != nil {
		return errors.Wrap(err, "mkdir failed")
	}
	return nil
}

func (l *CommitLog) Open() error {
	files, err := ioutil.ReadDir(l.Path)
	if err != nil {
		return errors.Wrap(err, "read dir failed")
	}
	for _, file := range files {
		if !strings.HasSuffix(file.Name(), ".log") {
			continue
		}
		offsetStr := strings.TrimSuffix(file.Name(), ".log")
		baseOffset, err := strconv.Atoi(offsetStr)
		segment, err := NewSegment(l.Path, int64(baseOffset), l.MaxSegmentBytes)
		if err != nil {
			return err
		}
		l.segments = append(l.segments, segment)
	}
	if len(l.segments) == 0 {
		segment, err := NewSegment(l.Path, 0, l.MaxSegmentBytes)
		if err != nil {
			return err
		}
		l.segments = append(l.segments, segment)
	}
	l.vActiveSegment.Store(l.segments[len(l.segments)-1])
	return nil
}

func (l *CommitLog) Append(ms MessageSet) (offset int64, err error) {
	if l.checkSplit() {
		if err := l.split(); err != nil {
			return offset, err
		}
	}
	position := l.activeSegment().Position
	offset = l.activeSegment().NextOffset
	ms.PutOffset(offset)
	if _, err := l.activeSegment().Write(ms); err != nil {
		return offset, err
	}
	e := Entry{
		Offset:   offset,
		Position: position,
	}
	if err := l.activeSegment().Index.WriteEntry(e); err != nil {
		return offset, err
	}
	return offset, nil
}

func (l *CommitLog) Read(p []byte) (n int, err error) {
	l.mu.Lock()
	defer l.mu.Unlock()
	return l.activeSegment().Read(p)
}
func (l *CommitLog) NewestOffset() int64 {
	return l.activeSegment().NextOffset
}

func (l *CommitLog) OldestOffset() int64 {
	l.mu.RLock()
	defer l.mu.RUnlock()
	return l.segments[0].BaseOffset
}

func (l *CommitLog) activeSegment() *Segment {
	return l.vActiveSegment.Load().(*Segment)
}

func (l *CommitLog) Close() error {
	l.mu.Lock()
	defer l.mu.Unlock()
	for _, segment := range l.segments {
		if err := segment.Close(); err != nil {
			return err
		}
	}
	return nil
}

func (l *CommitLog) DeleteAll() error {
	if err := l.Close(); err != nil {
		return err
	}
	return os.RemoveAll(l.Path)
}

func (l *CommitLog) TruncateTo(offset int64) error {
	l.mu.Lock()
	defer l.mu.Unlock()
	var segments []*Segment
	for _, segment := range l.segments {
		if segment.BaseOffset < offset {
			if err := segment.Delete(); err != nil {
				return err
			}
		} else {
			segments = append(segments, segment)
		}
	}
	l.segments = segments
	return nil
}

func (l *CommitLog) Segments() []*Segment {
	l.mu.Lock()
	defer l.mu.Unlock()
	return l.segments
}

func (l *CommitLog) checkSplit() bool {
	return l.activeSegment().IsFull()
}

func (l *CommitLog) split() error {
	segment, err := NewSegment(l.Path, l.NewestOffset(), l.MaxSegmentBytes)
	if err != nil {
		return err
	}
	l.mu.Lock()
	segments := append(l.segments, segment)
	segments, err = l.cleaner.Clean(segments)
	if err != nil {
		return err
	}
	l.segments = segments
	l.mu.Unlock()
	l.vActiveSegment.Store(segment)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	gcontext "context"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/Microsoft/hcsshim/internal/hcs"
	"github.com/Microsoft/hcsshim/internal/oc"
	"github.com/containerd/containerd/runtime/v2/task"
	"github.com/gogo/protobuf/proto"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
	"go.opencensus.io/trace"
)

// LimitedRead reads at max `readLimitBytes` bytes from the file at path `filePath`. If the file has
// more than `readLimitBytes` bytes of data then first `readLimitBytes` will be returned.
func limitedRead(filePath string, readLimitBytes int64) ([]byte, error) {
	f, err := os.Open(filePath)
	if err != nil {
		return nil, errors.Wrapf(err, "limited read failed to open file: %s", filePath)
	}
	defer f.Close()
	if fi, err := f.Stat(); err == nil {
		if fi.Size() < readLimitBytes {
			readLimitBytes = fi.Size()
		}
		buf := make([]byte, readLimitBytes)
		_, err := f.Read(buf)
		if err != nil {
			return []byte{}, errors.Wrapf(err, "limited read failed during file read: %s", filePath)
		}
		return buf, nil
	}
	return []byte{}, errors.Wrapf(err, "limited read failed during file stat: %s", filePath)
}

var deleteCommand = cli.Command{
	Name: "delete",
	Usage: `
This command allows containerd to delete any container resources created, mounted, and/or run by a shim when containerd can no longer communicate over rpc. This happens if a shim is SIGKILL'd with a running container. These resources will need to be cleaned up when containerd loses the connection to a shim. This is also used when containerd boots and reconnects to shims. If a bundle is still on disk but containerd cannot connect to a shim, the delete command is invoked.

The delete command will be executed in the container's bundle as its cwd.
`,
	SkipArgReorder: true,
	Action: func(context *cli.Context) (err error) {
		// We cant write anything to stdout for this cmd other than the
		// task.DeleteResponse by protcol. We can write to stderr which will be
		// warning logged in containerd.

		ctx, span := trace.StartSpan(gcontext.Background(), "delete")
		defer span.End()
		defer func() { oc.SetSpanStatus(span, err) }()

		bundleFlag := context.GlobalString("bundle")
		if bundleFlag == "" {
			return errors.New("bundle is required")
		}

		// hcsshim shim writes panic logs in the bundle directory in a file named "panic.log"
		// log those messages (if any) on stderr so that it shows up in containerd's log.
		// This should be done as the first thing so that we don't miss any panic logs even if
		// something goes wrong during delete op.
		// The file can be very large so read only first 1MB of data.
		readLimit := int64(1024 * 1024) // 1MB
		logBytes, err := limitedRead(filepath.Join(bundleFlag, "panic.log"), readLimit)
		if err == nil && len(logBytes) > 0 {
			if int64(len(logBytes)) == readLimit {
				logrus.Warnf("shim panic log file %s is larger than 1MB, logging only first 1MB", filepath.Join(bundleFlag, "panic.log"))
			}
			logrus.WithField("log", string(logBytes)).Warn("found shim panic logs during delete")
		} else if err != nil && !errors.Is(err, os.ErrNotExist) {
			logrus.WithError(err).Warn("failed to open shim panic log")
		}

		// Attempt to find the hcssystem for this bundle and terminate it.
		if sys, _ := hcs.OpenComputeSystem(ctx, idFlag); sys != nil {
			defer sys.Close()
			if err := sys.Terminate(ctx); err != nil {
				fmt.Fprintf(os.Stderr, "failed to terminate '%s': %v", idFlag, err)
			} else {
				ch := make(chan error, 1)
				go func() { ch <- sys.Wait() }()
				t := time.NewTimer(time.Second * 30)
				select {
				case <-t.C:
					sys.Close()
					return fmt.Errorf("timed out waiting for '%s' to terminate", idFlag)
				case err := <-ch:
					t.Stop()
					if err != nil {
						fmt.Fprintf(os.Stderr, "failed to wait for '%s' to terminate: %v", idFlag, err)
					}
				}
			}
		}

		// Determine if the config file was a POD and if so kill the whole POD.
		if s, err := getSpecAnnotations(bundleFlag); err != nil {
			if !os.IsNotExist(err) {
				return err
			}
		} else {
			if containerType := s["io.kubernetes.cri.container-type"]; containerType == "container" {
				if sandboxID := s["io.kubernetes.cri.sandbox-id"]; sandboxID != "" {
					if sys, _ := hcs.OpenComputeSystem(ctx, sandboxID); sys != nil {
						if err := sys.Terminate(ctx); err != nil {
							fmt.Fprintf(os.Stderr, "failed to terminate '%s': %v", idFlag, err)
						} else if err := sys.Wait(); err != nil {
							fmt.Fprintf(os.Stderr, "failed to wait for '%s' to terminate: %v", idFlag, err)
						}
						sys.Close()
					}
				}
			}
		}

		// Remove the bundle on disk
		if err := os.RemoveAll(bundleFlag); err != nil && !os.IsNotExist(err) {
			return err
		}

		if data, err := proto.Marshal(&task.DeleteResponse{
			ExitedAt:   time.Now(),
			ExitStatus: 255,
		}); err != nil {
			return err
		} else {
			if _, err := os.Stdout.Write(data); err != nil {
				return err
			}
		}
		return nil
	},
}
<commit_msg>shim: Clean up delete invocation behavior<commit_after>package main

import (
	gcontext "context"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/Microsoft/hcsshim/internal/hcs"
	"github.com/Microsoft/hcsshim/internal/oc"
	"github.com/containerd/containerd/runtime/v2/task"
	"github.com/gogo/protobuf/proto"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
	"go.opencensus.io/trace"
)

// LimitedRead reads at max `readLimitBytes` bytes from the file at path `filePath`. If the file has
// more than `readLimitBytes` bytes of data then first `readLimitBytes` will be returned.
func limitedRead(filePath string, readLimitBytes int64) ([]byte, error) {
	f, err := os.Open(filePath)
	if err != nil {
		return nil, errors.Wrapf(err, "limited read failed to open file: %s", filePath)
	}
	defer f.Close()
	if fi, err := f.Stat(); err == nil {
		if fi.Size() < readLimitBytes {
			readLimitBytes = fi.Size()
		}
		buf := make([]byte, readLimitBytes)
		_, err := f.Read(buf)
		if err != nil {
			return []byte{}, errors.Wrapf(err, "limited read failed during file read: %s", filePath)
		}
		return buf, nil
	}
	return []byte{}, errors.Wrapf(err, "limited read failed during file stat: %s", filePath)
}

var deleteCommand = cli.Command{
	Name: "delete",
	Usage: `
This command allows containerd to delete any container resources created, mounted, and/or run by a shim when containerd can no longer communicate over rpc. This happens if a shim is SIGKILL'd with a running container. These resources will need to be cleaned up when containerd loses the connection to a shim. This is also used when containerd boots and reconnects to shims. If a bundle is still on disk but containerd cannot connect to a shim, the delete command is invoked.

The delete command will be executed in the container's bundle as its cwd.
`,
	SkipArgReorder: true,
	Action: func(context *cli.Context) (err error) {
		// We cant write anything to stdout for this cmd other than the
		// task.DeleteResponse by protcol. We can write to stderr which will be
		// warning logged in containerd.

		ctx, span := trace.StartSpan(gcontext.Background(), "delete")
		defer span.End()
		defer func() { oc.SetSpanStatus(span, err) }()

		bundleFlag := context.GlobalString("bundle")
		if bundleFlag == "" {
			return errors.New("bundle is required")
		}

		// hcsshim shim writes panic logs in the bundle directory in a file named "panic.log"
		// log those messages (if any) on stderr so that it shows up in containerd's log.
		// This should be done as the first thing so that we don't miss any panic logs even if
		// something goes wrong during delete op.
		// The file can be very large so read only first 1MB of data.
		readLimit := int64(1024 * 1024) // 1MB
		logBytes, err := limitedRead(filepath.Join(bundleFlag, "panic.log"), readLimit)
		if err == nil && len(logBytes) > 0 {
			if int64(len(logBytes)) == readLimit {
				logrus.Warnf("shim panic log file %s is larger than 1MB, logging only first 1MB", filepath.Join(bundleFlag, "panic.log"))
			}
			logrus.WithField("log", string(logBytes)).Warn("found shim panic logs during delete")
		} else if err != nil && !errors.Is(err, os.ErrNotExist) {
			logrus.WithError(err).Warn("failed to open shim panic log")
		}

		// Attempt to find the hcssystem for this bundle and terminate it.
		if sys, _ := hcs.OpenComputeSystem(ctx, idFlag); sys != nil {
			defer sys.Close()
			if err := sys.Terminate(ctx); err != nil {
				fmt.Fprintf(os.Stderr, "failed to terminate '%s': %v", idFlag, err)
			} else {
				ch := make(chan error, 1)
				go func() { ch <- sys.Wait() }()
				t := time.NewTimer(time.Second * 30)
				select {
				case <-t.C:
					sys.Close()
					return fmt.Errorf("timed out waiting for '%s' to terminate", idFlag)
				case err := <-ch:
					t.Stop()
					if err != nil {
						fmt.Fprintf(os.Stderr, "failed to wait for '%s' to terminate: %v", idFlag, err)
					}
				}
			}
		}

		if data, err := proto.Marshal(&task.DeleteResponse{
			ExitedAt:   time.Now(),
			ExitStatus: 255,
		}); err != nil {
			return err
		} else {
			if _, err := os.Stdout.Write(data); err != nil {
				return err
			}
		}
		return nil
	},
}
<|endoftext|>