<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"text/template"
)

type copyfiles struct {
	dir  string
	spec string
}

const (
	bbList = `{{.Uroot}}/src/bb/bbsh
init`
)

var (
	config struct {
		Goroot    string
		Gosrcroot string
		Uroot	  string
		Arch      string
		Goos      string
		Letter    string
		Gopath    string
		TempDir   string
		Go        string
		Debug     bool
	}
	letter = map[string]string{
		"amd64": "6",
		"arm":   "5",
		"ppc":   "9",
	}
)

func getenv(e, d string) string {
	v := os.Getenv(e)
	if v == "" {
		v = d
	}
	return v
}

func lsr(n string, w *os.File) error {
	n = n + "/"
	err := filepath.Walk(n, func(name string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		cn := strings.TrimPrefix(name, n)
		fmt.Fprintf(w, "%v\n", cn)
		return nil
	})
	return err
}

// we'll keep using cpio and hope the kernel gets fixed some day.
func cpiop(c string) error {

	t := template.Must(template.New("filelist").Parse(c))
	var b bytes.Buffer
	if err := t.Execute(&b, config); err != nil {
		log.Fatalf("spec %v: %v\n", c, err)
	}

	n := strings.Split(b.String(), "\n")
	if config.Debug {
		fmt.Fprintf(os.Stderr, "Strings :%v:\n", n)
	}

	r, w, err := os.Pipe()
	if err != nil {
		log.Fatalf("%v\n", err)
	}
	cmd := exec.Command("cpio", "--make-directories", "-p", config.TempDir)
	cmd.Dir = n[0]
	cmd.Stdin = r
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	if config.Debug {
		log.Printf("Run %v @ %v", cmd, cmd.Dir)
	}
	err = cmd.Start()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
	}

	for _, v := range n[1:] {
		if config.Debug {
			fmt.Fprintf(os.Stderr, "%v\n", v)
		}
		err := filepath.Walk(path.Join(n[0], v), func(name string, fi os.FileInfo, err error) error {
			if err != nil {
				fmt.Printf(" WALK FAIL%v: %v\n", name, err)
				// That's ok, sometimes things are not there.
				return filepath.SkipDir
			}
			cn := strings.TrimPrefix(name, n[0]+"/")
			if cn == ".git" {
				return filepath.SkipDir
			}
			fmt.Fprintf(w, "%v\n", cn)
			fmt.Printf("c.dir %v %v %v\n", n[0], name, cn)
			return nil
		})
		if err != nil {
			fmt.Printf("%s: %v\n", v, err)
		}
	}
	w.Close()
	err = cmd.Wait()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
	}
	return nil
}

func sanity() {
	goBinGo := path.Join(config.Goroot, "bin/go")
	_, err := os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	// but does the one in go/bin/OS_ARCH exist too?
	goBinGo = path.Join(config.Goroot, fmt.Sprintf("bin/%s_%s/go", config.Goos, config.Arch))
	_, err = os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	if config.Go == "" {
		log.Fatalf("Can't find a go binary! Is GOROOT set correctly?")
	}
}

// sad news. If I concat the Go cpio with the other cpios, for reasons I don't understand,
// the kernel can't unpack it. Don't know why, don't care. Need to create one giant cpio and unpack that.
// It's not size related: if the go archive is first or in the middle it still fails.
func main() {
	flag.BoolVar(&config.Debug, "d", false, "Debugging")
	flag.Parse()
	var err error
	config.Arch = getenv("GOARCH", "amd64")
	config.Goroot = getenv("GOROOT", "/")
	config.Gosrcroot = path.Dir(config.Goroot)
	config.Gopath = getenv("GOPATH", "")
	config.Uroot = getenv("UROOT", "")
	config.Goos = "linux"
	config.Letter = letter[config.Arch]
	config.TempDir, err = ioutil.TempDir("", "u-root")
	config.Go = ""
	if err != nil {
		log.Fatalf("%v", err)
	}

	// Build init
	cmd := exec.Command("go", "build", "-o", "init", ".")
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Dir = path.Join(config.Uroot, "src/bb/bbsh")
fmt.Printf("cmd.DIr %v\n", cmd.Dir)
fmt.Printf("cmd %v\n", cmd)

	err = cmd.Run()
	if err != nil {
		log.Fatalf("%v\n", err)
		os.Exit(1)
	}
fmt.Printf("BUILT bbsh\n")

	r, w, err := os.Pipe()
	if err != nil {
		log.Fatalf("%v\n", err)
	}

	// First create the archive and put the device cpio in it.
	dev, err := ioutil.ReadFile("dev.cpio")
	if err != nil {
		log.Fatal("%v %v\n", dev, err)
	}

	oname := fmt.Sprintf("/tmp/initramfs.%v_%v.cpio", config.Goos, config.Arch)
	if err := ioutil.WriteFile(oname, dev, 0600); err != nil {
		log.Fatal("%v\n", err)
	}

	// Now use the append option for cpio to append to it.
	// That way we get one cpio.
	cmd = exec.Command("cpio", "-H", "newc", "-o", "-A", "-F", oname)
	cmd.Dir = path.Join(config.Uroot, "src/bb/bbsh")
	cmd.Stdin = r
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	if config.Debug {
		fmt.Fprintf(os.Stderr, "Run %v @ %v", cmd, cmd.Dir)
	}
	err = cmd.Start()
	if err != nil {
		log.Fatalf("%v\n", err)
	}
	w.Write([]byte("init\n"))
	w.Close()
	err = cmd.Wait()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
	}
	fmt.Printf("Output file is in %v\n", oname)
}
<commit_msg>make bbramfs use the full path of dev.cpio<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"text/template"
)

type copyfiles struct {
	dir  string
	spec string
}

const (
	bbList = `{{.Uroot}}/src/bb/bbsh
init`
)

var (
	config struct {
		Goroot    string
		Gosrcroot string
		Uroot	  string
		Arch      string
		Goos      string
		Letter    string
		Gopath    string
		TempDir   string
		Go        string
		Debug     bool
	}
	letter = map[string]string{
		"amd64": "6",
		"arm":   "5",
		"ppc":   "9",
	}
)

func getenv(e, d string) string {
	v := os.Getenv(e)
	if v == "" {
		v = d
	}
	return v
}

func lsr(n string, w *os.File) error {
	n = n + "/"
	err := filepath.Walk(n, func(name string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		cn := strings.TrimPrefix(name, n)
		fmt.Fprintf(w, "%v\n", cn)
		return nil
	})
	return err
}

// we'll keep using cpio and hope the kernel gets fixed some day.
func cpiop(c string) error {

	t := template.Must(template.New("filelist").Parse(c))
	var b bytes.Buffer
	if err := t.Execute(&b, config); err != nil {
		log.Fatalf("spec %v: %v\n", c, err)
	}

	n := strings.Split(b.String(), "\n")
	if config.Debug {
		fmt.Fprintf(os.Stderr, "Strings :%v:\n", n)
	}

	r, w, err := os.Pipe()
	if err != nil {
		log.Fatalf("%v\n", err)
	}
	cmd := exec.Command("cpio", "--make-directories", "-p", config.TempDir)
	cmd.Dir = n[0]
	cmd.Stdin = r
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	if config.Debug {
		log.Printf("Run %v @ %v", cmd, cmd.Dir)
	}
	err = cmd.Start()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
	}

	for _, v := range n[1:] {
		if config.Debug {
			fmt.Fprintf(os.Stderr, "%v\n", v)
		}
		err := filepath.Walk(path.Join(n[0], v), func(name string, fi os.FileInfo, err error) error {
			if err != nil {
				fmt.Printf(" WALK FAIL%v: %v\n", name, err)
				// That's ok, sometimes things are not there.
				return filepath.SkipDir
			}
			cn := strings.TrimPrefix(name, n[0]+"/")
			if cn == ".git" {
				return filepath.SkipDir
			}
			fmt.Fprintf(w, "%v\n", cn)
			fmt.Printf("c.dir %v %v %v\n", n[0], name, cn)
			return nil
		})
		if err != nil {
			fmt.Printf("%s: %v\n", v, err)
		}
	}
	w.Close()
	err = cmd.Wait()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
	}
	return nil
}

func sanity() {
	goBinGo := path.Join(config.Goroot, "bin/go")
	_, err := os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	// but does the one in go/bin/OS_ARCH exist too?
	goBinGo = path.Join(config.Goroot, fmt.Sprintf("bin/%s_%s/go", config.Goos, config.Arch))
	_, err = os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	if config.Go == "" {
		log.Fatalf("Can't find a go binary! Is GOROOT set correctly?")
	}
}

// sad news. If I concat the Go cpio with the other cpios, for reasons I don't understand,
// the kernel can't unpack it. Don't know why, don't care. Need to create one giant cpio and unpack that.
// It's not size related: if the go archive is first or in the middle it still fails.
func main() {
	flag.BoolVar(&config.Debug, "d", false, "Debugging")
	flag.Parse()
	var err error
	config.Arch = getenv("GOARCH", "amd64")
	config.Goroot = getenv("GOROOT", "/")
	config.Gosrcroot = path.Dir(config.Goroot)
	config.Gopath = getenv("GOPATH", "")
	config.Uroot = getenv("UROOT", "")
	config.Goos = "linux"
	config.Letter = letter[config.Arch]
	config.TempDir, err = ioutil.TempDir("", "u-root")
	config.Go = ""
	if err != nil {
		log.Fatalf("%v", err)
	}

	// Build init
	cmd := exec.Command("go", "build", "-o", "init", ".")
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Dir = path.Join(config.Uroot, "src/bb/bbsh")
fmt.Printf("cmd.DIr %v\n", cmd.Dir)
fmt.Printf("cmd %v\n", cmd)

	err = cmd.Run()
	if err != nil {
		log.Fatalf("%v\n", err)
		os.Exit(1)
	}
fmt.Printf("BUILT bbsh\n")

	r, w, err := os.Pipe()
	if err != nil {
		log.Fatalf("%v\n", err)
	}

	// First create the archive and put the device cpio in it.
	dev, err := ioutil.ReadFile(path.Join(config.Uroot, "scripts", "dev.cpio"))
	if err != nil {
		log.Fatal("%v %v\n", dev, err)
	}

	oname := fmt.Sprintf("/tmp/initramfs.%v_%v.cpio", config.Goos, config.Arch)
	if err := ioutil.WriteFile(oname, dev, 0600); err != nil {
		log.Fatal("%v\n", err)
	}

	// Now use the append option for cpio to append to it.
	// That way we get one cpio.
	cmd = exec.Command("cpio", "-H", "newc", "-o", "-A", "-F", oname)
	cmd.Dir = path.Join(config.Uroot, "src/bb/bbsh")
	cmd.Stdin = r
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	if config.Debug {
		fmt.Fprintf(os.Stderr, "Run %v @ %v", cmd, cmd.Dir)
	}
	err = cmd.Start()
	if err != nil {
		log.Fatalf("%v\n", err)
	}
	w.Write([]byte("init\n"))
	w.Close()
	err = cmd.Wait()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
	}
	fmt.Printf("Output file is in %v\n", oname)
}
<|endoftext|>


<commit_before>// Use of this source code is governed by a MIT license that can be found in the LICENSE file.
// Giorgis (habtom@giorgis.io)

//Package avcodec contains the codecs (decoders and encoders) provided by the libavcodec library
//Provides some generic global options, which can be set on all the encoders and decoders.
package avcodec

//#cgo pkg-config: libavformat libavcodec libavutil libswresample
//#include <stdio.h>
//#include <stdlib.h>
//#include <inttypes.h>
//#include <stdint.h>
//#include <string.h>
//#include <libavformat/avformat.h>
//#include <libavcodec/avcodec.h>
//#include <libavutil/avutil.h>
import "C"
import (
	"unsafe"
	"github.com/targodan/goav/avutil"
)

type (
	Codec                         C.struct_AVCodec
	CodecContext                  C.struct_AVCodecContext
	Descriptor                    C.struct_AVCodecDescriptor
	Parser                        C.struct_AVCodecParser
	ParserContext                 C.struct_AVCodecParserContext
	Dictionary                    C.struct_AVDictionary
	Frame                         C.struct_AVFrame
	MediaType                     C.enum_AVMediaType
	Packet                        C.struct_AVPacket
	BitStreamFilter               C.struct_AVBitStreamFilter
	BitStreamFilterContext        C.struct_AVBitStreamFilterContext
	Rational                      C.struct_AVRational
	Class                         C.struct_AVClass
	AvHWAccel                     C.struct_AVHWAccel
	AvPacketSideData              C.struct_AVPacketSideData
	AvPanScan                     C.struct_AVPanScan
	Picture                       C.struct_AVPicture
	AvProfile                     C.struct_AVProfile
	AvSubtitle                    C.struct_AVSubtitle
	AvSubtitleRect                C.struct_AVSubtitleRect
	RcOverride                    C.struct_RcOverride
	AvBufferRef                   C.struct_AVBufferRef
    AvCodecParameters             C.struct_AVCodecParameters
	AvAudioServiceType            C.enum_AVAudioServiceType
	AvChromaLocation              C.enum_AVChromaLocation
	CodecId                       C.enum_AVCodecID
	AvColorPrimaries              C.enum_AVColorPrimaries
	AvColorRange                  C.enum_AVColorRange
	AvColorSpace                  C.enum_AVColorSpace
	AvColorTransferCharacteristic C.enum_AVColorTransferCharacteristic
	AvDiscard                     C.enum_AVDiscard
	AvFieldOrder                  C.enum_AVFieldOrder
	AvPacketSideDataType          C.enum_AVPacketSideDataType
	PixelFormat                   C.enum_AVPixelFormat
	AvSampleFormat                C.enum_AVSampleFormat
)

func (c *Codec) AvCodecGetMaxLowres() int {
	return int(C.av_codec_get_max_lowres((*C.struct_AVCodec)(c)))
}

//If c is NULL, returns the first registered codec, if c is non-NULL,
func (c *Codec) AvCodecNext() *Codec {
	return (*Codec)(C.av_codec_next((*C.struct_AVCodec)(c)))
}

//Register the codec codec and initialize libavcodec.
func (c *Codec) AvcodecRegister() {
	C.avcodec_register((*C.struct_AVCodec)(c))
}

//Return a name for the specified profile, if available.
func (c *Codec) AvGetProfileName(p int) string {
	return C.GoString(C.av_get_profile_name((*C.struct_AVCodec)(c), C.int(p)))
}

//Allocate an Context and set its fields to default values.
func (c *Codec) AvcodecAllocContext3() *CodecContext {
	return (*CodecContext)(C.avcodec_alloc_context3((*C.struct_AVCodec)(c)))
}

func (c *Codec) AvCodecIsEncoder() int {
	return int(C.av_codec_is_encoder((*C.struct_AVCodec)(c)))
}

func (c *Codec) AvCodecIsDecoder() int {
	return int(C.av_codec_is_decoder((*C.struct_AVCodec)(c)))
}

func (c *Codec) Name() string {
	return C.GoString(c.name)
}

func (c *Codec) LongName() string {
	return C.GoString(c.long_name)
}

//Same behaviour av_fast_malloc but the buffer has additional FF_INPUT_BUFFER_PADDING_SIZE at the end which will always be 0.
func AvFastPaddedMalloc(p unsafe.Pointer, s *uint, t uintptr) {
	C.av_fast_padded_malloc(p, (*C.uint)(unsafe.Pointer(s)), (C.size_t)(t))
}

//Return the LIBAvCODEC_VERSION_INT constant.
func AvcodecVersion() uint {
	return uint(C.avcodec_version())
}

//Return the libavcodec build-time configuration.
func AvcodecConfiguration() string {
	return C.GoString(C.avcodec_configuration())

}

//Return the libavcodec license.
func AvcodecLicense() string {
	return C.GoString(C.avcodec_license())
}

//Register all the codecs, parsers and bitstream filters which were enabled at configuration time.
func AvcodecRegisterAll() {
	C.avcodec_register_all()
}

//Get the Class for Context.
func AvcodecGetClass() *Class {
	return (*Class)(C.avcodec_get_class())
}

//Get the Class for Frame.
func AvcodecGetFrameClass() *Class {
	return (*Class)(C.avcodec_get_frame_class())
}

//Get the Class for AvSubtitleRect.
func AvcodecGetSubtitleRectClass() *Class {
	return (*Class)(C.avcodec_get_subtitle_rect_class())
}

//Free all allocated data in the given subtitle struct.
func AvsubtitleFree(s *AvSubtitle) {
	C.avsubtitle_free((*C.struct_AVSubtitle)(s))
}

//Pack a dictionary for use in side_data.
func AvPacketPackDictionary(d *Dictionary, s *int) *uint8 {
	return (*uint8)(C.av_packet_pack_dictionary((*C.struct_AVDictionary)(d), (*C.int)(unsafe.Pointer(s))))
}

//Unpack a dictionary from side_data.
func AvPacketUnpackDictionary(d *uint8, s int, dt **Dictionary) int {
	return int(C.av_packet_unpack_dictionary((*C.uint8_t)(d), C.int(s), (**C.struct_AVDictionary)(unsafe.Pointer(dt))))
}

//Find a registered decoder with a matching codec ID.
func AvcodecFindDecoder(id CodecId) *Codec {
	return (*Codec)(C.avcodec_find_decoder((C.enum_AVCodecID)(id)))
}

//Find a registered decoder with the specified name.
func AvcodecFindDecoderByName(n string) *Codec {
	return (*Codec)(C.avcodec_find_decoder_by_name(C.CString(n)))
}

//Converts AvChromaLocation to swscale x/y chroma position.
func AvcodecEnumToChromaPos(x, y *int, l AvChromaLocation) int {
	return int(C.avcodec_enum_to_chroma_pos((*C.int)(unsafe.Pointer(x)), (*C.int)(unsafe.Pointer(y)), (C.enum_AVChromaLocation)(l)))
}

//Converts swscale x/y chroma position to AvChromaLocation.
func AvcodecChromaPosToEnum(x, y int) AvChromaLocation {
	return (AvChromaLocation)(C.avcodec_chroma_pos_to_enum(C.int(x), C.int(y)))
}

//Find a registered encoder with a matching codec ID.
func AvcodecFindEncoder(id CodecId) *Codec {
	return (*Codec)(C.avcodec_find_encoder((C.enum_AVCodecID)(id)))
}

//Find a registered encoder with the specified name.
func AvcodecFindEncoderByName(c string) *Codec {
	return (*Codec)(C.avcodec_find_encoder_by_name(C.CString(c)))
}

//Put a string representing the codec tag codec_tag in buf.
func AvGetCodecTagString(b string, bf uintptr, c uint) uintptr {
	return uintptr(C.av_get_codec_tag_string(C.CString(b), C.size_t(bf), C.uint(c)))
}

func AvcodecString(b string, bs int, ctxt *CodecContext, e int) {
	C.avcodec_string(C.CString(b), C.int(bs), (*C.struct_AVCodecContext)(ctxt), C.int(e))
}

//Fill Frame audio data and linesize pointers.
func AvcodecFillAudioFrame(f *Frame, c int, s AvSampleFormat, b *uint8, bs, a int) int {
	return int(C.avcodec_fill_audio_frame((*C.struct_AVFrame)(f), C.int(c), (C.enum_AVSampleFormat)(s), (*C.uint8_t)(b), C.int(bs), C.int(a)))
}

//Return codec bits per sample.
func AvGetBitsPerSample(c CodecId) int {
	return int(C.av_get_bits_per_sample((C.enum_AVCodecID)(c)))
}

//Return the PCM codec associated with a sample format.
func AvGetPcmCodec(f AvSampleFormat, b int) CodecId {
	return (CodecId)(C.av_get_pcm_codec((C.enum_AVSampleFormat)(f), C.int(b)))
}

//Return codec bits per sample.
func AvGetExactBitsPerSample(c CodecId) int {
	return int(C.av_get_exact_bits_per_sample((C.enum_AVCodecID)(c)))
}

//Same behaviour av_fast_padded_malloc except that buffer will always be 0-initialized after call.
func AvFastPaddedMallocz(p unsafe.Pointer, s *uint, t uintptr) {
	C.av_fast_padded_mallocz(p, (*C.uint)(unsafe.Pointer(s)), (C.size_t)(t))
}

//Encode extradata length to a buffer.
func AvXiphlacing(s *string, v uint) uint {
	return uint(C.av_xiphlacing((*C.uchar)(unsafe.Pointer(s)), (C.uint)(v)))
}

//If hwaccel is NULL, returns the first registered hardware accelerator, if hwaccel is non-NULL,
//returns the next registered hardware accelerator after hwaccel, or NULL if hwaccel is the last one.
func (a *AvHWAccel) AvHwaccelNext() *AvHWAccel {
	return (*AvHWAccel)(C.av_hwaccel_next((*C.struct_AVHWAccel)(a)))
}

//Get the type of the given codec.
func AvcodecGetType(c CodecId) avutil.MediaType {
	return (avutil.MediaType)(C.avcodec_get_type((C.enum_AVCodecID)(c)))
}

//Get the name of a codec.
func AvcodecGetName(d CodecId) string {
	return C.GoString(C.avcodec_get_name((C.enum_AVCodecID)(d)))
}

//const Descriptor *avcodec_descriptor_get (enum CodecId id)
func AvcodecDescriptorGet(id CodecId) *Descriptor {
	return (*Descriptor)(C.avcodec_descriptor_get((C.enum_AVCodecID)(id)))
}

//Iterate over all codec descriptors known to libavcodec.
func (d *Descriptor) AvcodecDescriptorNext() *Descriptor {
	return (*Descriptor)(C.avcodec_descriptor_next((*C.struct_AVCodecDescriptor)(d)))
}

func AvcodecDescriptorGetByName(n string) *Descriptor {
	return (*Descriptor)(C.avcodec_descriptor_get_by_name(C.CString(n)))
}

func avcodec_parameters_to_context(codec *CodecContext, AvCodecPar)
<commit_msg>Added avcodec_parameters_to_context function<commit_after>// Use of this source code is governed by a MIT license that can be found in the LICENSE file.
// Giorgis (habtom@giorgis.io)

//Package avcodec contains the codecs (decoders and encoders) provided by the libavcodec library
//Provides some generic global options, which can be set on all the encoders and decoders.
package avcodec

//#cgo pkg-config: libavformat libavcodec libavutil libswresample
//#include <stdio.h>
//#include <stdlib.h>
//#include <inttypes.h>
//#include <stdint.h>
//#include <string.h>
//#include <libavformat/avformat.h>
//#include <libavcodec/avcodec.h>
//#include <libavutil/avutil.h>
import "C"
import (
	"unsafe"

	"github.com/targodan/goav/avutil"
)

type (
	Codec                         C.struct_AVCodec
	CodecContext                  C.struct_AVCodecContext
	Descriptor                    C.struct_AVCodecDescriptor
	Parser                        C.struct_AVCodecParser
	ParserContext                 C.struct_AVCodecParserContext
	Dictionary                    C.struct_AVDictionary
	Frame                         C.struct_AVFrame
	MediaType                     C.enum_AVMediaType
	Packet                        C.struct_AVPacket
	BitStreamFilter               C.struct_AVBitStreamFilter
	BitStreamFilterContext        C.struct_AVBitStreamFilterContext
	Rational                      C.struct_AVRational
	Class                         C.struct_AVClass
	AvHWAccel                     C.struct_AVHWAccel
	AvPacketSideData              C.struct_AVPacketSideData
	AvPanScan                     C.struct_AVPanScan
	Picture                       C.struct_AVPicture
	AvProfile                     C.struct_AVProfile
	AvSubtitle                    C.struct_AVSubtitle
	AvSubtitleRect                C.struct_AVSubtitleRect
	RcOverride                    C.struct_RcOverride
	AvBufferRef                   C.struct_AVBufferRef
	AvCodecParameters             C.struct_AVCodecParameters
	AvAudioServiceType            C.enum_AVAudioServiceType
	AvChromaLocation              C.enum_AVChromaLocation
	CodecId                       C.enum_AVCodecID
	AvColorPrimaries              C.enum_AVColorPrimaries
	AvColorRange                  C.enum_AVColorRange
	AvColorSpace                  C.enum_AVColorSpace
	AvColorTransferCharacteristic C.enum_AVColorTransferCharacteristic
	AvDiscard                     C.enum_AVDiscard
	AvFieldOrder                  C.enum_AVFieldOrder
	AvPacketSideDataType          C.enum_AVPacketSideDataType
	PixelFormat                   C.enum_AVPixelFormat
	AvSampleFormat                C.enum_AVSampleFormat
)

func (c *Codec) AvCodecGetMaxLowres() int {
	return int(C.av_codec_get_max_lowres((*C.struct_AVCodec)(c)))
}

//If c is NULL, returns the first registered codec, if c is non-NULL,
func (c *Codec) AvCodecNext() *Codec {
	return (*Codec)(C.av_codec_next((*C.struct_AVCodec)(c)))
}

//Register the codec codec and initialize libavcodec.
func (c *Codec) AvcodecRegister() {
	C.avcodec_register((*C.struct_AVCodec)(c))
}

//Return a name for the specified profile, if available.
func (c *Codec) AvGetProfileName(p int) string {
	return C.GoString(C.av_get_profile_name((*C.struct_AVCodec)(c), C.int(p)))
}

//Allocate an Context and set its fields to default values.
func (c *Codec) AvcodecAllocContext3() *CodecContext {
	return (*CodecContext)(C.avcodec_alloc_context3((*C.struct_AVCodec)(c)))
}

func (c *Codec) AvCodecIsEncoder() int {
	return int(C.av_codec_is_encoder((*C.struct_AVCodec)(c)))
}

func (c *Codec) AvCodecIsDecoder() int {
	return int(C.av_codec_is_decoder((*C.struct_AVCodec)(c)))
}

func (c *Codec) Name() string {
	return C.GoString(c.name)
}

func (c *Codec) LongName() string {
	return C.GoString(c.long_name)
}

//Same behaviour av_fast_malloc but the buffer has additional FF_INPUT_BUFFER_PADDING_SIZE at the end which will always be 0.
func AvFastPaddedMalloc(p unsafe.Pointer, s *uint, t uintptr) {
	C.av_fast_padded_malloc(p, (*C.uint)(unsafe.Pointer(s)), (C.size_t)(t))
}

//Return the LIBAvCODEC_VERSION_INT constant.
func AvcodecVersion() uint {
	return uint(C.avcodec_version())
}

//Return the libavcodec build-time configuration.
func AvcodecConfiguration() string {
	return C.GoString(C.avcodec_configuration())

}

//Return the libavcodec license.
func AvcodecLicense() string {
	return C.GoString(C.avcodec_license())
}

//Register all the codecs, parsers and bitstream filters which were enabled at configuration time.
func AvcodecRegisterAll() {
	C.avcodec_register_all()
}

//Get the Class for Context.
func AvcodecGetClass() *Class {
	return (*Class)(C.avcodec_get_class())
}

//Get the Class for Frame.
func AvcodecGetFrameClass() *Class {
	return (*Class)(C.avcodec_get_frame_class())
}

//Get the Class for AvSubtitleRect.
func AvcodecGetSubtitleRectClass() *Class {
	return (*Class)(C.avcodec_get_subtitle_rect_class())
}

//Free all allocated data in the given subtitle struct.
func AvsubtitleFree(s *AvSubtitle) {
	C.avsubtitle_free((*C.struct_AVSubtitle)(s))
}

//Pack a dictionary for use in side_data.
func AvPacketPackDictionary(d *Dictionary, s *int) *uint8 {
	return (*uint8)(C.av_packet_pack_dictionary((*C.struct_AVDictionary)(d), (*C.int)(unsafe.Pointer(s))))
}

//Unpack a dictionary from side_data.
func AvPacketUnpackDictionary(d *uint8, s int, dt **Dictionary) int {
	return int(C.av_packet_unpack_dictionary((*C.uint8_t)(d), C.int(s), (**C.struct_AVDictionary)(unsafe.Pointer(dt))))
}

//Find a registered decoder with a matching codec ID.
func AvcodecFindDecoder(id CodecId) *Codec {
	return (*Codec)(C.avcodec_find_decoder((C.enum_AVCodecID)(id)))
}

//Find a registered decoder with the specified name.
func AvcodecFindDecoderByName(n string) *Codec {
	return (*Codec)(C.avcodec_find_decoder_by_name(C.CString(n)))
}

//Converts AvChromaLocation to swscale x/y chroma position.
func AvcodecEnumToChromaPos(x, y *int, l AvChromaLocation) int {
	return int(C.avcodec_enum_to_chroma_pos((*C.int)(unsafe.Pointer(x)), (*C.int)(unsafe.Pointer(y)), (C.enum_AVChromaLocation)(l)))
}

//Converts swscale x/y chroma position to AvChromaLocation.
func AvcodecChromaPosToEnum(x, y int) AvChromaLocation {
	return (AvChromaLocation)(C.avcodec_chroma_pos_to_enum(C.int(x), C.int(y)))
}

//Find a registered encoder with a matching codec ID.
func AvcodecFindEncoder(id CodecId) *Codec {
	return (*Codec)(C.avcodec_find_encoder((C.enum_AVCodecID)(id)))
}

//Find a registered encoder with the specified name.
func AvcodecFindEncoderByName(c string) *Codec {
	return (*Codec)(C.avcodec_find_encoder_by_name(C.CString(c)))
}

//Put a string representing the codec tag codec_tag in buf.
func AvGetCodecTagString(b string, bf uintptr, c uint) uintptr {
	return uintptr(C.av_get_codec_tag_string(C.CString(b), C.size_t(bf), C.uint(c)))
}

func AvcodecString(b string, bs int, ctxt *CodecContext, e int) {
	C.avcodec_string(C.CString(b), C.int(bs), (*C.struct_AVCodecContext)(ctxt), C.int(e))
}

//Fill Frame audio data and linesize pointers.
func AvcodecFillAudioFrame(f *Frame, c int, s AvSampleFormat, b *uint8, bs, a int) int {
	return int(C.avcodec_fill_audio_frame((*C.struct_AVFrame)(f), C.int(c), (C.enum_AVSampleFormat)(s), (*C.uint8_t)(b), C.int(bs), C.int(a)))
}

//Return codec bits per sample.
func AvGetBitsPerSample(c CodecId) int {
	return int(C.av_get_bits_per_sample((C.enum_AVCodecID)(c)))
}

//Return the PCM codec associated with a sample format.
func AvGetPcmCodec(f AvSampleFormat, b int) CodecId {
	return (CodecId)(C.av_get_pcm_codec((C.enum_AVSampleFormat)(f), C.int(b)))
}

//Return codec bits per sample.
func AvGetExactBitsPerSample(c CodecId) int {
	return int(C.av_get_exact_bits_per_sample((C.enum_AVCodecID)(c)))
}

//Same behaviour av_fast_padded_malloc except that buffer will always be 0-initialized after call.
func AvFastPaddedMallocz(p unsafe.Pointer, s *uint, t uintptr) {
	C.av_fast_padded_mallocz(p, (*C.uint)(unsafe.Pointer(s)), (C.size_t)(t))
}

//Encode extradata length to a buffer.
func AvXiphlacing(s *string, v uint) uint {
	return uint(C.av_xiphlacing((*C.uchar)(unsafe.Pointer(s)), (C.uint)(v)))
}

//If hwaccel is NULL, returns the first registered hardware accelerator, if hwaccel is non-NULL,
//returns the next registered hardware accelerator after hwaccel, or NULL if hwaccel is the last one.
func (a *AvHWAccel) AvHwaccelNext() *AvHWAccel {
	return (*AvHWAccel)(C.av_hwaccel_next((*C.struct_AVHWAccel)(a)))
}

//Get the type of the given codec.
func AvcodecGetType(c CodecId) avutil.MediaType {
	return (avutil.MediaType)(C.avcodec_get_type((C.enum_AVCodecID)(c)))
}

//Get the name of a codec.
func AvcodecGetName(d CodecId) string {
	return C.GoString(C.avcodec_get_name((C.enum_AVCodecID)(d)))
}

//const Descriptor *avcodec_descriptor_get (enum CodecId id)
func AvcodecDescriptorGet(id CodecId) *Descriptor {
	return (*Descriptor)(C.avcodec_descriptor_get((C.enum_AVCodecID)(id)))
}

//Iterate over all codec descriptors known to libavcodec.
func (d *Descriptor) AvcodecDescriptorNext() *Descriptor {
	return (*Descriptor)(C.avcodec_descriptor_next((*C.struct_AVCodecDescriptor)(d)))
}

func AvcodecDescriptorGetByName(n string) *Descriptor {
	return (*Descriptor)(C.avcodec_descriptor_get_by_name(C.CString(n)))
}

func AvcodecParametersToContext(codec *CodecContext, par *AvCodecParameters) error {
	avutil.CodeToError(C.avcodec_to_parameters((*C.AVCodecContext)(codec), (*C.AVCodecParameters)(par)))
}
<|endoftext|>


<commit_before>package plugins

import (
	"html"
	"regexp"
)

type Vimeo struct {
	plugin
	spoiler, title, user *regexp.Regexp
}

func (plug *Vimeo) Setup(write chan IRCMessage, conf PluginConf) {
	plug.write = write
	plug.match = regexp.MustCompile(`(?:https?://|)(?:www\.|)(vimeo.com/\S+)`)
	plug.spoiler = regexp.MustCompile(`(?i)(.*spoil.*)`)
	//plug.title = regexp.MustCompile(`.*<title>(.+)(?: on Vimeo){1}</title>.*`)
	plug.title = regexp.MustCompile(`<[^>]*meta[^>]*property="og:title"[^>]*content="(.+)"[^>]*>`)
	//plug.user = regexp.MustCompile(`.*<div[^>]+(?: +itemtype="http://schema.org/Person" +|[^>]+?| +itemprop="author" +){3,}>(?s:.*?)<[^>]*meta[^>]*itemprop="name"[^>]*content="(.+)"[^>]*>.*`)
	//plug.user = regexp.MustCompile(`<[^>]*meta[^>]*itemprop="name"[^>]*content="(.+)"[^>]*>`)
	plug.user = regexp.MustCompile(`<a rel="author" href="/[^>]+?">(.+?)</a>`)
	plug.event = make(chan IRCMessage, 1000)
	scrapeAndSend(plug)
	return
}

func (plug *Vimeo) FindUri(candidate *string) (uri *string, err error) {
	uri, err = GetFirstMatch(plug.match, candidate)
	if err != nil {
		uri = nil
		return
	}
	full := "http://" + *uri
	uri = &full
	return
}

func (plug Vimeo) Write(msg *IRCMessage, body *string) (err error) {
	title, err := GetFirstMatch(plug.title, body)
	if err != nil {
		return
	}

	user, err := GetFirstMatch(plug.user, body)
	if err != nil {
		return
	}

	_, notFound := GetFirstMatch(plug.spoiler, title)
	if notFound != nil {
		plug.write <- IRCMessage{Channel: msg.Channel, User: msg.User, When: msg.When,
			Msg: "[Vimeo] " + html.UnescapeString(*title+" uploaded by "+*user)}
	} else {
		plug.write <- IRCMessage{Channel: msg.Channel, User: msg.User, When: msg.When,
			Msg: "[Vimeo] [[Title omitted due to possible spoilers]] uploaded by " + *user}
	}

	return
}

func (plug Vimeo) Match() *regexp.Regexp {
	return plug.match
}

func (plug Vimeo) Event() chan IRCMessage {
	return plug.event
}
<commit_msg>Vimeo: Switch to using Vimeo's API instead of scraping<commit_after>package plugins

import (
	"log"
	"encoding/json"
	"html"
	"regexp"
	"errors"
	"strings"
)

var VimeoNoResultsError = errors.New("Vimeo: No results")
var VimeoEmptyResultsError = errors.New("Vimeo: Some of the fields were empty :(")

type resultWrapper struct {
	Results []jsonResult
}

type jsonResult struct {
	User string `json:"user_name"`
	Title string `json:"title"`
}

type Vimeo struct {
	plugin
	spoiler, title, user *regexp.Regexp
}

func (plug *Vimeo) Setup(write chan IRCMessage, conf PluginConf) {
	plug.write = write
	plug.match = regexp.MustCompile(`(?:https?://|)(?:www\.|)(?:vimeo.com/)(\S+)`)
	plug.spoiler = regexp.MustCompile(`(?i)(.*spoil.*)`)
	//plug.title = regexp.MustCompile(`.*<title>(.+)(?: on Vimeo){1}</title>.*`)
	plug.title = regexp.MustCompile(`<[^>]*meta[^>]*property="og:title"[^>]*content="(.+)"[^>]*>`)
	//plug.user = regexp.MustCompile(`.*<div[^>]+(?: +itemtype="http://schema.org/Person" +|[^>]+?| +itemprop="author" +){3,}>(?s:.*?)<[^>]*meta[^>]*itemprop="name"[^>]*content="(.+)"[^>]*>.*`)
	//plug.user = regexp.MustCompile(`<[^>]*meta[^>]*itemprop="name"[^>]*content="(.+)"[^>]*>`)
	plug.user = regexp.MustCompile(`<a rel="author" href="/[^>]+?">(.+?)</a>`)
	plug.event = make(chan IRCMessage, 1000)
	scrapeAndSend(plug)
	return
}

func (plug *Vimeo) FindUri(candidate *string) (uri *string, err error) {
	uri, err = GetFirstMatch(plug.match, candidate)
	if err != nil {
		uri = nil
		return
	}
	full := "http://vimeo.com/api/v2/video/" + *uri + ".json"
	uri = &full
	return
}

func (plug Vimeo) Write(msg *IRCMessage, body *string) (err error) {
	var result []jsonResult
	log.Println(*body)
	err = json.Unmarshal([]byte(*body), &result)
	if err != nil {
		return
	}

	if len(result) != 1 {
		err = VimeoNoResultsError
		return
	}

	title := strings.TrimSpace(result[0].Title)
	user := strings.TrimSpace(result[0].User)
	if title == "" || user == "" {
		err = VimeoEmptyResultsError
		return
	}

	_, notFound := GetFirstMatch(plug.spoiler, &title)
	if notFound != nil {
		plug.write <- IRCMessage{Channel: msg.Channel, User: msg.User, When: msg.When,
			Msg: "[Vimeo] " + html.UnescapeString(title+" uploaded by "+user)}
	} else {
		plug.write <- IRCMessage{Channel: msg.Channel, User: msg.User, When: msg.When,
			Msg: "[Vimeo] [[Title omitted due to possible spoilers]] uploaded by " + user}
	}

	return
}

func (plug Vimeo) Match() *regexp.Regexp {
	return plug.match
}

func (plug Vimeo) Event() chan IRCMessage {
	return plug.event
}
<|endoftext|>


<commit_before>// Copyright 2015 flannel authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import (
	"fmt"
	"strings"
	"sync"

	log "github.com/golang/glog"
	"golang.org/x/net/context"

	"github.com/coreos/flannel/subnet"
)

var backendCtors map[string]BackendCtor = make(map[string]BackendCtor)

type Manager interface {
	GetBackend(backendType string) (Backend, error)
	Wait()
}

type manager struct {
	ctx      context.Context
	sm       subnet.Manager
	extIface *ExternalInterface
	mux      sync.Mutex
	active   map[string]Backend
	wg       sync.WaitGroup
}

func NewManager(ctx context.Context, sm subnet.Manager, extIface *ExternalInterface) Manager {
	return &manager{
		ctx:      ctx,
		sm:       sm,
		extIface: extIface,
		active:   make(map[string]Backend),
	}
}

func (bm *manager) GetBackend(backendType string) (Backend, error) {
	bm.mux.Lock()
	defer bm.mux.Unlock()

	betype := strings.ToLower(backendType)
	// see if one is already running
	if be, ok := bm.active[betype]; ok {
		return be, nil
	}

	// first request, need to create and run it
	befunc, ok := backendCtors[betype]
	if !ok {
		return nil, fmt.Errorf("unknown backend type: %v", betype)
	}

	be, err := befunc(bm.sm, bm.extIface)
	if err != nil {
		return nil, err
	}
	bm.active[betype] = be

	bm.wg.Add(1)
	go func() {
		be.Run(bm.ctx)

		// TODO(eyakubovich): this obviosly introduces a race.
		// GetBackend() could get called while we are here.
		// Currently though, all backends' Run exit only
		// on shutdown

		bm.mux.Lock()
		delete(bm.active, betype)
		bm.mux.Unlock()

		bm.wg.Done()
	}()

	return be, nil
}

func (bm *manager) Wait() {
	bm.wg.Wait()
}

func Register(name string, ctor BackendCtor) {
	log.Infof("Register: %v", name)
	backendCtors[name] = ctor
}
<commit_msg>backend: do not log in Register<commit_after>// Copyright 2015 flannel authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import (
	"fmt"
	"strings"
	"sync"

	"golang.org/x/net/context"

	"github.com/coreos/flannel/subnet"
)

var backendCtors map[string]BackendCtor = make(map[string]BackendCtor)

type Manager interface {
	GetBackend(backendType string) (Backend, error)
	Wait()
}

type manager struct {
	ctx      context.Context
	sm       subnet.Manager
	extIface *ExternalInterface
	mux      sync.Mutex
	active   map[string]Backend
	wg       sync.WaitGroup
}

func NewManager(ctx context.Context, sm subnet.Manager, extIface *ExternalInterface) Manager {
	return &manager{
		ctx:      ctx,
		sm:       sm,
		extIface: extIface,
		active:   make(map[string]Backend),
	}
}

func (bm *manager) GetBackend(backendType string) (Backend, error) {
	bm.mux.Lock()
	defer bm.mux.Unlock()

	betype := strings.ToLower(backendType)
	// see if one is already running
	if be, ok := bm.active[betype]; ok {
		return be, nil
	}

	// first request, need to create and run it
	befunc, ok := backendCtors[betype]
	if !ok {
		return nil, fmt.Errorf("unknown backend type: %v", betype)
	}

	be, err := befunc(bm.sm, bm.extIface)
	if err != nil {
		return nil, err
	}
	bm.active[betype] = be

	bm.wg.Add(1)
	go func() {
		be.Run(bm.ctx)

		// TODO(eyakubovich): this obviosly introduces a race.
		// GetBackend() could get called while we are here.
		// Currently though, all backends' Run exit only
		// on shutdown

		bm.mux.Lock()
		delete(bm.active, betype)
		bm.mux.Unlock()

		bm.wg.Done()
	}()

	return be, nil
}

func (bm *manager) Wait() {
	bm.wg.Wait()
}

func Register(name string, ctor BackendCtor) {
	backendCtors[name] = ctor
}
<|endoftext|>


<commit_before>package fate

import (
	"bufio"
	"os"
	"testing"
)

func TestReply(t *testing.T) {
	model := NewModel(Config{})

	text := "this is a test"

	model.Learn(text)
	reply := model.Reply(text)

	if reply != text {
		t.Errorf("Reply(this is a test) => %s, want %s", reply, text)
	}
}

func TestBabble(t *testing.T) {
	model := NewModel(Config{})

	text := "this is a test"

	model.Learn(text)

	for i := 0; i < 1000; i++ {
		reply := model.Reply("unknown")

		if reply != text {
			t.Errorf("Reply(this is a test) => %s, want %s", reply, text)
		}

		if _, ok := model.tokens.CheckID("unknown"); ok {
			t.Errorf("Reply(\"unknown\") registered token")
		}
	}
}

func TestDuel(t *testing.T) {
	model := NewModel(Config{})

	model.Learn("this is a test")
	model.Learn("this is another test")

	for i := 0; i < 1000; i++ {
		reply := model.Reply("this")

		if reply != "this is a test" && reply != "this is another test" {
			t.Errorf("Reply(this is a test) => %s, want %s", reply, "this is (a|another) test")
		}
	}
}

func learnFile(m *Model, filename string) error {
	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()

	s := bufio.NewScanner(file)
	for s.Scan() {
		m.Learn(s.Text())
	}

	return s.Err()
}
<commit_msg>Add BenchmarkReply<commit_after>package fate

import (
	"bufio"
	"os"
	"testing"
)

func TestReply(t *testing.T) {
	model := NewModel(Config{})

	text := "this is a test"

	model.Learn(text)
	reply := model.Reply(text)

	if reply != text {
		t.Errorf("Reply(this is a test) => %s, want %s", reply, text)
	}
}

func TestBabble(t *testing.T) {
	model := NewModel(Config{})

	text := "this is a test"

	model.Learn(text)

	for i := 0; i < 1000; i++ {
		reply := model.Reply("unknown")

		if reply != text {
			t.Errorf("Reply(this is a test) => %s, want %s", reply, text)
		}

		if _, ok := model.tokens.CheckID("unknown"); ok {
			t.Errorf("Reply(\"unknown\") registered token")
		}
	}
}

func TestDuel(t *testing.T) {
	model := NewModel(Config{})

	model.Learn("this is a test")
	model.Learn("this is another test")

	for i := 0; i < 1000; i++ {
		reply := model.Reply("this")

		if reply != "this is a test" && reply != "this is another test" {
			t.Errorf("Reply(this is a test) => %s, want %s", reply, "this is (a|another) test")
		}
	}
}

func learnFile(m *Model, filename string) error {
	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()

	s := bufio.NewScanner(file)
	for s.Scan() {
		m.Learn(s.Text())
	}

	return s.Err()
}

func BenchmarkReply(b *testing.B) {
	m := NewModel(Config{})

	m.Learn("On two occasions I have been asked, 'Pray, Mr. Babbage, if you put into the machine wrong figures, will the right answers come out?' I am not able rightly to apprehend the kind of confusion of ideas that could provoke such a question.")

	b.ReportAllocs()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		m.Reply("apprehend")
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	"fmt"
	"github.com/jacobsa/aws/sdb"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/crypto"
	"time"
)

const (
	// The item name we use for "domain is already in use" markers.
	markerItemName = "comeback_marker"

	// The attribute name we use for storing crypto-compatibility data.
	cryptoMarkerAttributeName = "encrypted_data"
)

// A record in the backup registry describing a successful backup job.
type CompletedJob struct {
	// The name of the backup job.
	Name string

	// The time at which the backup was started.
	StartTime time.Time

	// The score representing the contents of the backup.
	Score blob.Score
}

type Registry interface {
	// Record that the named backup job has completed.
	RecordBackup(j CompletedJob) (err error)

	// Return a list of the most recent completed backups.
	ListRecentBackups() (jobs []CompletedJob, err error)
}

// Create a registry that stores data in the supplied SimpleDB domain.
//
// Before doing so, check to see whether this domain has been used as a
// registry before. If not, write an encrypted marker with the supplied
// crypter. If it has been used before, make sure that it was used with a
// crypter compatible with the supplied one. This prevents accidentally writing
// data with the wrong key, as if the user entered the wrong password.
//
// The crypter must be set up such that it is guaranteed to return an error if
// it is used to decrypt ciphertext encrypted with a different key. In that
// case, this function will return an *IncompatibleCrypterError.
func NewRegistry(
	crypter crypto.Crypter,
	domain sdb.Domain) (r Registry, err error) {
	// Set up a tentative result.
	r = &registry{crypter, domain}

	// Ask for the data that will tell us whether the crypter is compatible with
	// the previous one used in this domain, if any.
	attrs, err := domain.GetAttributes(
		markerItemName,
		false,  // No need to ask for a consistent read
		[]string{cryptoMarkerAttributeName},
	)

	if err != nil {
		err = fmt.Errorf("GetAttributes: %v", err)
		return
	}

	// If we got back an attribute and its data is decryptable, we're done.
	if len(attrs) > 0 {
		ciphertext := []byte(attrs[0].Value)
		if _, err = crypter.Decrypt(ciphertext); err != nil {
			err = fmt.Errorf("Decrypt: %v", err)
			return
		}

		return
	}

	// Otherwise, we want to claim this domain. Encrypt some random data, then
	// write it out. Make sure to use a precondition to defeat the race condition
	// where another machine is doing the same simultaneously.
	plaintext := getRandBytes()
	ciphertext, err := crypter.Encrypt(plaintext)
	if err != nil {
		err = fmt.Errorf("Encrypt: %v", err)
		return
	}

	err = domain.PutAttributes(
		markerItemName,
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: cryptoMarkerAttributeName, Value: string(ciphertext)},
		},
		&sdb.Precondition{Name: cryptoMarkerAttributeName, Value: nil},
	)

	if err != nil {
		err = fmt.Errorf("PutAttributes: %v", err)
		return
	}

	// All is good.
	return
}

func getRandBytes() []byte

type registry struct {
	crypter crypto.Crypter
	domain sdb.Domain
}

func (r *registry) RecordBackup(j CompletedJob) (err error) {
	err = fmt.Errorf("TODO")
	return
}

func (r *registry) ListRecentBackups() (jobs []CompletedJob, err error) {
	err = fmt.Errorf("TODO")
	return
}

type IncompatibleCrypterError struct {
}

func (e *IncompatibleCrypterError) Error() string {
	return "TODO"
}
<commit_msg>Implemented getRandBytes.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	"fmt"
	"github.com/jacobsa/aws/sdb"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/crypto"
	"math/rand"
	"time"
)

const (
	// The item name we use for "domain is already in use" markers.
	markerItemName = "comeback_marker"

	// The attribute name we use for storing crypto-compatibility data.
	cryptoMarkerAttributeName = "encrypted_data"
)

// A record in the backup registry describing a successful backup job.
type CompletedJob struct {
	// The name of the backup job.
	Name string

	// The time at which the backup was started.
	StartTime time.Time

	// The score representing the contents of the backup.
	Score blob.Score
}

type Registry interface {
	// Record that the named backup job has completed.
	RecordBackup(j CompletedJob) (err error)

	// Return a list of the most recent completed backups.
	ListRecentBackups() (jobs []CompletedJob, err error)
}

// Create a registry that stores data in the supplied SimpleDB domain.
//
// Before doing so, check to see whether this domain has been used as a
// registry before. If not, write an encrypted marker with the supplied
// crypter. If it has been used before, make sure that it was used with a
// crypter compatible with the supplied one. This prevents accidentally writing
// data with the wrong key, as if the user entered the wrong password.
//
// The crypter must be set up such that it is guaranteed to return an error if
// it is used to decrypt ciphertext encrypted with a different key. In that
// case, this function will return an *IncompatibleCrypterError.
func NewRegistry(
	crypter crypto.Crypter,
	domain sdb.Domain) (r Registry, err error) {
	// Set up a tentative result.
	r = &registry{crypter, domain}

	// Ask for the data that will tell us whether the crypter is compatible with
	// the previous one used in this domain, if any.
	attrs, err := domain.GetAttributes(
		markerItemName,
		false,  // No need to ask for a consistent read
		[]string{cryptoMarkerAttributeName},
	)

	if err != nil {
		err = fmt.Errorf("GetAttributes: %v", err)
		return
	}

	// If we got back an attribute and its data is decryptable, we're done.
	if len(attrs) > 0 {
		ciphertext := []byte(attrs[0].Value)
		if _, err = crypter.Decrypt(ciphertext); err != nil {
			err = fmt.Errorf("Decrypt: %v", err)
			return
		}

		return
	}

	// Otherwise, we want to claim this domain. Encrypt some random data, then
	// write it out. Make sure to use a precondition to defeat the race condition
	// where another machine is doing the same simultaneously.
	plaintext := getRandBytes()
	ciphertext, err := crypter.Encrypt(plaintext)
	if err != nil {
		err = fmt.Errorf("Encrypt: %v", err)
		return
	}

	err = domain.PutAttributes(
		markerItemName,
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: cryptoMarkerAttributeName, Value: string(ciphertext)},
		},
		&sdb.Precondition{Name: cryptoMarkerAttributeName, Value: nil},
	)

	if err != nil {
		err = fmt.Errorf("PutAttributes: %v", err)
		return
	}

	// All is good.
	return
}

func getRandBytes() []byte {
	a := rand.Uint32()
	b := rand.Uint32()

	return []byte{
		byte(a),
		byte(a >> 8),
		byte(a >> 16),
		byte(a >> 24),
		byte(b),
		byte(b >> 8),
		byte(b >> 16),
		byte(b >> 24),
	}
}

type registry struct {
	crypter crypto.Crypter
	domain sdb.Domain
}

func (r *registry) RecordBackup(j CompletedJob) (err error) {
	err = fmt.Errorf("TODO")
	return
}

func (r *registry) ListRecentBackups() (jobs []CompletedJob, err error) {
	err = fmt.Errorf("TODO")
	return
}

type IncompatibleCrypterError struct {
}

func (e *IncompatibleCrypterError) Error() string {
	return "TODO"
}
<|endoftext|>


<commit_before>/*
 * Copyright 2017 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package badger

import (
	"bytes"
	"sync"

	"github.com/dgraph-io/badger/y"
)

// KVItem is returned during iteration. Both the Key() and Value() output is only valid until
// iterator.Next() is called.
type KVItem struct {
	wg         sync.WaitGroup
	key        []byte
	vptr       []byte
	meta       byte
	val        []byte
	casCounter uint16
	slice      *y.Slice
	next       *KVItem
}

// Key returns the key. Remember to copy if you need to access it outside the iteration loop.
func (item *KVItem) Key() []byte {
	return item.key
}

// Value returns the value, generally fetched from the value log. This call can block while
// the value is populated asynchronously via a disk read. Remember to parse or copy it if you
// need to access it outside the iterator loop.
func (item *KVItem) Value() []byte {
	item.wg.Wait()
	return item.val
}

// Counter returns the CAS counter associated with the value.
func (item *KVItem) Counter() uint16 {
	return item.casCounter
}

type list struct {
	head *KVItem
	tail *KVItem
}

func (l *list) push(i *KVItem) {
	i.next = nil
	if l.tail == nil {
		l.head = i
		l.tail = i
		return
	}
	l.tail.next = i
	l.tail = i
}

func (l *list) pop() *KVItem {
	if l.head == nil {
		return nil
	}
	i := l.head
	if l.head == l.tail {
		l.tail = nil
		l.head = nil
	} else {
		l.head = i.next
	}
	i.next = nil
	return i
}

type IteratorOptions struct {
	PrefetchSize int  // How many KV pairs to prefetch while iterating.
	FetchValues  bool // Controls whether the values should be fetched from the value log.
	Reverse      bool // Direction of iteration. False is forward, true is backward.
}

var DefaultIteratorOptions = IteratorOptions{
	PrefetchSize: 100,
	FetchValues:  true,
	Reverse:      false,
}

// Iterator helps iterating over the KV pairs in a lexicographically sorted order.
type Iterator struct {
	kv   *KV
	iitr y.Iterator

	opt   IteratorOptions
	item  *KVItem
	data  list
	waste list
}

func (it *Iterator) newItem() *KVItem {
	item := it.waste.pop()
	if item == nil {
		item = &KVItem{slice: new(y.Slice)}
	}
	return item
}

// Item returns pointer to the current KVItem.
// This item is only valid until it.Next() gets called.
func (it *Iterator) Item() *KVItem { return it.item }

// Valid returns false when iteration is done.
func (it *Iterator) Valid() bool { return it.item != nil }

// Close would close the iterator. It is important to call this when you're done with iteration.
func (it *Iterator) Close() {
	it.iitr.Close()
}

// Next would advance the iterator by one. Always check it.Valid() after a Next()
// to ensure you have access to a valid it.Item().
func (it *Iterator) Next() {
	// Reuse current item
	it.item.wg.Wait() // Just cleaner to wait before pushing to avoid doing ref counting.
	it.waste.push(it.item)

	// Set next item to current
	it.item = it.data.pop()

	// Advance internal iterator until entry is not deleted
	for it.iitr.Next(); it.iitr.Valid(); it.iitr.Next() {
		if bytes.HasPrefix(it.iitr.Key(), badgerPrefix) {
			continue
		}
		if it.iitr.Value().Meta&BitDelete == 0 { // Not deleted.
			break
		}
	}

	if !it.iitr.Valid() {
		return
	}
	item := it.newItem()
	it.fill(item)
	it.data.push(item)
}

func (it *Iterator) fill(item *KVItem) {
	vs := it.iitr.Value()
	item.meta = vs.Meta
	item.casCounter = vs.CASCounter
	item.key = y.Safecopy(item.key, it.iitr.Key())
	item.vptr = y.Safecopy(item.vptr, vs.Value)
	if it.opt.FetchValues {
		item.wg.Add(1)
		go func() {
			it.kv.fillItem(item)
			item.wg.Done()
		}()
	}
}

func (it *Iterator) prefetch() {
	i := it.iitr
	var count int
	it.item = nil
	for ; i.Valid(); i.Next() {
		if bytes.HasPrefix(it.iitr.Key(), badgerPrefix) {
			continue
		}
		if i.Value().Meta&BitDelete > 0 {
			continue
		}
		count++

		item := it.newItem()
		it.fill(item)
		if it.item == nil {
			it.item = item
		} else {
			it.data.push(item)
		}
		if count == it.opt.PrefetchSize {
			break
		}
	}
}

// Seek would seek to the provided key if present. If absent, it would seek to the next smallest key
// greater than provided if iterating in the forward direction. Behavior would be reversed is
// iterating backwards.
func (it *Iterator) Seek(key []byte) {
	for i := it.data.pop(); i != nil; i = it.data.pop() {
		i.wg.Wait()
		it.waste.push(i)
	}
	it.iitr.Seek(key)
	for it.iitr.Valid() && bytes.HasPrefix(it.iitr.Key(), badgerPrefix) {
		it.iitr.Next()
	}
	it.prefetch()
}

// Rewind would rewind the iterator cursor all the way to zero-th position, which would be the
// smallest key if iterating forward, and largest if iterating backward. It does not keep track of
// whether the cursor started with a Seek().
func (it *Iterator) Rewind() {
	i := it.data.pop()
	for i != nil {
		i.wg.Wait() // Just cleaner to wait before pushing. No ref counting needed.
		it.waste.push(i)
		i = it.data.pop()
	}

	it.iitr.Rewind()
	for it.iitr.Valid() && bytes.HasPrefix(it.iitr.Key(), badgerPrefix) {
		it.iitr.Next()
	}
	it.prefetch()
}

// NewIterator returns a new iterator. Depending upon the options, either only keys, or both
// key-value pairs would be fetched. The keys are returned in lexicographically sorted order.
// Usage:
//   opt := badger.DefaultIteratorOptions
//   itr := kv.NewIterator(opt)
//   for itr.Rewind(); itr.Valid(); itr.Next() {
//     item := itr.Item()
//     key := item.Key()
//     val := item.Value() // This could block while value is fetched from value log.
//                         // For key only iteration, set opt.FetchValues to false, and don't call
//                         // item.Value().
//
//     // Remember that both key, val would become invalid in the next iteration of the loop.
//     // So, if you need access to them outside, copy them or parse them.
//   }
//   itr.Close()
func (s *KV) NewIterator(opt IteratorOptions) *Iterator {
	tables, decr := s.getMemTables()
	defer decr()
	var iters []y.Iterator
	for i := 0; i < len(tables); i++ {
		iters = append(iters, tables[i].NewUniIterator(opt.Reverse))
	}
	iters = s.lc.appendIterators(iters, opt.Reverse) // This will increment references.
	res := &Iterator{
		kv:   s,
		iitr: y.NewMergeIterator(iters, opt.Reverse),
		opt:  opt,
	}
	return res
}
<commit_msg>Clarify how to use Value() result.<commit_after>/*
 * Copyright 2017 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package badger

import (
	"bytes"
	"sync"

	"github.com/dgraph-io/badger/y"
)

// KVItem is returned during iteration. Both the Key() and Value() output is only valid until
// iterator.Next() is called.
type KVItem struct {
	wg         sync.WaitGroup
	key        []byte
	vptr       []byte
	meta       byte
	val        []byte
	casCounter uint16
	slice      *y.Slice
	next       *KVItem
}

// Key returns the key. Remember to copy if you need to access it outside the iteration loop.
func (item *KVItem) Key() []byte {
	return item.key
}

// Value returns the value, generally fetched from the value log. This call can block while
// the value is populated asynchronously via a disk read. Remember to parse or copy it if you
// need to reuse it. DO NOT append to this slice, it would result in internal data overwrite.
func (item *KVItem) Value() []byte {
	item.wg.Wait()
	return item.val
}

// Counter returns the CAS counter associated with the value.
func (item *KVItem) Counter() uint16 {
	return item.casCounter
}

type list struct {
	head *KVItem
	tail *KVItem
}

func (l *list) push(i *KVItem) {
	i.next = nil
	if l.tail == nil {
		l.head = i
		l.tail = i
		return
	}
	l.tail.next = i
	l.tail = i
}

func (l *list) pop() *KVItem {
	if l.head == nil {
		return nil
	}
	i := l.head
	if l.head == l.tail {
		l.tail = nil
		l.head = nil
	} else {
		l.head = i.next
	}
	i.next = nil
	return i
}

type IteratorOptions struct {
	PrefetchSize int  // How many KV pairs to prefetch while iterating.
	FetchValues  bool // Controls whether the values should be fetched from the value log.
	Reverse      bool // Direction of iteration. False is forward, true is backward.
}

var DefaultIteratorOptions = IteratorOptions{
	PrefetchSize: 100,
	FetchValues:  true,
	Reverse:      false,
}

// Iterator helps iterating over the KV pairs in a lexicographically sorted order.
type Iterator struct {
	kv   *KV
	iitr y.Iterator

	opt   IteratorOptions
	item  *KVItem
	data  list
	waste list
}

func (it *Iterator) newItem() *KVItem {
	item := it.waste.pop()
	if item == nil {
		item = &KVItem{slice: new(y.Slice)}
	}
	return item
}

// Item returns pointer to the current KVItem.
// This item is only valid until it.Next() gets called.
func (it *Iterator) Item() *KVItem { return it.item }

// Valid returns false when iteration is done.
func (it *Iterator) Valid() bool { return it.item != nil }

// Close would close the iterator. It is important to call this when you're done with iteration.
func (it *Iterator) Close() {
	it.iitr.Close()
}

// Next would advance the iterator by one. Always check it.Valid() after a Next()
// to ensure you have access to a valid it.Item().
func (it *Iterator) Next() {
	// Reuse current item
	it.item.wg.Wait() // Just cleaner to wait before pushing to avoid doing ref counting.
	it.waste.push(it.item)

	// Set next item to current
	it.item = it.data.pop()

	// Advance internal iterator until entry is not deleted
	for it.iitr.Next(); it.iitr.Valid(); it.iitr.Next() {
		if bytes.HasPrefix(it.iitr.Key(), badgerPrefix) {
			continue
		}
		if it.iitr.Value().Meta&BitDelete == 0 { // Not deleted.
			break
		}
	}

	if !it.iitr.Valid() {
		return
	}
	item := it.newItem()
	it.fill(item)
	it.data.push(item)
}

func (it *Iterator) fill(item *KVItem) {
	vs := it.iitr.Value()
	item.meta = vs.Meta
	item.casCounter = vs.CASCounter
	item.key = y.Safecopy(item.key, it.iitr.Key())
	item.vptr = y.Safecopy(item.vptr, vs.Value)
	if it.opt.FetchValues {
		item.wg.Add(1)
		go func() {
			it.kv.fillItem(item)
			item.wg.Done()
		}()
	}
}

func (it *Iterator) prefetch() {
	i := it.iitr
	var count int
	it.item = nil
	for ; i.Valid(); i.Next() {
		if bytes.HasPrefix(it.iitr.Key(), badgerPrefix) {
			continue
		}
		if i.Value().Meta&BitDelete > 0 {
			continue
		}
		count++

		item := it.newItem()
		it.fill(item)
		if it.item == nil {
			it.item = item
		} else {
			it.data.push(item)
		}
		if count == it.opt.PrefetchSize {
			break
		}
	}
}

// Seek would seek to the provided key if present. If absent, it would seek to the next smallest key
// greater than provided if iterating in the forward direction. Behavior would be reversed is
// iterating backwards.
func (it *Iterator) Seek(key []byte) {
	for i := it.data.pop(); i != nil; i = it.data.pop() {
		i.wg.Wait()
		it.waste.push(i)
	}
	it.iitr.Seek(key)
	for it.iitr.Valid() && bytes.HasPrefix(it.iitr.Key(), badgerPrefix) {
		it.iitr.Next()
	}
	it.prefetch()
}

// Rewind would rewind the iterator cursor all the way to zero-th position, which would be the
// smallest key if iterating forward, and largest if iterating backward. It does not keep track of
// whether the cursor started with a Seek().
func (it *Iterator) Rewind() {
	i := it.data.pop()
	for i != nil {
		i.wg.Wait() // Just cleaner to wait before pushing. No ref counting needed.
		it.waste.push(i)
		i = it.data.pop()
	}

	it.iitr.Rewind()
	for it.iitr.Valid() && bytes.HasPrefix(it.iitr.Key(), badgerPrefix) {
		it.iitr.Next()
	}
	it.prefetch()
}

// NewIterator returns a new iterator. Depending upon the options, either only keys, or both
// key-value pairs would be fetched. The keys are returned in lexicographically sorted order.
// Usage:
//   opt := badger.DefaultIteratorOptions
//   itr := kv.NewIterator(opt)
//   for itr.Rewind(); itr.Valid(); itr.Next() {
//     item := itr.Item()
//     key := item.Key()
//     val := item.Value() // This could block while value is fetched from value log.
//                         // For key only iteration, set opt.FetchValues to false, and don't call
//                         // item.Value().
//
//     // Remember that both key, val would become invalid in the next iteration of the loop.
//     // So, if you need access to them outside, copy them or parse them.
//   }
//   itr.Close()
func (s *KV) NewIterator(opt IteratorOptions) *Iterator {
	tables, decr := s.getMemTables()
	defer decr()
	var iters []y.Iterator
	for i := 0; i < len(tables); i++ {
		iters = append(iters, tables[i].NewUniIterator(opt.Reverse))
	}
	iters = s.lc.appendIterators(iters, opt.Reverse) // This will increment references.
	res := &Iterator{
		kv:   s,
		iitr: y.NewMergeIterator(iters, opt.Reverse),
		opt:  opt,
	}
	return res
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"testing"
	"time"
)

var test_program = "/$/ { }"

func startMtail(t *testing.T, log_pathnames []string, prog_pathname string) {
	w, err := NewInotifyWatcher()
	if err != nil {
		t.Errorf("Couldn't create watcher: %s", err)
	}
	p := NewProgLoader(w)
	// start server
	prog, errors := Compile("test", strings.NewReader(test_program))
	if len(errors) > 0 {
		t.Errorf("Couldn't compile program: %s", errors)
	}
	p.e.addVm("test", prog)
	if prog_pathname != "" {
		p.LoadProgs(prog_pathname)
	}
	lines := make(chan string)
	stop := make(chan bool, 1)
	line_count.Set(0)
	go p.e.run(lines, stop)
	StartMtail(lines, log_pathnames)
}

func TestHandleLogUpdates(t *testing.T) {
	if testing.Short() {
		return
	}
	// make temp dir
	workdir, err := ioutil.TempDir("", "mtail_test")
	if err != nil {
		t.Errorf("could not create temporary directory: %s", err)
	}
	defer func() {
		err := os.RemoveAll(workdir)
		if err != nil {
			t.Errorf("Could not remove temp dir: %s", err)
		}
	}()
	// touch log file
	log_filepath := path.Join(workdir, "log")
	log_file, err := os.Create(log_filepath)
	if err != nil {
		t.Errorf("could not touch log file: %s", err)
	}
	defer log_file.Close()
	pathnames := []string{log_filepath}
	startMtail(t, pathnames, "")
	ex_lines := []string{"hi", "hi2", "hi3"}
	for i, x := range ex_lines {
		// write to log file
		log_file.WriteString(x + "\n")
		// TODO(jaq): remove slow sleep
		time.Sleep(100 * time.Millisecond)
		// check log line count increase
		expected := fmt.Sprintf("%d", i+1)
		if line_count.String() != expected {
			t.Errorf("Line count not increased\n\texpected: %s\n\treceived: %s", expected, line_count.String())
		}
	}
}

func TestHandleLogRotation(t *testing.T) {
	if testing.Short() {
		return
	}
	// make temp dir
	workdir, err := ioutil.TempDir("", "mtail_test")
	if err != nil {
		t.Errorf("could not create temporary directory: %s", err)
	}
	defer func() {
		err := os.RemoveAll(workdir)
		if err != nil {
			t.Errorf("Could not remove temp dir: %s", err)
		}
	}()
	log_filepath := path.Join(workdir, "log")
	// touch log file
	log_file, err := os.Create(log_filepath)
	if err != nil {
		t.Errorf("could not touch log file: %s", err)
	}
	defer log_file.Close()
	// Create a logger
	stop := make(chan bool, 1)
	hup := make(chan bool, 1)
	pathnames := []string{log_filepath}
	startMtail(t, pathnames, "")

	go func() {
		log_file := log_file
		var err error
		i := 0
		running := true
		for running {
			select {
			case <-hup:
				// touch log file
				log_file, err = os.Create(log_filepath)
				if err != nil {
					t.Errorf("could not touch log file: %s", err)
				}
				defer log_file.Close()
			default:
				log_file.WriteString(fmt.Sprintf("%d\n", i))
				time.Sleep(100 * time.Millisecond)
				i++
				if i >= 10 {
					running = false
				}
			}
		}
		stop <- true
	}()
	go func() {
		for {
			select {
			case <-time.After(5 * 100 * time.Millisecond):
				err = os.Rename(log_filepath, log_filepath+".1")
				if err != nil {
					t.Errorf("could not rename log file: %s", err)
				}
				hup <- true
				return
			}
		}
	}()
	<-stop
	expected := "10"
	if line_count.String() != expected {
		t.Errorf("Line count not increased\n\texpected: %s\n\treceived: %s", expected, line_count.String())
	}
}

func TestHandleNewLogAfterStart(t *testing.T) {
	if testing.Short() {
		return
	}
	// make temp dir
	workdir, err := ioutil.TempDir("", "mtail_test")
	if err != nil {
		t.Errorf("could not create temporary directory: %s", err)
	}
	defer func() {
		err := os.RemoveAll(workdir)
		if err != nil {
			t.Errorf("Could not remove temp dir: %s", err)
		}
	}()
	// Start up mtail
	log_filepath := path.Join(workdir, "log")
	pathnames := []string{log_filepath}
	startMtail(t, pathnames, "")

	// touch log file
	log_file, err := os.Create(log_filepath)
	if err != nil {
		t.Errorf("could not touch log file: %s", err)
	}
	defer log_file.Close()
	ex_lines := []string{"hi", "hi2", "hi3"}
	for _, x := range ex_lines {
		// write to log file
		log_file.WriteString(x + "\n")
	}
	// TODO(jaq): remove slow sleep
	time.Sleep(100 * time.Millisecond)
	// check log line count increase
	expected := fmt.Sprintf("%d", len(ex_lines))
	if line_count.String() != expected {
		t.Errorf("Line count not increased\n\texpected: %s\n\treceived: %s", expected, line_count.String())
	}
}

func TestHandleNewLogIgnored(t *testing.T) {
	if testing.Short() {
		return
	}
	// make temp dir
	workdir, err := ioutil.TempDir("", "mtail_test")
	if err != nil {
		t.Errorf("could not create temporary directory: %s", err)
	}
	defer func() {
		err := os.RemoveAll(workdir)
		if err != nil {
			t.Errorf("Could not remove temp dir: %s", err)
		}
	}()
	// Start mtail
	log_filepath := path.Join(workdir, "log")
	pathnames := []string{log_filepath}
	startMtail(t, pathnames, "")

	// touch log file
	new_log_filepath := path.Join(workdir, "log1")

	log_file, err := os.Create(new_log_filepath)
	if err != nil {
		t.Errorf("could not touch log file: %s", err)
	}
	defer log_file.Close()
	expected := "0"
	if line_count.String() != expected {
		t.Errorf("Line count not increased\n\texpected: %s\n\treceived: %s", expected, line_count.String())
	}
}

func makeTempDir(t *testing.T) (workdir string) {
	var err error
	if workdir, err = ioutil.TempDir("", "mtail_test"); err != nil {
		t.Errorf("ioutil.TempDir failed: %s", err)
	}
	return
}

func removeTempDir(t *testing.T, workdir string) {
	if err := os.RemoveAll(workdir); err != nil {
		t.Errorf("os.RemoveAll failed: %s", err)
	}
}

// TODO(jaq): The sleeps in here are racy.  What can we use to sync through inotify?
func TestHandleNewProgram(t *testing.T) {
	if testing.Short() {
		return
	}

	workdir := makeTempDir(t)
	defer removeTempDir(t, workdir)

	startMtail(t, []string{}, workdir)

	expected_prog_loads := "{}"
	if prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, prog_loads.String())
	}

	prog_path := path.Join(workdir, "prog.em")
	prog_file, err := os.Create(prog_path)
	if err != nil {
		t.Errorf("prog create failed: %s", err)
	}
	prog_file.WriteString("/$/ {}\n")
	prog_file.Close()

	// Wait for inotify
	time.Sleep(100 * time.Millisecond)
	expected_prog_loads = `{"prog.em": 1}`
	if prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, prog_loads.String())
	}

	bad_prog_path := path.Join(workdir, "prog.em.dpkg-dist")
	bad_prog_file, err := os.Create(bad_prog_path)
	if err != nil {
		t.Errorf("prog create failed: %s", err)
	}
	bad_prog_file.WriteString("/$/ {}\n")
	bad_prog_file.Close()

	time.Sleep(100 * time.Millisecond)
	expected_prog_loads = `{"prog.em": 1}`
	if prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, prog_loads.String())
	}
	expected_prog_errs := `{}`
	if prog_load_errors.String() != expected_prog_errs {
		t.Errorf("Prog errors not same\n\texpected: %s\n\treceived: %s", expected_prog_errs, prog_load_errors.String())
	}

	os.Rename(bad_prog_path, prog_path)
	time.Sleep(100 * time.Millisecond)
	expected_prog_loads = `{"prog.em": 2}`
	if prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, prog_loads.String())
	}
	expected_prog_errs = `{}`
	if prog_load_errors.String() != expected_prog_errs {
		t.Errorf("Prog errors not same\n\texpected: %s\n\treceived: %s", expected_prog_errs, prog_load_errors.String())
	}

	broken_prog_path := path.Join(workdir, "broken.em")
	broken_prog_file, err := os.Create(broken_prog_path)
	if err != nil {
		t.Errorf("prog create failed: %s", err)
	}
	broken_prog_file.WriteString("?\n")
	broken_prog_file.Close()

	time.Sleep(100 * time.Millisecond)

	expected_prog_loads = `{"prog.em": 2}`
	if prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, prog_loads.String())
	}
	expected_prog_errs = `{"broken.em": 1}`
	if prog_load_errors.String() != expected_prog_errs {
		t.Errorf("Prog errors not same\n\texpected: %s\n\treceived: %s", expected_prog_errs, prog_load_errors.String())
	}

}
<commit_msg>Stop the mtail under test in functional tests.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"testing"
	"time"
)

var test_program = "/$/ { }"

func startMtail(t *testing.T, log_pathnames []string, prog_pathname string) chan bool {
	w, err := NewInotifyWatcher()
	if err != nil {
		t.Errorf("Couldn't create watcher: %s", err)
	}
	p := NewProgLoader(w)
	// start server
	prog, errors := Compile("test", strings.NewReader(test_program))
	if len(errors) > 0 {
		t.Errorf("Couldn't compile program: %s", errors)
	}
	p.e.addVm("test", prog)
	if prog_pathname != "" {
		p.LoadProgs(prog_pathname)
	}
	lines := make(chan string)
	stop := make(chan bool, 1)
	line_count.Set(0)
	go p.e.run(lines, stop)
	StartMtail(lines, log_pathnames)
	return stop
}

func TestHandleLogUpdates(t *testing.T) {
	if testing.Short() {
		return
	}
	// make temp dir
	workdir, err := ioutil.TempDir("", "mtail_test")
	if err != nil {
		t.Errorf("could not create temporary directory: %s", err)
	}
	defer func() {
		err := os.RemoveAll(workdir)
		if err != nil {
			t.Errorf("Could not remove temp dir: %s", err)
		}
	}()
	// touch log file
	log_filepath := path.Join(workdir, "log")
	log_file, err := os.Create(log_filepath)
	if err != nil {
		t.Errorf("could not touch log file: %s", err)
	}
	defer log_file.Close()
	pathnames := []string{log_filepath}
	stop := startMtail(t, pathnames, "")
	defer func() { stop <- true }()
	ex_lines := []string{"hi", "hi2", "hi3"}
	for i, x := range ex_lines {
		// write to log file
		log_file.WriteString(x + "\n")
		// TODO(jaq): remove slow sleep
		time.Sleep(100 * time.Millisecond)
		// check log line count increase
		expected := fmt.Sprintf("%d", i+1)
		if line_count.String() != expected {
			t.Errorf("Line count not increased\n\texpected: %s\n\treceived: %s", expected, line_count.String())
		}
	}
}

func TestHandleLogRotation(t *testing.T) {
	if testing.Short() {
		return
	}
	// make temp dir
	workdir, err := ioutil.TempDir("", "mtail_test")
	if err != nil {
		t.Errorf("could not create temporary directory: %s", err)
	}
	defer func() {
		err := os.RemoveAll(workdir)
		if err != nil {
			t.Errorf("Could not remove temp dir: %s", err)
		}
	}()
	log_filepath := path.Join(workdir, "log")
	// touch log file
	log_file, err := os.Create(log_filepath)
	if err != nil {
		t.Errorf("could not touch log file: %s", err)
	}
	defer log_file.Close()
	// Create a logger
	stop := make(chan bool, 1)
	hup := make(chan bool, 1)
	pathnames := []string{log_filepath}
	end := startMtail(t, pathnames, "")
	defer func() { end <- true }()

	go func() {
		log_file := log_file
		var err error
		i := 0
		running := true
		for running {
			select {
			case <-hup:
				// touch log file
				log_file, err = os.Create(log_filepath)
				if err != nil {
					t.Errorf("could not touch log file: %s", err)
				}
				defer log_file.Close()
			default:
				log_file.WriteString(fmt.Sprintf("%d\n", i))
				time.Sleep(100 * time.Millisecond)
				i++
				if i >= 10 {
					running = false
				}
			}
		}
		stop <- true
	}()
	go func() {
		for {
			select {
			case <-time.After(5 * 100 * time.Millisecond):
				err = os.Rename(log_filepath, log_filepath+".1")
				if err != nil {
					t.Errorf("could not rename log file: %s", err)
				}
				hup <- true
				return
			}
		}
	}()
	<-stop
	expected := "10"
	if line_count.String() != expected {
		t.Errorf("Line count not increased\n\texpected: %s\n\treceived: %s", expected, line_count.String())
	}
}

func TestHandleNewLogAfterStart(t *testing.T) {
	if testing.Short() {
		return
	}
	// make temp dir
	workdir, err := ioutil.TempDir("", "mtail_test")
	if err != nil {
		t.Errorf("could not create temporary directory: %s", err)
	}
	defer func() {
		err := os.RemoveAll(workdir)
		if err != nil {
			t.Errorf("Could not remove temp dir: %s", err)
		}
	}()
	// Start up mtail
	log_filepath := path.Join(workdir, "log")
	pathnames := []string{log_filepath}
	stop := startMtail(t, pathnames, "")
	defer func() { stop <- true }()

	// touch log file
	log_file, err := os.Create(log_filepath)
	if err != nil {
		t.Errorf("could not touch log file: %s", err)
	}
	defer log_file.Close()
	ex_lines := []string{"hi", "hi2", "hi3"}
	for _, x := range ex_lines {
		// write to log file
		log_file.WriteString(x + "\n")
		log_file.Sync()
	}
	// TODO(jaq): remove slow sleep
	time.Sleep(100 * time.Millisecond)
	// check log line count increase
	expected := fmt.Sprintf("%d", len(ex_lines))
	if line_count.String() != expected {
		t.Errorf("Line count not increased\n\texpected: %s\n\treceived: %s", expected, line_count.String())
	}
}

func TestHandleNewLogIgnored(t *testing.T) {
	if testing.Short() {
		return
	}
	// make temp dir
	workdir, err := ioutil.TempDir("", "mtail_test")
	if err != nil {
		t.Errorf("could not create temporary directory: %s", err)
	}
	defer func() {
		err := os.RemoveAll(workdir)
		if err != nil {
			t.Errorf("Could not remove temp dir: %s", err)
		}
	}()
	// Start mtail
	log_filepath := path.Join(workdir, "log")
	pathnames := []string{log_filepath}
	stop := startMtail(t, pathnames, "")
	defer func() { stop <- true }()

	// touch log file
	new_log_filepath := path.Join(workdir, "log1")

	log_file, err := os.Create(new_log_filepath)
	if err != nil {
		t.Errorf("could not touch log file: %s", err)
	}
	defer log_file.Close()
	expected := "0"
	if line_count.String() != expected {
		t.Errorf("Line count not increased\n\texpected: %s\n\treceived: %s", expected, line_count.String())
	}
}

func makeTempDir(t *testing.T) (workdir string) {
	var err error
	if workdir, err = ioutil.TempDir("", "mtail_test"); err != nil {
		t.Errorf("ioutil.TempDir failed: %s", err)
	}
	return
}

func removeTempDir(t *testing.T, workdir string) {
	if err := os.RemoveAll(workdir); err != nil {
		t.Errorf("os.RemoveAll failed: %s", err)
	}
}

// TODO(jaq): The sleeps in here are racy.  What can we use to sync through inotify?
func TestHandleNewProgram(t *testing.T) {
	if testing.Short() {
		return
	}

	workdir := makeTempDir(t)
	defer removeTempDir(t, workdir)

	stop := startMtail(t, []string{}, workdir)
	defer func() { stop <- true }()

	expected_prog_loads := "{}"
	if prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, prog_loads.String())
	}

	prog_path := path.Join(workdir, "prog.em")
	prog_file, err := os.Create(prog_path)
	if err != nil {
		t.Errorf("prog create failed: %s", err)
	}
	prog_file.WriteString("/$/ {}\n")
	prog_file.Close()

	// Wait for inotify
	time.Sleep(100 * time.Millisecond)
	expected_prog_loads = `{"prog.em": 1}`
	if prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, prog_loads.String())
	}

	bad_prog_path := path.Join(workdir, "prog.em.dpkg-dist")
	bad_prog_file, err := os.Create(bad_prog_path)
	if err != nil {
		t.Errorf("prog create failed: %s", err)
	}
	bad_prog_file.WriteString("/$/ {}\n")
	bad_prog_file.Close()

	time.Sleep(100 * time.Millisecond)
	expected_prog_loads = `{"prog.em": 1}`
	if prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, prog_loads.String())
	}
	expected_prog_errs := `{}`
	if prog_load_errors.String() != expected_prog_errs {
		t.Errorf("Prog errors not same\n\texpected: %s\n\treceived: %s", expected_prog_errs, prog_load_errors.String())
	}

	os.Rename(bad_prog_path, prog_path)
	time.Sleep(100 * time.Millisecond)
	expected_prog_loads = `{"prog.em": 2}`
	if prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, prog_loads.String())
	}
	expected_prog_errs = `{}`
	if prog_load_errors.String() != expected_prog_errs {
		t.Errorf("Prog errors not same\n\texpected: %s\n\treceived: %s", expected_prog_errs, prog_load_errors.String())
	}

	broken_prog_path := path.Join(workdir, "broken.em")
	broken_prog_file, err := os.Create(broken_prog_path)
	if err != nil {
		t.Errorf("prog create failed: %s", err)
	}
	broken_prog_file.WriteString("?\n")
	broken_prog_file.Close()

	time.Sleep(100 * time.Millisecond)

	expected_prog_loads = `{"prog.em": 2}`
	if prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, prog_loads.String())
	}
	expected_prog_errs = `{"broken.em": 1}`
	if prog_load_errors.String() != expected_prog_errs {
		t.Errorf("Prog errors not same\n\texpected: %s\n\treceived: %s", expected_prog_errs, prog_load_errors.String())
	}

}
<|endoftext|>


<commit_before>package main

import (
  "flag"
  "fmt"
  "io"
  "log"
  "os"
  "path/filepath"
  "time"
)

var (
  output_name string
)

func init() {
  flag.StringVar(&output_name, "output", output_name, "Output filename (- for STDOUT, do not specify for default, multifasta.txt)")
  flag.Parse()
}

func defaultOutput(inputPath string) string {
  dir := filepath.Dir(inputPath)
  t := time.Now()
  outfile := fmt.Sprintf("multifasta_output_%d%02d%02d-%02d%02d.txt",
    t.Year(),
    t.Month(),
    t.Day(),
    t.Hour(),
    t.Minute())
  return filepath.Join(dir,outfile)
}

func openOutput(outputPath string) *os.File {
  switch outputPath {
  case "-":
    return os.Stdout
  default:
    out, err := os.Create(outputPath)
    if nil != err {
      log.Fatal(err)
    }
    return out
  }
}

func main() {
  if len(flag.Args()) <= 0 {
    fmt.Fprintf(os.Stderr,"Missing one or more input file(s)\n")
    flag.Usage()
    os.Exit(1)
  }
  if "" == output_name {
    output_name = defaultOutput(flag.Args()[0])
  }
  out := openOutput(output_name)

  defer out.Close()

  for _, infile := range flag.Args() {
    basefile := filepath.Base(infile)
    extension := filepath.Ext(basefile)
    fmt.Fprintf(out,">%s exported from %s\n", basefile[:len(basefile)-len(extension)], basefile)
    in, err := os.Open(infile)
    if err != nil {
      log.Fatal(err)
    }
    io.Copy(out,in)
    in.Close()
    fmt.Fprintln(out,"")
  }
}
<commit_msg>gofmt<commit_after>package main

import (
  "flag"
  "fmt"
  "io"
  "log"
  "os"
  "path/filepath"
  "time"
)

var (
  output_name string
)

func init() {
  flag.StringVar(&output_name, "output", output_name, "Output filename (- for STDOUT, do not specify for default, multifasta.txt)")
  flag.Parse()
}

func defaultOutput(inputPath string) string {
  dir := filepath.Dir(inputPath)
  t := time.Now()
  outfile := fmt.Sprintf("multifasta_output_%d%02d%02d-%02d%02d.txt",
    t.Year(),
    t.Month(),
    t.Day(),
    t.Hour(),
    t.Minute())
  return filepath.Join(dir, outfile)
}

func openOutput(outputPath string) *os.File {
  switch outputPath {
  case "-":
    return os.Stdout
  default:
    out, err := os.Create(outputPath)
    if nil != err {
      log.Fatal(err)
    }
    return out
  }
}

func main() {
  if len(flag.Args()) <= 0 {
    fmt.Fprintf(os.Stderr, "Missing one or more input file(s)\n")
    flag.Usage()
    os.Exit(1)
  }
  if "" == output_name {
    output_name = defaultOutput(flag.Args()[0])
  }
  out := openOutput(output_name)

  defer out.Close()

  for _, infile := range flag.Args() {
    basefile := filepath.Base(infile)
    extension := filepath.Ext(basefile)
    fmt.Fprintf(out, ">%s exported from %s\n", basefile[:len(basefile)-len(extension)], basefile)
    in, err := os.Open(infile)
    if err != nil {
      log.Fatal(err)
    }
    io.Copy(out, in)
    in.Close()
    fmt.Fprintln(out, "")
  }
}
<|endoftext|>


<commit_before>package redisync

import (
	"testing"
	"time"
)

func TestLock(t *testing.T) {
	ttl := time.Second * 2
	m, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	m.Lock()
	time.Sleep(ttl)
	ok := m.TryLock()
	if !ok {
		t.Error("Expected mutex to be lockable.")
		t.FailNow()
	}
}
<commit_msg>more tests<commit_after>package redisync

import (
	"testing"
	"time"
)

func TestLock(t *testing.T) {
	ttl := time.Second
	m, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	m.Lock()
	time.Sleep(ttl)
	ok := m.TryLock()
	if !ok {
		t.Error("Expected mutex to be lockable.")
		t.FailNow()
	}
	m.Unlock()
}

func TestLockLocked(t *testing.T) {
	ttl := time.Second
	m1, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	if ok := m1.TryLock(); !ok {
		t.Error("Expected mutex to be lockable.")
		t.FailNow()
	}
	m2, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	if ok := m2.TryLock(); ok {
		t.Error("Expected mutex not to be lockable.")
		t.FailNow()
	}
	m1.Unlock()
}

func TestUnlockOtherLocked(t *testing.T) {
	ttl := time.Second
	m1, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	if ok := m1.TryLock(); !ok {
		t.Error("Expected mutex to be lockable.")
		t.FailNow()
	}

	m2, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	if ok, _ := m2.Unlock(); ok {
		t.Error("Expected mutex not to be unlockable.")
		t.FailNow()
	}
}
<|endoftext|>


<commit_before>package mysql

import (
	"testing"
)

func SelectSingleRow(t *testing.T, q string) map[string]string {
	dbh, err := Connect("tcp", "", "127.0.0.1:3306", "test", "test", "")
	if err != nil {
		t.Error(err)
		t.FailNow()
	}
	if dbh == nil {
		t.Error("dbh is nil")
		t.FailNow()
	}
	dbh.Use("test")

	res, err := dbh.Query("SET NAMES utf8")
	res, err = dbh.Query(q)

	row := res.FetchRowMap()
	dbh.Quit()
	return row
}

func SelectSingleRowPrepared(t *testing.T, q string, p ...) map[string]string {
	dbh, err := Connect("tcp", "", "127.0.0.1:3306", "test", "test", "")
	if err != nil {
		t.Error(err)
		t.FailNow()
	}
	if dbh == nil {
		t.Error("dbh is nil")
		t.FailNow()
	}
	dbh.Use("test")

	res, err := dbh.Query("SET NAMES utf8")
	sth, err := dbh.Prepare(q)

	if err != nil {
		t.Error(err)
		t.FailNow()
	}
	res, err = sth.Execute(p)
	row := res.FetchRowMap()
	dbh.Quit()
	return row
}

func TestSelectString(t *testing.T) {
	row := SelectSingleRow(t, "SELECT * FROM test WHERE name='test1'")
	test := "1234567890abcdef"
	if row == nil || row["stuff"] != test {
		t.Error(row["stuff"], " != ", test)
	}
}

func TestSelectStringPrepared(t *testing.T) {
	row := SelectSingleRowPrepared(t, "SELECT * FROM test WHERE name=?", "test1")
	test := "1234567890abcdef"
	if row == nil || row["stuff"] != test {
		t.Error(row["stuff"], " != ", test)
	}
}

func TestSelectUFT8(t *testing.T) {
	row := SelectSingleRow(t, "SELECT * FROM test WHERE name='unicodetest1'")
	test := "l̡̡̡ ̴̡ı̴̴̡ ̡̡͡|̲̲̲͡͡͡ ̲▫̲͡ ̲̲̲͡͡π̲̲͡͡ ̲̲͡▫̲̲͡͡ ̲|̡̡̡ ̡ ̴̡ı̴̡̡ ̡"
	if row == nil || row["stuff"] != test {
		t.Error(row["stuff"], " != ", test)
	}
}

func TestSelectUFT8Prepared(t *testing.T) {
	row := SelectSingleRowPrepared(t, "SELECT * FROM test WHERE name=?", "unicodetest1")
	test := "l̡̡̡ ̴̡ı̴̴̡ ̡̡͡|̲̲̲͡͡͡ ̲▫̲͡ ̲̲̲͡͡π̲̲͡͡ ̲̲͡▫̲̲͡͡ ̲|̡̡̡ ̡ ̴̡ı̴̡̡ ̡"
	if row == nil || row["stuff"] != test {
		t.Error(row["stuff"], " != ", test)
	}
}

func TestSelectEmpty(t *testing.T) {
	row := SelectSingleRowPrepared(t, "SELECT * FROM test WHERE name='doesnotexist'")
	if row != nil {
		t.Error("Row is not nil")
	}
}

func TestError(t *testing.T) {
	dbh, err := Connect("tcp", "", "127.0.0.1:3306", "test", "test", "")
	if err != nil {
		t.Error(err)
		t.FailNow()
	}
	if dbh == nil {
		t.Error("dbh is nil")
		t.FailNow()
	}
	dbh.Use("test")

	res, err := dbh.Query("SELECT * FROM test WHERE namefail='foo'")
	if res != nil || err == nil {
		t.Error("err == nil, expected error")
	}
	dbh.Quit()
}
<commit_msg>Fimplify a bit. Add new test.<commit_after>package mysql

import (
	"testing"
)

func MakeDbh(t *testing.T) *MySQLInstance {
	dbh, err := Connect("tcp", "", "127.0.0.1:3306", "test", "test", "test")
	if err != nil {
		t.Error(err)
		t.FailNow()
	}
	if dbh == nil {
		t.Error("dbh is nil")
		t.FailNow()
	}
	return dbh
}

func CheckQuery(t *testing.T, dbh *MySQLInstance, q string) *MySQLResponse {
	res, err := dbh.Query(q)
	if err != nil {
		t.Error(err)
		t.FailNow()
	}
	return res
}

func SelectSingleRow(t *testing.T, q string) map[string]string {
	dbh := MakeDbh(t)
	dbh.Use("test")

	res := CheckQuery(t, dbh, "SET NAMES utf8")
	res = CheckQuery(t, dbh, q)
	row := res.FetchRowMap()
	dbh.Quit()
	return row
}

func SelectSingleRowPrepared(t *testing.T, q string, p ...) map[string]string {
	dbh := MakeDbh(t)
	dbh.Use("test")

	res, err := dbh.Query("SET NAMES utf8")
	sth, err := dbh.Prepare(q)

	if err != nil {
		t.Error(err)
		t.FailNow()
	}
	res, err = sth.Execute(p)
	row := res.FetchRowMap()
	dbh.Quit()
	return row
}

func TestUnfinished(t *testing.T) {
	dbh := MakeDbh(t)
	res := CheckQuery(t, dbh, "SELECT * FROM test")
	row := res.FetchRowMap()
	res = CheckQuery(t, dbh, "SELECT * FROM test WHERE name='test1'")
	row = res.FetchRowMap()
	test := "1234567890abcdef"
	if row == nil || row["stuff"] != test {
		t.Error(row["stuff"], " != ", test)
	}
	dbh.Quit()
}

func TestSelectString(t *testing.T) {
	row := SelectSingleRow(t, "SELECT * FROM test WHERE name='test1'")
	test := "1234567890abcdef"
	if row == nil || row["stuff"] != test {
		t.Error(row["stuff"], " != ", test)
	}
}

func TestSelectStringPrepared(t *testing.T) {
	row := SelectSingleRowPrepared(t, "SELECT * FROM test WHERE name=?", "test1")
	test := "1234567890abcdef"
	if row == nil || row["stuff"] != test {
		t.Error(row["stuff"], " != ", test)
	}
}

func TestSelectUFT8(t *testing.T) {
	row := SelectSingleRow(t, "SELECT * FROM test WHERE name='unicodetest1'")
	test := "l̡̡̡ ̴̡ı̴̴̡ ̡̡͡|̲̲̲͡͡͡ ̲▫̲͡ ̲̲̲͡͡π̲̲͡͡ ̲̲͡▫̲̲͡͡ ̲|̡̡̡ ̡ ̴̡ı̴̡̡ ̡"
	if row == nil || row["stuff"] != test {
		t.Error(row["stuff"], " != ", test)
	}
}

func TestSelectUFT8Prepared(t *testing.T) {
	row := SelectSingleRowPrepared(t, "SELECT * FROM test WHERE name=?", "unicodetest1")
	test := "l̡̡̡ ̴̡ı̴̴̡ ̡̡͡|̲̲̲͡͡͡ ̲▫̲͡ ̲̲̲͡͡π̲̲͡͡ ̲̲͡▫̲̲͡͡ ̲|̡̡̡ ̡ ̴̡ı̴̡̡ ̡"
	if row == nil || row["stuff"] != test {
		t.Error(row["stuff"], " != ", test)
	}
}

func TestSelectEmpty(t *testing.T) {
	row := SelectSingleRowPrepared(t, "SELECT * FROM test WHERE name='doesnotexist'")
	if row != nil {
		t.Error("Row is not nil")
	}
}

func TestError(t *testing.T) {
	dbh, err := Connect("tcp", "", "127.0.0.1:3306", "test", "test", "")
	if err != nil {
		t.Error(err)
		t.FailNow()
	}
	if dbh == nil {
		t.Error("dbh is nil")
		t.FailNow()
	}
	dbh.Use("test")

	res, err := dbh.Query("SELECT * FROM test WHERE namefail='foo'")
	if res != nil || err == nil {
		t.Error("err == nil, expected error")
	}
	dbh.Quit()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"net"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"
	"time"

	"github.com/docker/containerd/api/grpc/types"
	"github.com/docker/containerd/osutils"
	"github.com/golang/glog"
	"github.com/hyperhq/runv/containerd/api/grpc/server"
	"github.com/hyperhq/runv/driverloader"
	"github.com/hyperhq/runv/factory"
	"github.com/hyperhq/runv/hypervisor"
	"github.com/hyperhq/runv/supervisor"
	"google.golang.org/grpc"
)

func runvNamespaceDaemon() {
	var (
		namespace string
		state     string
		driver    string
		kernel    string
		initrd    string
	)
	flag.StringVar(&namespace, "namespace", "", "")
	flag.StringVar(&state, "state", "", "")
	flag.StringVar(&driver, "driver", "", "")
	flag.StringVar(&kernel, "kernel", "", "")
	flag.StringVar(&initrd, "initrd", "", "")
	flag.Parse()

	hypervisor.InterfaceCount = 0
	var err error
	if hypervisor.HDriver, err = driverloader.Probe(driver); err != nil {
		glog.V(1).Infof("%s\n", err.Error())
		os.Exit(1)
	}

	daemon(namespace, state, kernel, initrd)
}

func daemon(namespace, state, kernel, initrd string) error {
	// setup a standard reaper so that we don't leave any zombies if we are still alive
	// this is just good practice because we are spawning new processes
	s := make(chan os.Signal, 2048)
	signal.Notify(s, syscall.SIGCHLD, syscall.SIGTERM, syscall.SIGINT)

	// TODO: make the factory create only one vm atmost
	f := factory.NewFromConfigs(kernel, initrd, nil)
	sv, err := supervisor.New(state, namespace, f)
	if err != nil {
		return err
	}

	address := filepath.Join(namespace, "namespaced.sock")
	server, err := startServer(address, sv)
	if err != nil {
		return err
	}
	go namespaceShare(sv, namespace, state, server)

	for ss := range s {
		switch ss {
		case syscall.SIGCHLD:
			if _, err := osutils.Reap(); err != nil {
				glog.Infof("containerd: reap child processes")
			}
		default:
			glog.Infof("stopping containerd after receiving %s", ss)
			server.Stop()
			os.RemoveAll(namespace)
			os.Exit(0)
		}
	}
	return nil
}

func namespaceShare(sv *supervisor.Supervisor, namespace, state string, server *grpc.Server) {
	events := sv.Events.Events(time.Time{})
	containerCount := 0
	for e := range events {
		if e.Type == supervisor.EventContainerStart {
			os.Symlink(namespace, filepath.Join(state, e.ID, "namespace"))
			containerCount++
		} else if e.Type == supervisor.EventExit && e.PID == "init" {
			containerCount--
			if containerCount == 0 {
				server.Stop()
				os.RemoveAll(namespace)
				time.Sleep(time.Second)
				os.Exit(0)
			}
		}
	}
}

func startServer(address string, sv *supervisor.Supervisor) (*grpc.Server, error) {
	if err := os.RemoveAll(address); err != nil {
		return nil, err
	}
	l, err := net.Listen("unix", address)
	if err != nil {
		return nil, err
	}
	s := grpc.NewServer()
	types.RegisterAPIServer(s, server.NewServer(sv))
	go func() {
		glog.Infof("containerd: grpc api on %s", address)
		if err := s.Serve(l); err != nil {
			glog.Infof("containerd: serve grpc error")
		}
	}()
	return s, nil
}
<commit_msg>workaround: delay to stop server after receive the container exit event<commit_after>package main

import (
	"flag"
	"net"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"
	"time"

	"github.com/docker/containerd/api/grpc/types"
	"github.com/docker/containerd/osutils"
	"github.com/golang/glog"
	"github.com/hyperhq/runv/containerd/api/grpc/server"
	"github.com/hyperhq/runv/driverloader"
	"github.com/hyperhq/runv/factory"
	"github.com/hyperhq/runv/hypervisor"
	"github.com/hyperhq/runv/supervisor"
	"google.golang.org/grpc"
)

func runvNamespaceDaemon() {
	var (
		namespace string
		state     string
		driver    string
		kernel    string
		initrd    string
	)
	flag.StringVar(&namespace, "namespace", "", "")
	flag.StringVar(&state, "state", "", "")
	flag.StringVar(&driver, "driver", "", "")
	flag.StringVar(&kernel, "kernel", "", "")
	flag.StringVar(&initrd, "initrd", "", "")
	flag.Parse()

	hypervisor.InterfaceCount = 0
	var err error
	if hypervisor.HDriver, err = driverloader.Probe(driver); err != nil {
		glog.V(1).Infof("%s\n", err.Error())
		os.Exit(1)
	}

	daemon(namespace, state, kernel, initrd)
}

func daemon(namespace, state, kernel, initrd string) error {
	// setup a standard reaper so that we don't leave any zombies if we are still alive
	// this is just good practice because we are spawning new processes
	s := make(chan os.Signal, 2048)
	signal.Notify(s, syscall.SIGCHLD, syscall.SIGTERM, syscall.SIGINT)

	// TODO: make the factory create only one vm atmost
	f := factory.NewFromConfigs(kernel, initrd, nil)
	sv, err := supervisor.New(state, namespace, f)
	if err != nil {
		return err
	}

	address := filepath.Join(namespace, "namespaced.sock")
	server, err := startServer(address, sv)
	if err != nil {
		return err
	}
	go namespaceShare(sv, namespace, state, server)

	for ss := range s {
		switch ss {
		case syscall.SIGCHLD:
			if _, err := osutils.Reap(); err != nil {
				glog.Infof("containerd: reap child processes")
			}
		default:
			glog.Infof("stopping containerd after receiving %s", ss)
			server.Stop()
			os.RemoveAll(namespace)
			os.Exit(0)
		}
	}
	return nil
}

func namespaceShare(sv *supervisor.Supervisor, namespace, state string, server *grpc.Server) {
	events := sv.Events.Events(time.Time{})
	containerCount := 0
	for e := range events {
		if e.Type == supervisor.EventContainerStart {
			os.Symlink(namespace, filepath.Join(state, e.ID, "namespace"))
			containerCount++
		} else if e.Type == supervisor.EventExit && e.PID == "init" {
			containerCount--
			if containerCount == 0 {
				os.RemoveAll(namespace)
				time.Sleep(3 * time.Second)
				server.Stop()
				os.Exit(0)
			}
		}
	}
}

func startServer(address string, sv *supervisor.Supervisor) (*grpc.Server, error) {
	if err := os.RemoveAll(address); err != nil {
		return nil, err
	}
	l, err := net.Listen("unix", address)
	if err != nil {
		return nil, err
	}
	s := grpc.NewServer()
	types.RegisterAPIServer(s, server.NewServer(sv))
	go func() {
		glog.Infof("containerd: grpc api on %s", address)
		if err := s.Serve(l); err != nil {
			glog.Infof("containerd: serve grpc error")
		}
	}()
	return s, nil
}
<|endoftext|>