<commit_before>package tracker

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"os"
	"os/exec"
	"path"
	"strconv"
	"testing"
	"time"

	"github.com/jackpal/Taipei-Torrent/torrent"
)

func TestScrapeURL(t *testing.T) {
	tests := []struct{ announce, scrape string }{
		{"", ""},
		{"foo", ""},
		{"x/announce", "x/scrape"},
		{"x/announce?ad#3", "x/scrape?ad#3"},
		{"announce/x", ""},
	}
	for _, test := range tests {
		scrape := ScrapePattern(test.announce)
		if scrape != test.scrape {
			t.Errorf("ScrapeURL(%#v) = %#v. Expected %#v", test.announce, scrape, test.scrape)
		}
	}
}

func TestSwarm1(t *testing.T) {
	testSwarm(t, 1)
}

func TestSwarm10(t *testing.T) {
	testSwarm(t, 10)
}

/* Larger sizes don't work correctly.

func TestSwarm20(t *testing.T) {
	testSwarm(t, 20)
}

func TestSwarm50(t *testing.T) {
	testSwarm(t, 50)
}

func TestSwarm100(t *testing.T) {
	testSwarm(t, 100)
}

*/

func testSwarm(t *testing.T, leechCount int) {
	err := runSwarm(leechCount)
	if err != nil {
		t.Fatal("Error running testSwarm", err)
	}
}

type prog struct {
	instanceName string
	dirName      string
	cmd          *exec.Cmd
}

func (p *prog) start(doneCh chan *prog) (err error) {
	log.Println("starting", p.instanceName)
	out := logWriter(p.instanceName)
	p.cmd.Stdout = &out
	p.cmd.Stderr = &out
	err = p.cmd.Start()
	if err != nil {
		return
	}
	go func() {
		p.cmd.Wait()
		doneCh <- p
	}()
	return
}

func (p *prog) kill() (err error) {
	err = p.cmd.Process.Kill()
	return
}

func newProg(instanceName string, dir string, command string, arg ...string) (p *prog) {
	cmd := helperCommands(append([]string{command}, arg...)...)
	return &prog{instanceName: instanceName, dirName: dir, cmd: cmd}
}

func runSwarm(leechCount int) (err error) {
	var rootDir string
	rootDir, err = ioutil.TempDir("", "swarm")
	if err != nil {
		return
	}
	log.Printf("Temporary directory: %s", rootDir)
	seedDir := path.Join(rootDir, "seed")
	err = os.Mkdir(seedDir, 0700)
	if err != nil {
		return
	}
	seedData := path.Join(seedDir, "data")
	err = createDataFile(seedData, 1024*1024)
	if err != nil {
		return
	}
	torrentFile := path.Join(rootDir, "testSwarm.torrent")
	err = createTorrentFile(torrentFile, seedData, "127.0.0.1:8080/announce")
	if err != nil {
		return
	}

	doneCh := make(chan *prog, 1)

	tracker := newTracker("tracker", ":8080", rootDir, torrentFile)
	err = tracker.start(doneCh)
	if err != nil {
		return
	}
	defer tracker.kill()
	time.Sleep(100 * time.Microsecond)

	var seed, leech *prog
	seed = newTorrentClient("seed", 7000, torrentFile, seedDir, math.Inf(0))
	err = seed.start(doneCh)
	if err != nil {
		return
	}
	defer seed.kill()
	time.Sleep(50 * time.Microsecond)

	for l := 0; l < leechCount; l++ {
		leechDir := path.Join(rootDir, fmt.Sprintf("leech %d", l))
		err = os.Mkdir(leechDir, 0700)
		if err != nil {
			return
		}
		leech = newTorrentClient(fmt.Sprintf("leech %d", l), 7001+l, torrentFile, leechDir, 0)
		err = leech.start(doneCh)
		if err != nil {
			return
		}
		defer leech.kill()
	}

	timeout := make(chan bool, 1)
	go func() {
		// It takes about 3.5 seconds to complete the test on my computer.
		time.Sleep(50 * time.Second)
		timeout <- true
	}()

	for doneCount := 0; doneCount < leechCount; doneCount++ {
		select {
		case <-timeout:
			err = fmt.Errorf("Timout exceeded")
		case donePeer := <-doneCh:
			if donePeer == tracker || donePeer == seed {
				err = fmt.Errorf("%v finished before all leeches. Should not have.", donePeer)
			}
			err = compareData(seedData, donePeer.dirName)
		}
		if err != nil {
			return
		}
		log.Printf("Done: %d of %d", (doneCount + 1), leechCount)
	}
	if err != nil {
		return
	}
	// All is good. Clean up
	os.RemoveAll(rootDir)

	return
}

func newTracker(name string, addr string, fileDir string, torrentFile string) (p *prog) {
	return newProg(name, fileDir, "tracker", addr, torrentFile)
}

func newTorrentClient(name string, port int, torrentFile string, fileDir string, ratio float64) (p *prog) {
	return newProg(name, fileDir, "client",
		fmt.Sprintf("%v", port),
		fileDir,
		fmt.Sprintf("%v", ratio),
		torrentFile)
}

func createTorrentFile(torrentFileName, root, announcePath string) (err error) {
	var metaInfo *torrent.MetaInfo
	metaInfo, err = torrent.CreateMetaInfoFromFileSystem(nil, root, "127.0.0.1:8080", 0, false)
	if err != nil {
		return
	}
	metaInfo.CreatedBy = "testSwarm"
	var torrentFile *os.File
	torrentFile, err = os.Create(torrentFileName)
	if err != nil {
		return
	}
	defer torrentFile.Close()
	err = metaInfo.Bencode(torrentFile)
	if err != nil {
		return
	}
	return
}

func createDataFile(name string, length int64) (err error) {
	if (length & 3) != 0 {
		return fmt.Errorf("createDataFile only supports length that is a multiple of 4. Not %d", length)
	}
	var file *os.File
	file, err = os.Create(name)
	if err != nil {
		return
	}
	defer file.Close()
	err = file.Truncate(length)
	if err != nil {
		return
	}
	w := bufio.NewWriter(file)
	b := make([]byte, 4)
	for i := int64(0); i < length; i += 4 {
		b[0] = byte(i >> 24)
		b[1] = byte(i >> 16)
		b[2] = byte(i >> 8)
		b[3] = byte(i)
		_, err = w.Write(b)
		if err != nil {
			return
		}
	}
	return
}

func compareData(sourceName, copyDirName string) (err error) {
	_, base := path.Split(sourceName)
	copyName := path.Join(copyDirName, base)
	err = compare(sourceName, copyName)
	return
}

// Compare two files (or directories) for equality.
func compare(aName, bName string) (err error) {
	var aFileInfo, bFileInfo os.FileInfo
	aFileInfo, err = os.Stat(aName)
	if err != nil {
		return
	}
	bFileInfo, err = os.Stat(bName)
	if err != nil {
		return
	}
	aIsDir, bIsDir := aFileInfo.IsDir(), bFileInfo.IsDir()
	if aIsDir != bIsDir {
		return fmt.Errorf("%s.IsDir() == %v != %s.IsDir() == %v",
			aName, aIsDir,
			bName, bIsDir)
	}
	var aFile, bFile *os.File
	aFile, err = os.Open(aName)
	if err != nil {
		return
	}
	defer aFile.Close()
	bFile, err = os.Open(bName)
	if err != nil {
		return
	}
	defer bFile.Close()
	if !aIsDir {
		aSize, bSize := aFileInfo.Size(), bFileInfo.Size()
		if aSize != bSize {
			return fmt.Errorf("%s.Size() == %v != %s.Size() == %v",
				aName, aSize,
				bName, bSize)
		}
		var aBuf, bBuf bytes.Buffer
		bufferSize := int64(128 * 1024)
		for i := int64(0); i < aSize; i += bufferSize {
			toRead := bufferSize
			remainder := aSize - i
			if toRead > remainder {
				toRead = remainder
			}
			_, err = io.CopyN(&aBuf, aFile, toRead)
			if err != nil {
				return
			}
			_, err = io.CopyN(&bBuf, bFile, toRead)
			if err != nil {
				return
			}
			aBytes, bBytes := aBuf.Bytes(), bBuf.Bytes()
			for j := int64(0); j < toRead; j++ {
				a, b := aBytes[j], bBytes[j]
				if a != b {
					err = fmt.Errorf("%s[%d] %d != %d", aName, i+j, a, b)
					return
				}
			}
			aBuf.Reset()
			bBuf.Reset()
		}
	} else {
		var aNames, bNames []string
		aNames, err = aFile.Readdirnames(0)
		if err != nil {
			return
		}
		bNames, err = bFile.Readdirnames(0)
		if err != nil {
			return
		}
		if len(aNames) != len(bName) {
			err = fmt.Errorf("Directories %v and %v don't contain same number of files %d != %d",
				aName, bName, len(aNames), len(bNames))
		}
		for _, name := range aNames {
			err = compare(path.Join(aName, name), path.Join(bName, name))
			if err != nil {
				return
			}
		}
	}
	return
}

// type logWriter

type logWriter string

func (l logWriter) Write(p []byte) (n int, err error) {
	log.Println(l, string(p))
	n = len(p)
	return
}

// A test that's used to run multiple processes. From http://golang.org/src/pkg/os/exec/exec_test.go

func helperCommands(s ...string) *exec.Cmd {
	cs := []string{"-test.run=TestHelperProcess", "--"}
	cs = append(cs, s...)
	cmd := exec.Command(os.Args[0], cs...)
	cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}
	return cmd
}

func TestHelperProcess(*testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
		return
	}

	defer os.Exit(0)

	err := testHelperProcessImp(os.Args)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error %v\n", err)
		os.Exit(3)
	}
}

func testHelperProcessImp(args []string) (err error) {
	for len(args) > 0 {
		if args[0] == "--" {
			args = args[1:]
			break
		}
		args = args[1:]
	}

	if len(args) == 0 {
		err = fmt.Errorf("No commands\n")
		return
	}

	cmd, args := args[0], args[1:]
	switch cmd {
	case "tracker":
		if len(args) < 2 {
			err = fmt.Errorf("tracker expected 2 or more args\n")
			return
		}
		addr, torrentFiles := args[0], args[1:]

		err = StartTracker(addr, torrentFiles)
		if err != nil {
			return
		}
	case "client":
		if len(args) < 4 {
			err = fmt.Errorf("client expected 4 or more args\n")
			return
		}
		portStr, fileDir, seedRatioStr, torrentFiles :=
			args[0], args[1], args[2], args[3:]
		var port uint64
		port, err = strconv.ParseUint(portStr, 10, 16)
		if err != nil {
			return
		}
		var seedRatio float64
		seedRatio, err = strconv.ParseFloat(seedRatioStr, 64)
		torrentFlags := torrent.TorrentFlags{
			Port:               int(port),
			FileDir:            fileDir,
			SeedRatio:          seedRatio,
			FileSystemProvider: torrent.OsFsProvider{},
			InitialCheck:       true,
		}
		err = torrent.RunTorrents(&torrentFlags, torrentFiles)
		if err != nil {
			return
		}
	default:
		err = fmt.Errorf("Unknown command %q\n", cmd)
		return
	}
	return
}
<commit_msg>Update Tracker_test<commit_after>package tracker

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"os"
	"os/exec"
	"path"
	"strconv"
	"testing"
	"time"

	"github.com/jackpal/Taipei-Torrent/torrent"
)

func TestScrapeURL(t *testing.T) {
	tests := []struct{ announce, scrape string }{
		{"", ""},
		{"foo", ""},
		{"x/announce", "x/scrape"},
		{"x/announce?ad#3", "x/scrape?ad#3"},
		{"announce/x", ""},
	}
	for _, test := range tests {
		scrape := ScrapePattern(test.announce)
		if scrape != test.scrape {
			t.Errorf("ScrapeURL(%#v) = %#v. Expected %#v", test.announce, scrape, test.scrape)
		}
	}
}

func TestSwarm1(t *testing.T) {
	testSwarm(t, 1)
}

func TestSwarm10(t *testing.T) {
	testSwarm(t, 10)
}

/* Larger sizes don't work correctly.

func TestSwarm20(t *testing.T) {
	testSwarm(t, 20)
}

func TestSwarm50(t *testing.T) {
	testSwarm(t, 50)
}

func TestSwarm100(t *testing.T) {
	testSwarm(t, 100)
}

*/

func testSwarm(t *testing.T, leechCount int) {
	err := runSwarm(leechCount)
	if err != nil {
		t.Fatal("Error running testSwarm", err)
	}
}

type prog struct {
	instanceName string
	dirName      string
	cmd          *exec.Cmd
}

func (p *prog) start(doneCh chan *prog) (err error) {
	log.Println("starting", p.instanceName)
	out := logWriter(p.instanceName)
	p.cmd.Stdout = &out
	p.cmd.Stderr = &out
	err = p.cmd.Start()
	if err != nil {
		return
	}
	go func() {
		p.cmd.Wait()
		doneCh <- p
	}()
	return
}

func (p *prog) kill() (err error) {
	err = p.cmd.Process.Kill()
	return
}

func newProg(instanceName string, dir string, command string, arg ...string) (p *prog) {
	cmd := helperCommands(append([]string{command}, arg...)...)
	return &prog{instanceName: instanceName, dirName: dir, cmd: cmd}
}

func runSwarm(leechCount int) (err error) {
	var rootDir string
	rootDir, err = ioutil.TempDir("", "swarm")
	if err != nil {
		return
	}
	log.Printf("Temporary directory: %s", rootDir)
	seedDir := path.Join(rootDir, "seed")
	err = os.Mkdir(seedDir, 0700)
	if err != nil {
		return
	}
	seedData := path.Join(seedDir, "data")
	err = createDataFile(seedData, 1024*1024)
	if err != nil {
		return
	}
	torrentFile := path.Join(rootDir, "testSwarm.torrent")
	err = createTorrentFile(torrentFile, seedData, "127.0.0.1:8080/announce")
	if err != nil {
		return
	}

	doneCh := make(chan *prog, 1)

	tracker := newTracker("tracker", ":8080", rootDir, torrentFile)
	err = tracker.start(doneCh)
	if err != nil {
		return
	}
	defer tracker.kill()
	time.Sleep(100 * time.Microsecond)

	var seed, leech *prog
	seed = newTorrentClient("seed", 7000, torrentFile, seedDir, math.Inf(0))
	err = seed.start(doneCh)
	if err != nil {
		return
	}
	defer seed.kill()
	time.Sleep(50 * time.Microsecond)

	for l := 0; l < leechCount; l++ {
		leechDir := path.Join(rootDir, fmt.Sprintf("leech %d", l))
		err = os.Mkdir(leechDir, 0700)
		if err != nil {
			return
		}
		leech = newTorrentClient(fmt.Sprintf("leech %d", l), 7001+l, torrentFile, leechDir, 0)
		err = leech.start(doneCh)
		if err != nil {
			return
		}
		defer leech.kill()
	}

	timeout := make(chan bool, 1)
	go func() {
		// It takes about 3.5 seconds to complete the test on my computer.
		time.Sleep(50 * time.Second)
		timeout <- true
	}()

	for doneCount := 0; doneCount < leechCount; doneCount++ {
		select {
		case <-timeout:
			err = fmt.Errorf("Timout exceeded")
		case donePeer := <-doneCh:
			if donePeer == tracker || donePeer == seed {
				err = fmt.Errorf("%v finished before all leeches. Should not have.", donePeer)
			}
			err = compareData(seedData, donePeer.dirName)
		}
		if err != nil {
			return
		}
		log.Printf("Done: %d of %d", (doneCount + 1), leechCount)
	}
	if err != nil {
		return
	}
	// All is good. Clean up
	os.RemoveAll(rootDir)

	return
}

func newTracker(name string, addr string, fileDir string, torrentFile string) (p *prog) {
	return newProg(name, fileDir, "tracker", addr, torrentFile)
}

func newTorrentClient(name string, port int, torrentFile string, fileDir string, ratio float64) (p *prog) {
	return newProg(name, fileDir, "client",
		fmt.Sprintf("%v", port),
		fileDir,
		fmt.Sprintf("%v", ratio),
		torrentFile)
}

func createTorrentFile(torrentFileName, root, announcePath string) (err error) {
	var metaInfo *torrent.MetaInfo
	metaInfo, err = torrent.CreateMetaInfoFromFileSystem(nil, root, "127.0.0.1:8080", 0, false)
	if err != nil {
		return
	}
	metaInfo.CreatedBy = "testSwarm"
	var torrentFile *os.File
	torrentFile, err = os.Create(torrentFileName)
	if err != nil {
		return
	}
	defer torrentFile.Close()
	err = metaInfo.Bencode(torrentFile)
	if err != nil {
		return
	}
	return
}

func createDataFile(name string, length int64) (err error) {
	if (length & 3) != 0 {
		return fmt.Errorf("createDataFile only supports length that is a multiple of 4. Not %d", length)
	}
	var file *os.File
	file, err = os.Create(name)
	if err != nil {
		return
	}
	defer file.Close()
	err = file.Truncate(length)
	if err != nil {
		return
	}
	w := bufio.NewWriter(file)
	b := make([]byte, 4)
	for i := int64(0); i < length; i += 4 {
		b[0] = byte(i >> 24)
		b[1] = byte(i >> 16)
		b[2] = byte(i >> 8)
		b[3] = byte(i)
		_, err = w.Write(b)
		if err != nil {
			return
		}
	}
	return
}

func compareData(sourceName, copyDirName string) (err error) {
	_, base := path.Split(sourceName)
	copyName := path.Join(copyDirName, base)
	err = compare(sourceName, copyName)
	return
}

// Compare two files (or directories) for equality.
func compare(aName, bName string) (err error) {
	var aFileInfo, bFileInfo os.FileInfo
	aFileInfo, err = os.Stat(aName)
	if err != nil {
		return
	}
	bFileInfo, err = os.Stat(bName)
	if err != nil {
		return
	}
	aIsDir, bIsDir := aFileInfo.IsDir(), bFileInfo.IsDir()
	if aIsDir != bIsDir {
		return fmt.Errorf("%s.IsDir() == %v != %s.IsDir() == %v",
			aName, aIsDir,
			bName, bIsDir)
	}
	var aFile, bFile *os.File
	aFile, err = os.Open(aName)
	if err != nil {
		return
	}
	defer aFile.Close()
	bFile, err = os.Open(bName)
	if err != nil {
		return
	}
	defer bFile.Close()
	if !aIsDir {
		aSize, bSize := aFileInfo.Size(), bFileInfo.Size()
		if aSize != bSize {
			return fmt.Errorf("%s.Size() == %v != %s.Size() == %v",
				aName, aSize,
				bName, bSize)
		}
		var aBuf, bBuf bytes.Buffer
		bufferSize := int64(128 * 1024)
		for i := int64(0); i < aSize; i += bufferSize {
			toRead := bufferSize
			remainder := aSize - i
			if toRead > remainder {
				toRead = remainder
			}
			_, err = io.CopyN(&aBuf, aFile, toRead)
			if err != nil {
				return
			}
			_, err = io.CopyN(&bBuf, bFile, toRead)
			if err != nil {
				return
			}
			aBytes, bBytes := aBuf.Bytes(), bBuf.Bytes()
			for j := int64(0); j < toRead; j++ {
				a, b := aBytes[j], bBytes[j]
				if a != b {
					err = fmt.Errorf("%s[%d] %d != %d", aName, i+j, a, b)
					return
				}
			}
			aBuf.Reset()
			bBuf.Reset()
		}
	} else {
		var aNames, bNames []string
		aNames, err = aFile.Readdirnames(0)
		if err != nil {
			return
		}
		bNames, err = bFile.Readdirnames(0)
		if err != nil {
			return
		}
		if len(aNames) != len(bName) {
			err = fmt.Errorf("Directories %v and %v don't contain same number of files %d != %d",
				aName, bName, len(aNames), len(bNames))
		}
		for _, name := range aNames {
			err = compare(path.Join(aName, name), path.Join(bName, name))
			if err != nil {
				return
			}
		}
	}
	return
}

// type logWriter

type logWriter string

func (l logWriter) Write(p []byte) (n int, err error) {
	log.Println(l, string(p))
	n = len(p)
	return
}

// A test that's used to run multiple processes. From http://golang.org/src/pkg/os/exec/exec_test.go

func helperCommands(s ...string) *exec.Cmd {
	cs := []string{"-test.run=TestHelperProcess", "--"}
	cs = append(cs, s...)
	cmd := exec.Command(os.Args[0], cs...)
	cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}
	return cmd
}

func TestHelperProcess(*testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
		return
	}

	defer os.Exit(0)

	err := testHelperProcessImp(os.Args)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error %v\n", err)
		os.Exit(3)
	}
}

func testHelperProcessImp(args []string) (err error) {
	for len(args) > 0 {
		if args[0] == "--" {
			args = args[1:]
			break
		}
		args = args[1:]
	}

	if len(args) == 0 {
		err = fmt.Errorf("No commands\n")
		return
	}

	cmd, args := args[0], args[1:]
	switch cmd {
	case "tracker":
		if len(args) < 2 {
			err = fmt.Errorf("tracker expected 2 or more args\n")
			return
		}
		addr, torrentFiles := args[0], args[1:]

		err = StartTracker(addr, torrentFiles)
		if err != nil {
			return
		}
	case "client":
		if len(args) < 4 {
			err = fmt.Errorf("client expected 4 or more args\n")
			return
		}
		portStr, fileDir, seedRatioStr, torrentFiles :=
			args[0], args[1], args[2], args[3:]
		var port uint64
		port, err = strconv.ParseUint(portStr, 10, 16)
		if err != nil {
			return
		}
		var seedRatio float64
		seedRatio, err = strconv.ParseFloat(seedRatioStr, 64)
		torrentFlags := torrent.TorrentFlags{
			Port:               int(port),
			FileDir:            fileDir,
			SeedRatio:          seedRatio,
			FileSystemProvider: torrent.OsFsProvider{},
			InitialCheck:       true,
			MaxActive:          1,
			ExecOnSeeding:      "",
			Cacher:				torrent.NewRamCacheProvider(1),
		}
		err = torrent.RunTorrents(&torrentFlags, torrentFiles)
		if err != nil {
			return
		}
	default:
		err = fmt.Errorf("Unknown command %q\n", cmd)
		return
	}
	return
}
<|endoftext|>


<commit_before>package transcode

// mp3Codec contains the codec describing MP3
const mp3Codec = "MP3"

// mp3Ext contains the extension describing MP3
const mp3Ext = "mp3"

// mp3FFmpegCodec contains the ffmpeg codec describing MP3
const mp3FFmpegCodec = "libmp3lame"

// mp3MIMEType contains the MIME type describing MP3
const mp3MIMEType = "audio/mpeg"

// MP3CBROptions represents the options for a MP3 CBR transcoder
type MP3CBROptions struct {
	quality string
}

// Codec returns the codec used
func (m MP3CBROptions) Codec() string {
	return mp3Codec
}

// Ext returns the file extension used
func (m MP3CBROptions) Ext() string {
	return mp3Ext
}

// FFmpegFlags returns the flag used by ffmpeg to signify this encoding
func (m MP3CBROptions) FFmpegFlags() string {
	return "-b:a"
}

// FFmpegCodec returns the codec used by ffmpeg
func (m MP3CBROptions) FFmpegCodec() string {
	return mp3FFmpegCodec
}

// MIMEType returns the MIME type of this item
func (m MP3CBROptions) MIMEType() string {
	return mp3MIMEType
}

// Quality returns the quality used
func (m MP3CBROptions) Quality() string {
	return m.quality + "kbps"
}

// FFmpegQuality returns the quality flag used by ffmpeg
func (m MP3CBROptions) FFmpegQuality() string {
	return m.quality + "k"
}

// MP3VBROptions represents the options for a MP3 VBR transcoder
type MP3VBROptions struct {
	quality string
}

// Codec returns the codec used
func (m MP3VBROptions) Codec() string {
	return mp3Codec
}

// Ext returns the file extension used
func (m MP3VBROptions) Ext() string {
	return mp3Ext
}

// FFmpegCodec returns the codec used by ffmpeg
func (m MP3VBROptions) FFmpegCodec() string {
	return mp3FFmpegCodec
}

// FFmpegFlags returns the flag used by ffmpeg to signify this encoding
func (m MP3VBROptions) FFmpegFlags() string {
	return "-qscale:a"
}

// MIMEType returns the MIME type of this item
func (m MP3VBROptions) MIMEType() string {
	return mp3MIMEType
}

// Quality returns the quality used
func (m MP3VBROptions) Quality() string {
	return m.quality
}

// FFmpegQuality returns the quality flag used by ffmpeg
func (m MP3VBROptions) FFmpegQuality() string {
	// Return the number after 'V'
	return string(m.quality[1:])
}
<commit_msg>Use newer ffmpeg libmp3lame options<commit_after>package transcode

// mp3Codec contains the codec describing MP3
const mp3Codec = "MP3"

// mp3Ext contains the extension describing MP3
const mp3Ext = "mp3"

// mp3FFmpegCodec contains the ffmpeg codec describing MP3
const mp3FFmpegCodec = "libmp3lame"

// mp3MIMEType contains the MIME type describing MP3
const mp3MIMEType = "audio/mpeg"

// MP3CBROptions represents the options for a MP3 CBR transcoder
type MP3CBROptions struct {
	quality string
}

// Codec returns the codec used
func (m MP3CBROptions) Codec() string {
	return mp3Codec
}

// Ext returns the file extension used
func (m MP3CBROptions) Ext() string {
	return mp3Ext
}

// FFmpegFlags returns the flag used by ffmpeg to signify this encoding
func (m MP3CBROptions) FFmpegFlags() string {
	return "-ab"
}

// FFmpegCodec returns the codec used by ffmpeg
func (m MP3CBROptions) FFmpegCodec() string {
	return mp3FFmpegCodec
}

// MIMEType returns the MIME type of this item
func (m MP3CBROptions) MIMEType() string {
	return mp3MIMEType
}

// Quality returns the quality used
func (m MP3CBROptions) Quality() string {
	return m.quality + "kbps"
}

// FFmpegQuality returns the quality flag used by ffmpeg
func (m MP3CBROptions) FFmpegQuality() string {
	return m.quality + "k"
}

// MP3VBROptions represents the options for a MP3 VBR transcoder
type MP3VBROptions struct {
	quality string
}

// Codec returns the codec used
func (m MP3VBROptions) Codec() string {
	return mp3Codec
}

// Ext returns the file extension used
func (m MP3VBROptions) Ext() string {
	return mp3Ext
}

// FFmpegCodec returns the codec used by ffmpeg
func (m MP3VBROptions) FFmpegCodec() string {
	return mp3FFmpegCodec
}

// FFmpegFlags returns the flag used by ffmpeg to signify this encoding
func (m MP3VBROptions) FFmpegFlags() string {
	return "-aq"
}

// MIMEType returns the MIME type of this item
func (m MP3VBROptions) MIMEType() string {
	return mp3MIMEType
}

// Quality returns the quality used
func (m MP3VBROptions) Quality() string {
	return m.quality
}

// FFmpegQuality returns the quality flag used by ffmpeg
func (m MP3VBROptions) FFmpegQuality() string {
	// Return the number after 'V'
	return string(m.quality[1:])
}
<|endoftext|>


<commit_before>package main

// import "bufio"
import "fmt"
import "net"

func checkError(e error) {
  if e != nil {
    panic(e)
  }
}

func generateDaytime() string {
  return "today"
}

func main() {
  l, err := net.Listen("tcp", ":13")
  checkError(err)
  defer l.Close()
  fmt.Println("port connected")

  fmt.Println("server started")

  for {
    c, err := l.Accept()
    checkError(err)

    d := generateDaytime()
    // write the daytime to the connection
    c.Write([]byte(d))
    c.Close()
  }
}
<commit_msg>generate a daytime value using current time and formatted to RFC3339 format<commit_after>package main

import "fmt"
import "net"
import "time"

func checkError(e error) {
  if e != nil {
    panic(e)
  }
}

func generateDaytime() string {
  t := time.Now().UTC()
  return t.Format(time.RFC3339)
}

func main() {
  l, err := net.Listen("tcp", ":13")
  checkError(err)
  defer l.Close()
  fmt.Println("port connected")

  fmt.Println("server started")

  for {
    c, err := l.Accept()
    checkError(err)

    d := generateDaytime()
    // write the daytime to the connection
    c.Write([]byte(d))
    c.Close()
  }
}
<|endoftext|>


<commit_before>// Package protocol defines the communication between the components
// of the Kite infrastructure. It defines some constants and structures
// designed to be sent between those components.
//
// The following table shows the communication types:
//
// +-----------------+---------+----------+----------------+
// |                 | Library | Protocol | Authentication |
// +-----------------+---------+----------+----------------+
// | Browser-Kontrol | moh     | JSON     | SessionID      |
// | Kite-Kontrol    | moh     | JSON     | Koding Key     |
// | Browser-Kite    | Go-RPC  | dnode    | token          |
// | Kite-Kite       | Go-RPC  | gob      | token          |
// +-----------------+---------+----------+----------------+
//
package protocol

import (
	"koding/tools/dnode"
	"time"
)

const HEARTBEAT_INTERVAL = time.Millisecond * 1000
const HEARTBEAT_DELAY = time.Millisecond * 1000

// Kite's HTTP server runs a RPC server here
const WEBSOCKET_PATH = "/sock"

// Kite is the base struct containing the public fields.
// It is usually embeded in other structs, including the db model.
type Kite struct {
	// Short name identifying the type of the kite. Example: fs, terminal...
	Name string `bson:"name" json:"name"`

	// Owner of the Kite
	Username string `bson:"username" json:"username"`

	// Every Kite instance has different identifier.
	// If a kite is restarted, it's id will change.
	// This is generated on the Kite.
	ID string `bson:"_id" json:"id"`

	// This is used temporary to distinguish kites that are used for Koding
	// client-side. An example is to use it with value "vm"
	Kind string `bson:"kind" json:"kind"`

	Version  string `bson:"version" json:"version"`
	Hostname string `bson:"hostname" json:"hostname"`
	PublicIP string `bson:"publicIP" json:"publicIP"`
	Port     string `bson:"port" json:"port"`
}

func (k *Kite) Addr() string {
	return k.PublicIP + ":" + k.Port
}

// KiteRequest is a structure that is used in Kite-to-Kite communication.
type KiteRequest struct {
	Kite   `json:"kite"`
	Token  string      `json:"token"`
	Method string      `json:"method"`
	Args   interface{} `json:"args"`
}

// KiteDnodeRequest is the data structure sent when a request is made
// from a client to the RPC server of a Kite.
type KiteDnodeRequest struct {
	Kite
	Method string
	Args   *dnode.Partial // Must include a token for authentication
}

// KiteToKontrolRequest is a structure of message sent
// from Kites and Kontrol.
type KiteToKontrolRequest struct {
	Kite      Kite                   `json:"kite"`
	KodingKey string                 `json:"kodingKey"`
	Method    Method                 `json:"method"`
	Args      map[string]interface{} `json:"args"`
}

// BrowserToKontrolRequest is a structure of message sent
// from Browser to Kontrol.
type BrowserToKontrolRequest struct {
	Username  string `json:"username"`
	Kitename  string `json:"kitename"`
	SessionID string `json:"sessionID"`
}

type Method string

const (
	Pong         Method = "PONG"
	RegisterKite Method = "REGISTER_KITE"
	GetKites     Method = "GET_KITES"
)

// RegisterResponse is a response to Register request from Kite to Kontrol.
type RegisterResponse struct {
	Result RegisterResult `json:"result"`

	// Username is sent in response because the kite does not know
	// it's own user's name on start.
	Username string `json:"username"`
}

type RegisterResult string

const (
	AllowKite  RegisterResult = "ALLOW"
	RejectKite RegisterResult = "REJECT"
)

type GetKitesResponse []KiteWithToken

type KiteWithToken struct {
	Kite  `json:"kite"`
	Token string `json:"token"`
}

// KontrolMessage is a structure that is published from Kontrol to Kite
// to notify some events.
type KontrolMessage struct {
	Type MessageType            `json:"type"`
	Args map[string]interface{} `json:"args"`
}

type MessageType string

const (
	KiteRegistered   MessageType = "KITE_REGISTERED"
	KiteDisconnected MessageType = "KITE_DISCONNECTED"
	KiteUpdated      MessageType = "KITE_UPDATED"
	Ping             MessageType = "PING"
)

type Options struct {
	Username     string `json:"username"`
	Kitename     string `json:"kitename"`
	LocalIP      string `json:"localIP"`
	PublicIP     string `json:"publicIP"`
	Port         string `json:"port"`
	Version      string `json:"version"`
	Kind         string `json:"kind"`
	KontrolAddr  string `json:"kontrolAddr"`
	Dependencies string `json:"dependencies"`
}
<commit_msg>use net.JoinHostPort<commit_after>// Package protocol defines the communication between the components
// of the Kite infrastructure. It defines some constants and structures
// designed to be sent between those components.
//
// The following table shows the communication types:
//
// +-----------------+---------+----------+----------------+
// |                 | Library | Protocol | Authentication |
// +-----------------+---------+----------+----------------+
// | Browser-Kontrol | moh     | JSON     | SessionID      |
// | Kite-Kontrol    | moh     | JSON     | Koding Key     |
// | Browser-Kite    | Go-RPC  | dnode    | token          |
// | Kite-Kite       | Go-RPC  | gob      | token          |
// +-----------------+---------+----------+----------------+
//
package protocol

import (
	"koding/tools/dnode"
	"net"
	"time"
)

const HEARTBEAT_INTERVAL = time.Millisecond * 1000
const HEARTBEAT_DELAY = time.Millisecond * 1000

// Kite's HTTP server runs a RPC server here
const WEBSOCKET_PATH = "/sock"

// Kite is the base struct containing the public fields.
// It is usually embeded in other structs, including the db model.
type Kite struct {
	// Short name identifying the type of the kite. Example: fs, terminal...
	Name string `bson:"name" json:"name"`

	// Owner of the Kite
	Username string `bson:"username" json:"username"`

	// Every Kite instance has different identifier.
	// If a kite is restarted, it's id will change.
	// This is generated on the Kite.
	ID string `bson:"_id" json:"id"`

	// This is used temporary to distinguish kites that are used for Koding
	// client-side. An example is to use it with value "vm"
	Kind string `bson:"kind" json:"kind"`

	Version  string `bson:"version" json:"version"`
	Hostname string `bson:"hostname" json:"hostname"`
	PublicIP string `bson:"publicIP" json:"publicIP"`
	Port     string `bson:"port" json:"port"`
}

func (k *Kite) Addr() string {
	return net.JoinHostPort(k.PublicIP, k.Port)
}

// KiteRequest is a structure that is used in Kite-to-Kite communication.
type KiteRequest struct {
	Kite   `json:"kite"`
	Token  string      `json:"token"`
	Method string      `json:"method"`
	Args   interface{} `json:"args"`
}

// KiteDnodeRequest is the data structure sent when a request is made
// from a client to the RPC server of a Kite.
type KiteDnodeRequest struct {
	Kite
	Method string
	Args   *dnode.Partial // Must include a token for authentication
}

// KiteToKontrolRequest is a structure of message sent
// from Kites and Kontrol.
type KiteToKontrolRequest struct {
	Kite      Kite                   `json:"kite"`
	KodingKey string                 `json:"kodingKey"`
	Method    Method                 `json:"method"`
	Args      map[string]interface{} `json:"args"`
}

// BrowserToKontrolRequest is a structure of message sent
// from Browser to Kontrol.
type BrowserToKontrolRequest struct {
	Username  string `json:"username"`
	Kitename  string `json:"kitename"`
	SessionID string `json:"sessionID"`
}

type Method string

const (
	Pong         Method = "PONG"
	RegisterKite Method = "REGISTER_KITE"
	GetKites     Method = "GET_KITES"
)

// RegisterResponse is a response to Register request from Kite to Kontrol.
type RegisterResponse struct {
	Result RegisterResult `json:"result"`

	// Username is sent in response because the kite does not know
	// it's own user's name on start.
	Username string `json:"username"`
}

type RegisterResult string

const (
	AllowKite  RegisterResult = "ALLOW"
	RejectKite RegisterResult = "REJECT"
)

type GetKitesResponse []KiteWithToken

type KiteWithToken struct {
	Kite  `json:"kite"`
	Token string `json:"token"`
}

// KontrolMessage is a structure that is published from Kontrol to Kite
// to notify some events.
type KontrolMessage struct {
	Type MessageType            `json:"type"`
	Args map[string]interface{} `json:"args"`
}

type MessageType string

const (
	KiteRegistered   MessageType = "KITE_REGISTERED"
	KiteDisconnected MessageType = "KITE_DISCONNECTED"
	KiteUpdated      MessageType = "KITE_UPDATED"
	Ping             MessageType = "PING"
)

type Options struct {
	Username     string `json:"username"`
	Kitename     string `json:"kitename"`
	LocalIP      string `json:"localIP"`
	PublicIP     string `json:"publicIP"`
	Port         string `json:"port"`
	Version      string `json:"version"`
	Kind         string `json:"kind"`
	KontrolAddr  string `json:"kontrolAddr"`
	Dependencies string `json:"dependencies"`
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"corefont"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"pdfjet"
	"pdfjet/src/color"
	"pdfjet/src/compliance"
	"pdfjet/src/imagetype"
	"strings"
	"time"
)

// Example50 shows how to fill in an existing PDF form.
func Example50(fileName string) {
	file, err := os.Create("Example_50.pdf")
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()
	w := bufio.NewWriter(file)

	buf, err := ioutil.ReadFile("data/testPDFs/" + fileName)
	if err != nil {
		log.Fatal(err)
	}

	pdf := pdfjet.NewPDF(w, compliance.PDF15)

	objects := pdf.Read(buf)

	file1, err := os.Open("fonts/Droid/DroidSans.ttf.stream")
	if err != nil {
		log.Fatal(err)
	}
	defer file1.Close()
	reader := bufio.NewReader(file1)
	font1 := pdfjet.NewFontStream2(&objects, reader)
	font1.SetSize(12.0)

	file2, err := os.Open("fonts/Droid/DroidSans-Bold.ttf.stream")
	if err != nil {
		log.Fatal(err)
	}
	defer file2.Close()
	reader = bufio.NewReader(file2)
	font2 := pdfjet.NewFontStream2(&objects, reader)
	font2.SetSize(12.0)

	file3, err := os.Open("images/qrcode.png")
	if err != nil {
		log.Fatal(err)
	}
	defer file3.Close()
	reader = bufio.NewReader(file3)
	image1 := pdfjet.NewImage2(&objects, reader, imagetype.PNG)
	image1.SetLocation(495.0, 65.0)
	image1.ScaleBy(0.40)

	pages := pdf.GetPageObjects(objects)
	page := pdfjet.NewPageFromObject(pdf, pages[0])
	// page.InvertYAxis()

	page.AddImageResource(image1, &objects)
	page.AddFontResource(font1, &objects)
	page.AddFontResource(font2, &objects)
	font3 := page.AddCoreFontResource(corefont.Helvetica(), &objects)
	font3.SetSize(12.0)

	image1.DrawOn(page)

	x := float32(23.0)
	y := float32(185.0)
	dx := float32(15.0)
	dy := float32(24.0)

	page.SetBrushColor(color.Blue)

	// First Name and Initial
	page.DrawString(font2, nil, "Иван", x, y)

	// Last Name
	page.DrawString(font3, nil, "Jones", x+258.0, y)

	// Social Insurance Number
	page.DrawArrayOfCharacters(font1, stripSpacesAndDashes("243-590-129"), x+437.0, y, dx)

	// Last Name at Birth
	y += dy
	page.DrawString(font1, nil, "Culverton", x, y)

	// Mailing Address
	y += dy
	page.DrawString(font1, nil, "10 Elm Street", x, y)

	// City
	y += dy
	page.DrawString(font1, nil, "Toronto", x, y)

	// Province or Territory
	page.DrawString(font1, nil, "Ontario", x+365.0, y)

	// Postal Code
	page.DrawArrayOfCharacters(font1, stripSpacesAndDashes("L7B 2E9"), x+482.0, y, dx)

	// Home Address
	y += dy
	page.DrawString(font1, nil, "10 Oak Road", x, y)

	// City
	y += dy
	page.DrawString(font1, nil, "Toronto", x, y)

	// Previous Province or Territory
	page.DrawString(font1, nil, "Ontario", x+365.0, y)

	// Postal Code
	page.DrawArrayOfCharacters(font1, stripSpacesAndDashes("L7B 2E9"), x+482.0, y, dx)

	// Home telephone number
	page.DrawString(font1, nil, "905-222-3333", x, y+dy)
	// Work telephone number
	y += dy
	page.DrawString(font1, nil, "416-567-9903", x+279.0, y)

	// Previous province or territory
	y += dy
	page.DrawString(font1, nil, "British Columbia", x+452.0, y)

	// Move date from previous province or territory
	y += dy
	page.DrawArrayOfCharacters(font1, stripSpacesAndDashes("2016-04-12"), x+452.0, y, dx)

	// Date new marital status began
	page.DrawArrayOfCharacters(font1, stripSpacesAndDashes("2014-11-02"), x+452.0, 467.0, dx)

	// First name of spouse
	y = 521.0
	page.DrawString(font1, nil, "Melanie", x, y)
	// Last name of spouse
	page.DrawString(font1, nil, "Jones", x+258.0, y)

	// Social Insurance number of spouse
	page.DrawArrayOfCharacters(font1, stripSpacesAndDashes("192-760-427"), x+437.0, y, dx)

	// Spouse or common-law partner's address
	page.DrawString(font1, nil, "12 Smithfield Drive", x, 554.0)

	// Signature Date
	page.DrawString(font1, nil, "2016-08-07", x+475.0, 615.0)

	// Signature Date of spouse
	page.DrawString(font1, nil, "2016-08-07", x+475.0, 651.0)

	// Female Checkbox 1
	// xMarkCheckBox(page, 477.5, 197.5, 7.0)

	// Male Checkbox 1
	xMarkCheckBox(page, 534.5, 197.5, 7.0)

	// Married
	xMarkCheckBox(page, 34.5, 424.0, 7.0)

	// Living common-law
	// xMarkCheckBox(page, 121.5, 424.0, 7.0)

	// Widowed
	// xMarkCheckBox(page, 235.5, 424.0, 7.0)

	// Divorced
	// xMarkCheckBox(page, 325.5, 424.0, 7.0)

	// Separated
	// xMarkCheckBox(page, 415.5, 424.0, 7.0)

	// Single
	// xMarkCheckBox(page, 505.5, 424.0, 7.0)

	// Female Checkbox 2
	xMarkCheckBox(page, 478.5, 536.5, 7.0)

	// Male Checkbox 2
	// xMarkCheckBox(page, 535.5, 536.5, 7.0)

	page.Complete(&objects)

	pdf.AddObjects(&objects)

	pdf.Complete()
}

func xMarkCheckBox(page *pdfjet.Page, x, y, diagonal float32) {
	page.SetPenColor(color.Blue)
	page.SetPenWidth(diagonal / 5.0)
	page.MoveTo(x, y)
	page.LineTo(x+diagonal, y+diagonal)
	page.MoveTo(x, y+diagonal)
	page.LineTo(x+diagonal, y)
	page.StrokePath()
}

func stripSpacesAndDashes(str string) string {
	var buf strings.Builder
	runes := []rune(str)
	for _, ch := range runes {
		if ch != ' ' && ch != '-' {
			buf.WriteRune(ch)
		}
	}
	return buf.String()
}

func main() {
	start := time.Now()
	// Example50("rc65-16e.pdf")
	Example50("PDF32000_2008.pdf")
	elapsed := time.Since(start).String()
	fmt.Printf("Example_50 => %s\n", elapsed[:strings.Index(elapsed, ".")])
}
<commit_msg>Added to the project.<commit_after>package main

import (
	"bufio"
	"corefont"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"pdfjet"
	"pdfjet/src/color"
	"pdfjet/src/compliance"
	"pdfjet/src/imagetype"
	"strings"
	"time"
)

// Example50 shows how to fill in an existing PDF form.
func Example50(fileName string) {
	file, err := os.Create("Example_50.pdf")
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()
	w := bufio.NewWriter(file)

	buf, err := ioutil.ReadFile("data/testPDFs/" + fileName)
	if err != nil {
		log.Fatal(err)
	}

	pdf := pdfjet.NewPDF(w, compliance.PDF15)

	objects := pdf.Read(buf)

	file1, err := os.Open("fonts/Droid/DroidSans.ttf.stream")
	if err != nil {
		log.Fatal(err)
	}
	defer file1.Close()
	reader := bufio.NewReader(file1)
	font1 := pdfjet.NewFontStream2(&objects, reader)
	font1.SetSize(12.0)

	file2, err := os.Open("fonts/Droid/DroidSans-Bold.ttf.stream")
	if err != nil {
		log.Fatal(err)
	}
	defer file2.Close()
	reader = bufio.NewReader(file2)
	font2 := pdfjet.NewFontStream2(&objects, reader)
	font2.SetSize(12.0)

	file3, err := os.Open("images/qrcode.png")
	if err != nil {
		log.Fatal(err)
	}
	defer file3.Close()
	reader = bufio.NewReader(file3)
	image1 := pdfjet.NewImage2(&objects, reader, imagetype.PNG)
	image1.SetLocation(495.0, 65.0)
	image1.ScaleBy(0.40)

	pages := pdf.GetPageObjects(objects)
	page := pdfjet.NewPageFromObject(pdf, pages[0])
	// page.InvertYAxis()

	page.AddImageResource(image1, &objects)
	page.AddFontResource(font1, &objects)
	page.AddFontResource(font2, &objects)
	font3 := page.AddCoreFontResource(corefont.Helvetica(), &objects)
	font3.SetSize(12.0)

	image1.DrawOn(page)

	x := float32(23.0)
	y := float32(185.0)
	dx := float32(15.0)
	dy := float32(24.0)

	page.SetBrushColor(color.Blue)

	// First Name and Initial
	page.DrawString(font2, nil, "Иван", x, y)

	// Last Name
	page.DrawString(font3, nil, "Jones", x+258.0, y)

	// Social Insurance Number
	page.DrawArrayOfCharacters(font1, stripSpacesAndDashes("243-590-129"), x+437.0, y, dx)

	// Last Name at Birth
	y += dy
	page.DrawString(font1, nil, "Culverton", x, y)

	// Mailing Address
	y += dy
	page.DrawString(font1, nil, "10 Elm Street", x, y)

	// City
	y += dy
	page.DrawString(font1, nil, "Toronto", x, y)

	// Province or Territory
	page.DrawString(font1, nil, "Ontario", x+365.0, y)

	// Postal Code
	page.DrawArrayOfCharacters(font1, stripSpacesAndDashes("L7B 2E9"), x+482.0, y, dx)

	// Home Address
	y += dy
	page.DrawString(font1, nil, "10 Oak Road", x, y)

	// City
	y += dy
	page.DrawString(font1, nil, "Toronto", x, y)

	// Previous Province or Territory
	page.DrawString(font1, nil, "Ontario", x+365.0, y)

	// Postal Code
	page.DrawArrayOfCharacters(font1, stripSpacesAndDashes("L7B 2E9"), x+482.0, y, dx)

	// Home telephone number
	page.DrawString(font1, nil, "905-222-3333", x, y+dy)
	// Work telephone number
	y += dy
	page.DrawString(font1, nil, "416-567-9903", x+279.0, y)

	// Previous province or territory
	y += dy
	page.DrawString(font1, nil, "British Columbia", x+452.0, y)

	// Move date from previous province or territory
	y += dy
	page.DrawArrayOfCharacters(font1, stripSpacesAndDashes("2016-04-12"), x+452.0, y, dx)

	// Date new marital status began
	page.DrawArrayOfCharacters(font1, stripSpacesAndDashes("2014-11-02"), x+452.0, 467.0, dx)

	// First name of spouse
	y = 521.0
	page.DrawString(font1, nil, "Melanie", x, y)
	// Last name of spouse
	page.DrawString(font1, nil, "Jones", x+258.0, y)

	// Social Insurance number of spouse
	page.DrawArrayOfCharacters(font1, stripSpacesAndDashes("192-760-427"), x+437.0, y, dx)

	// Spouse or common-law partner's address
	page.DrawString(font1, nil, "12 Smithfield Drive", x, 554.0)

	// Signature Date
	page.DrawString(font1, nil, "2016-08-07", x+475.0, 615.0)

	// Signature Date of spouse
	page.DrawString(font1, nil, "2016-08-07", x+475.0, 651.0)

	// Female Checkbox 1
	// xMarkCheckBox(page, 477.5, 197.5, 7.0)

	// Male Checkbox 1
	xMarkCheckBox(page, 534.5, 197.5, 7.0)

	// Married
	xMarkCheckBox(page, 34.5, 424.0, 7.0)

	// Living common-law
	// xMarkCheckBox(page, 121.5, 424.0, 7.0)

	// Widowed
	// xMarkCheckBox(page, 235.5, 424.0, 7.0)

	// Divorced
	// xMarkCheckBox(page, 325.5, 424.0, 7.0)

	// Separated
	// xMarkCheckBox(page, 415.5, 424.0, 7.0)

	// Single
	// xMarkCheckBox(page, 505.5, 424.0, 7.0)

	// Female Checkbox 2
	xMarkCheckBox(page, 478.5, 536.5, 7.0)

	// Male Checkbox 2
	// xMarkCheckBox(page, 535.5, 536.5, 7.0)

	page.Complete(&objects)

	pdf.AddObjects(&objects)

	pdf.Complete()
}

func xMarkCheckBox(page *pdfjet.Page, x, y, diagonal float32) {
	page.SetPenColor(color.Blue)
	page.SetPenWidth(diagonal / 5.0)
	page.MoveTo(x, y)
	page.LineTo(x+diagonal, y+diagonal)
	page.MoveTo(x, y+diagonal)
	page.LineTo(x+diagonal, y)
	page.StrokePath()
}

func stripSpacesAndDashes(str string) string {
	var buf strings.Builder
	runes := []rune(str)
	for _, ch := range runes {
		if ch != ' ' && ch != '-' {
			buf.WriteRune(ch)
		}
	}
	return buf.String()
}

func main() {
	start := time.Now()
	Example50("rc65-16e.pdf")
	elapsed := time.Since(start).String()
	fmt.Printf("Example_50 => %s\n", elapsed[:strings.Index(elapsed, ".")])
}
<|endoftext|>


<commit_before>package netlog

import (
	"errors"
	"fmt"
	"io"
	"net/url"
	"os"
	"time"
)

type Logger interface {
	Debug(format string, v ...interface{})
	Info(format string, v ...interface{})
	Warning(format string, v ...interface{})
	Err(format string, v ...interface{})
	Crit(format string, v ...interface{})

	// SetDebug can enable/disable debug mode.
	SetDebug(bool)
}

const (
	logHeaderDebug   = "debug: "
	logHeaderInfo    = "info:  "
	logHeaderWarning = "warn:  "
	logHeaderErr     = "error: "
	logHeaderCrit    = "crit:  "
)

type Facility int

const (
	LogSystem      = facilitySystem
	LogApplication = facilityApplication
	LogService     = facilityService
	LogSecurity    = facilitySecurity
)

func parseFacility(s string) (Facility, error) {
	switch s {
	case "sys", "system":
		return LogSystem, nil
	case "app", "application":
		return LogApplication, nil
	case "service":
		return LogService, nil
	case "security":
		return LogSecurity, nil
	default:
		return LogSystem, errors.New("unknown scheme")
	}
}

var (
	DefaultLogger Logger = &consoleLogger{w: os.Stderr}
)

const stampFormat = "2006/01/02 15:04:05.000000"

type consoleLogger struct {
	w io.Writer
	d bool
}

func (c consoleLogger) stamp(t time.Time) {
	fmt.Fprintf(c.w, "%s ", t.Format(stampFormat))
}

func (c consoleLogger) print(format string, v ...interface{}) {
	c.stamp(time.Now())
	fmt.Fprintf(c.w, format, v...)
	if format[len(format)-1] != '\n' {
		fmt.Fprintf(c.w, "\n")
	}
}

func (c consoleLogger) Debug(format string, v ...interface{}) {
	if c.d {
		c.print(logHeaderDebug+format, v...)
	}
}

func (c consoleLogger) Info(format string, v ...interface{}) {
	c.print(logHeaderInfo+format, v...)
}

func (c consoleLogger) Warning(format string, v ...interface{}) {
	c.print(logHeaderWarning+format, v...)
}

func (c consoleLogger) Err(format string, v ...interface{}) {
	c.print(logHeaderErr+format, v...)
}

func (c consoleLogger) Crit(format string, v ...interface{}) {
	c.print(logHeaderCrit+format, v...)
	os.Exit(2)
}

// SetDebug can enable/disable debug mode.
func (c *consoleLogger) SetDebug(status bool) {
	c.d = status
}

// SetOutputURL is to set output for netlog.
func SetOutputURL(s string, debug ...bool) (err error) {
	var u *url.URL
	if u, err = url.Parse(s); err != nil {
		return
	}

	var isDebug bool
	if len(debug) > 0 {
		isDebug = debug[0]
	}

	q := u.Query()
	facility := LogApplication
	t := q.Get("facility")
	if t != "" {
		if facility, err = parseFacility(t); err != nil {
			return
		}
	}
	tag := q.Get("tag")

	switch u.Scheme {
	case "file":
		// file:///var/log/xxx.log
		var fp *os.File
		if fp, err = os.OpenFile(u.Path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666); err != nil {
			return
		}
		DefaultLogger = &consoleLogger{w: fp, d: isDebug}
		return nil
	case "net":
		// net:///?facility=x&tag=x
		DefaultLogger, err = NewLogger(facility, tag, isDebug)
		return
	case "tcp":
		// tcp://localhost:port/?facility=x&tag=x
		DefaultLogger, err = NewLogger(facility, tag, isDebug, u.Host)
		return
	case "tcp4", "tcp6":
		return errors.New("not implemented")
	default:
		// ??
		return errors.New("unsupported scheme")
	}
}

func Debug(format string, v ...interface{}) {
	DefaultLogger.Debug(format, v...)
}

func Info(format string, v ...interface{}) {
	DefaultLogger.Info(format, v...)
}

func Warning(format string, v ...interface{}) {
	DefaultLogger.Warning(format, v...)
}

func Err(format string, v ...interface{}) {
	DefaultLogger.Err(format, v...)
}

func Crit(format string, v ...interface{}) {
	DefaultLogger.Crit(format, v...)
}

// SetDebug can enable/disable debug mode.
func SetDebug(state bool) {
	DefaultLogger.SetDebug(state)
}
<commit_msg>added comments to the public methods<commit_after>package netlog

import (
	"errors"
	"fmt"
	"io"
	"net/url"
	"os"
	"time"
)

type Logger interface {
	Debug(format string, v ...interface{})
	Info(format string, v ...interface{})
	Warning(format string, v ...interface{})
	Err(format string, v ...interface{})
	Crit(format string, v ...interface{})

	// SetDebug can enable/disable debug mode.
	SetDebug(bool)
}

const (
	logHeaderDebug   = "debug: "
	logHeaderInfo    = "info:  "
	logHeaderWarning = "warn:  "
	logHeaderErr     = "error: "
	logHeaderCrit    = "crit:  "
)

type Facility int

const (
	LogSystem      = facilitySystem
	LogApplication = facilityApplication
	LogService     = facilityService
	LogSecurity    = facilitySecurity
)

func parseFacility(s string) (Facility, error) {
	switch s {
	case "sys", "system":
		return LogSystem, nil
	case "app", "application":
		return LogApplication, nil
	case "service":
		return LogService, nil
	case "security":
		return LogSecurity, nil
	default:
		return LogSystem, errors.New("unknown scheme")
	}
}

var (
	DefaultLogger Logger = &consoleLogger{w: os.Stderr}
)

const stampFormat = "2006/01/02 15:04:05.000000"

type consoleLogger struct {
	w io.Writer
	d bool
}

func (c consoleLogger) stamp(t time.Time) {
	fmt.Fprintf(c.w, "%s ", t.Format(stampFormat))
}

func (c consoleLogger) print(format string, v ...interface{}) {
	c.stamp(time.Now())
	fmt.Fprintf(c.w, format, v...)
	if format[len(format)-1] != '\n' {
		fmt.Fprintf(c.w, "\n")
	}
}

func (c consoleLogger) Debug(format string, v ...interface{}) {
	if c.d {
		c.print(logHeaderDebug+format, v...)
	}
}

func (c consoleLogger) Info(format string, v ...interface{}) {
	c.print(logHeaderInfo+format, v...)
}

func (c consoleLogger) Warning(format string, v ...interface{}) {
	c.print(logHeaderWarning+format, v...)
}

func (c consoleLogger) Err(format string, v ...interface{}) {
	c.print(logHeaderErr+format, v...)
}

func (c consoleLogger) Crit(format string, v ...interface{}) {
	c.print(logHeaderCrit+format, v...)
	os.Exit(2)
}

// SetDebug can enable/disable debug mode.
func (c *consoleLogger) SetDebug(status bool) {
	c.d = status
}

// SetOutputURL is to set output for netlog.
func SetOutputURL(s string, debug ...bool) (err error) {
	var u *url.URL
	if u, err = url.Parse(s); err != nil {
		return
	}

	var isDebug bool
	if len(debug) > 0 {
		isDebug = debug[0]
	}

	q := u.Query()
	facility := LogApplication
	t := q.Get("facility")
	if t != "" {
		if facility, err = parseFacility(t); err != nil {
			return
		}
	}
	tag := q.Get("tag")

	switch u.Scheme {
	case "file":
		// file:///var/log/xxx.log
		var fp *os.File
		if fp, err = os.OpenFile(u.Path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666); err != nil {
			return
		}
		DefaultLogger = &consoleLogger{w: fp, d: isDebug}
		return nil
	case "net":
		// net:///?facility=x&tag=x
		DefaultLogger, err = NewLogger(facility, tag, isDebug)
		return
	case "tcp":
		// tcp://localhost:port/?facility=x&tag=x
		DefaultLogger, err = NewLogger(facility, tag, isDebug, u.Host)
		return
	case "tcp4", "tcp6":
		return errors.New("not implemented")
	default:
		// ??
		return errors.New("unsupported scheme")
	}
}

// Debug outputs debug level log output.
// It is usually used to output detailed debug information.
// If debug status is not enabled, no output is generated.
// This log level need not be treated as an anomaly.
// The debug status can be set from SetDebug.
func Debug(format string, v ...interface{}) {
	DefaultLogger.Debug(format, v...)
}

// Info outputs information level log output.
// It is usually used to output interesting events.
// This log level need not be treated as an anomaly.
func Info(format string, v ...interface{}) {
	DefaultLogger.Info(format, v...)
}

// Warning outputs warning level log output.
// It is usually used to output exceptional occurrences that are not errors.
// This log level need not be treated as an anomaly.
func Warning(format string, v ...interface{}) {
	DefaultLogger.Warning(format, v...)
}

// Err outputs error level log output.
// It is usually used to output execution-time errors that do not require
// immediate action but should typically be logged and monitored.
// This log level need be treated as an anomaly.
func Err(format string, v ...interface{}) {
	DefaultLogger.Err(format, v...)
}

// Crit outputs critical level log output.
// It is usually used to output critical conditions.
// When this method is executed, the process abends after outputting the log.
// This log level need be treated as an anomaly.
func Crit(format string, v ...interface{}) {
	DefaultLogger.Crit(format, v...)
}

// SetDebug can enable/disable debug mode.
func SetDebug(state bool) {
	DefaultLogger.SetDebug(state)
}
<|endoftext|>


<commit_before>package naprrql

import (
	goxml "encoding/xml"
	"log"

	"github.com/nats-io/nuid"
	"github.com/nsip/nias2/xml"
	"github.com/syndtr/goleveldb/leveldb"
)

// internal check table for guid collisions
var fileGuids map[string]bool

// internal flag for data quality
var unfit bool

func IngestResultsFile(resultsFilePath string) {

	fileGuids = make(map[string]bool)
	db := GetDB()
	ge := GobEncoder{}

	// open the data file for streaming read
	xmlFile, err := OpenResultsFile(resultsFilePath)
	if err != nil {
		log.Fatalln("unable to open results file")
	}

	log.Printf("Reading data file [%s]", resultsFilePath)

	batch := new(leveldb.Batch)

	decoder := goxml.NewDecoder(xmlFile)
	totalTests := 0
	totalTestlets := 0
	totalTestItems := 0
	totalTestScoreSummarys := 0
	totalEvents := 0
	totalResponses := 0
	totalCodeFrames := 0
	totalSchools := 0
	totalStudents := 0
	var inElement string
	for {
		// Read tokens from the XML document in a stream.
		t, _ := decoder.Token()
		if t == nil {
			break
		}
		// Inspect the type of the token just read.
		switch se := t.(type) {
		case goxml.StartElement:
			// If we just read a StartElement token
			inElement = se.Name.Local
			// ...handle by type
			switch inElement {
			case "NAPTest":
				var t xml.NAPTest
				decoder.DecodeElement(&t, &se)
				gt, err := ge.Encode(t)
				if err != nil {
					log.Println("Unable to gob-encode nap test: ", err)
				}

				if isGuidCollision(db, t.TestID) {
					continue
				}

				// {NAPTest} = object
				batch.Put([]byte(t.TestID), gt)

				// NAPTest-type:{id} = id
				key := []byte("NAPTest:" + t.TestID)
				batch.Put(key, []byte(t.TestID))

				totalTests++

			case "NAPTestlet":
				var tl xml.NAPTestlet
				decoder.DecodeElement(&tl, &se)
				gtl, err := ge.Encode(tl)
				if err != nil {
					log.Println("Unable to gob-encode nap testlet: ", err)
				}

				if isGuidCollision(db, tl.TestletID) {
					continue
				}

				// {NAPTestlet} = object
				batch.Put([]byte(tl.TestletID), gtl)

				// NAPTestlet-type:{id} = {id}
				key := []byte("NAPTestlet:" + tl.TestletID)
				batch.Put(key, []byte(tl.TestletID))

				totalTestlets++

			case "NAPTestItem":
				var ti xml.NAPTestItem
				decoder.DecodeElement(&ti, &se)
				gti, err := ge.Encode(ti)
				if err != nil {
					log.Println("Unable to gob-encode nap test item: ", err)
				}

				if isGuidCollision(db, ti.ItemID) {
					continue
				}

				// {NAPTestItem} = object
				batch.Put([]byte(ti.ItemID), gti)

				// NapTestItem-type:{id} = {id}
				key := []byte("NAPTestItem:" + ti.ItemID)
				batch.Put(key, []byte(ti.ItemID))

				totalTestItems++

			case "NAPTestScoreSummary":
				var tss xml.NAPTestScoreSummary
				decoder.DecodeElement(&tss, &se)
				gtss, err := ge.Encode(tss)
				if err != nil {
					log.Println("Unable to gob-encode nap test-score-summary: ", err)
				}

				if isGuidCollision(db, tss.SummaryID) {
					continue
				}

				// {NAPTestScoreSummary} = object
				batch.Put([]byte(tss.SummaryID), gtss)

				// NAPTestScoreSummary-type:{id} = {id}
				key := []byte("NAPTestScoreSummary:" + tss.SummaryID)
				batch.Put(key, []byte(tss.SummaryID))

				// {school}:NAPTestScoreSummary-type:{id} = {id}
				key = []byte(tss.SchoolInfoRefId + ":NAPTestScoreSummary:" + tss.SummaryID)
				batch.Put(key, []byte(tss.SummaryID))

				// {test}:NAPTestScoreSummary-type:{school}:{id} = {id}
				key = []byte(tss.NAPTestRefId + ":NAPTestScoreSummary:" + tss.SchoolInfoRefId + ":" + tss.SummaryID)
				batch.Put(key, []byte(tss.SummaryID))

				totalTestScoreSummarys++

			case "NAPEventStudentLink":
				var e xml.NAPEvent
				decoder.DecodeElement(&e, &se)
				gev, err := ge.Encode(e)
				if err != nil {
					log.Println("Unable to gob-encode nap event link: ", err)
				}

				if isGuidCollision(db, e.EventID) {
					continue
				}

				// {NAPEventStudentLink} = object
				batch.Put([]byte(e.EventID), gev)

				// NAPEventStudentLink-type:{id} = {id}
				key := []byte("NAPEventStudentLink:" + e.EventID)
				batch.Put(key, []byte(e.EventID))

				// {school}:NAPEventStudentLink-type:{id} = {id}
				key = []byte(e.SchoolRefID + ":NAPEventStudentLink:" + e.EventID)
				batch.Put(key, []byte(e.EventID))

				// {student}:NAPEventStudentLink-type:{id} = {id}
				key = []byte(e.SPRefID + ":NAPEventStudentLink:" + e.EventID)
				batch.Put(key, []byte(e.EventID))

				// {test}:NAPEventStudentLink-type:{school}:{id} = {id}
				key = []byte(e.TestID + ":NAPEventStudentLink:" + e.SchoolRefID + ":" + e.EventID)
				batch.Put(key, []byte(e.EventID))

				totalEvents++

			case "NAPStudentResponseSet":
				var r xml.NAPResponseSet
				decoder.DecodeElement(&r, &se)
				gr, err := ge.Encode(r)
				if err != nil {
					log.Println("Unable to gob-encode student response set: ", err)
				}

				if isGuidCollision(db, r.ResponseID) {
					continue
				}

				// {response-id} = object
				batch.Put([]byte(r.ResponseID), gr)

				// response-type:{id} = {id}
				key := []byte("NAPStudentResponseSet:" + r.ResponseID)
				batch.Put(key, []byte(r.ResponseID))

				// {test}:NAPStudentResponseSet-type:{student} = {id}
				key = []byte(r.TestID + ":NAPStudentResponseSet:" + r.StudentID)
				batch.Put(key, []byte(r.ResponseID))

				// responseset_by_student:{sprefid}:{id} = {id}
				key = []byte("responseset_by_student:" + r.StudentID + ":" + r.ResponseID)
				batch.Put(key, []byte(r.ResponseID))

				totalResponses++

			case "NAPCodeFrame":
				var cf xml.NAPCodeFrame
				decoder.DecodeElement(&cf, &se)
				gcf, err := ge.Encode(cf)
				if err != nil {
					log.Println("Unable to gob-encode nap codeframe: ", err)
				}

				if isGuidCollision(db, cf.RefId) {
					continue
				}

				// {NAPCodeFrame-id} = object
				batch.Put([]byte(cf.RefId), gcf)

				// NAPCodeFrame-type:{id} = {id}
				key := []byte("NAPCodeFrame:" + cf.RefId)
				batch.Put(key, []byte(cf.RefId))

				totalCodeFrames++

			case "SchoolInfo":
				var si xml.SchoolInfo
				decoder.DecodeElement(&si, &se)
				gsi, err := ge.Encode(si)
				if err != nil {
					log.Println("Unable to gob-encode schoolinfo: ", err)
				}

				if isGuidCollision(db, si.RefId) {
					continue
				}

				// {SchoolInfo-id} = object
				batch.Put([]byte(si.RefId), gsi)

				// SchoolInfo-type:{id} = {id}
				key := []byte("SchoolInfo:" + si.RefId)
				batch.Put(key, []byte(si.RefId))

				// ASL lookup
				// {acara-id} = {refid}
				key = []byte(si.ACARAId + ":")
				batch.Put(key, []byte(si.RefId))

				// SchoolDetails lookup object
				// not a sif object so needs a guid
				sd_id := nuid.Next()
				key = []byte(sd_id)
				sd := SchoolDetails{
					SchoolName: si.SchoolName,
					ACARAId:    si.ACARAId,
				}
				gsd, err := ge.Encode(sd)
				if err != nil {
					log.Println("Unable to gob-encode schooldetails: ", err)
				}
				// {SchoolDetails-id} = object
				batch.Put(key, gsd)

				// SchoolDetails-type:{id} = {id}
				key = []byte("SchoolDetails:" + sd_id)
				batch.Put(key, []byte(sd_id))

				totalSchools++

			case "StudentPersonal":
				var sp xml.RegistrationRecord
				decoder.DecodeElement(&sp, &se)
				gsp, err := ge.Encode(sp)
				if err != nil {
					log.Println("Unable to gob-encode studentpersonal: ", err)
				}

				if isGuidCollision(db, sp.RefId) {
					continue
				}

				// {StudentPersonal-id} = object
				batch.Put([]byte(sp.RefId), gsp)

				// StudentPersonal-type:{id} = {id}
				key := []byte("StudentPersonal:" + sp.RefId)
				batch.Put(key, []byte(sp.RefId))

				// student_by_acaraid:{asl-id}:{studentpersonal-id} = {id}
				key = []byte("student_by_acaraid:" + sp.ASLSchoolId + ":" + sp.RefId)
				batch.Put(key, []byte(sp.RefId))

				totalStudents++

			}
			// write the batch out regularly to prevent
			// memory exhaustion for large inputs
			if (batch.Len() > 0) && (batch.Len()%20000) == 0 {
				batcherr := db.Write(batch, nil)
				if batcherr != nil {
					log.Fatalln("batch error: ", batcherr)
				}
			}
		default:
		}

	}

	// write any remaining batch entries
	// since last flush
	batcherr := db.Write(batch, nil)
	if batcherr != nil {
		log.Fatalln("batch error: ", batcherr)
	}

	log.Println("Data file read complete...")
	log.Printf("Total tests: %d \n", totalTests)
	log.Printf("Total codeframes: %d \n", totalCodeFrames)
	log.Printf("Total testlets: %d \n", totalTestlets)
	log.Printf("Total test items: %d \n", totalTestItems)
	log.Printf("Total test score summaries: %d \n", totalTestScoreSummarys)
	log.Printf("Total events: %d \n", totalEvents)
	log.Printf("Total responses: %d \n", totalResponses)
	log.Printf("Total schools: %d \n", totalSchools)
	log.Printf("Total students: %d \n", totalStudents)

	log.Printf("ingestion complete for [%s]", resultsFilePath)

}

//
// guids should only ever exist once in the db, if they are already present
// flag a warning to the console
//
func isGuidCollision(db *leveldb.DB, guid string) bool {
	_, exists := fileGuids[guid]
	if exists {
		log.Printf("Illegal attempt to assign guid {%s} to more than one object", guid)
		unfit = true // flag this data set has issues
		return exists
	}
	fileGuids[guid] = true
	return exists
}

//
// pick up any quality errors found during ingest
//
func DataUnfit() bool {

	return unfit

}
<commit_msg>batch ingest update<commit_after>package naprrql

import (
	goxml "encoding/xml"
	"log"

	"github.com/nats-io/nuid"
	"github.com/nsip/nias2/xml"
	"github.com/syndtr/goleveldb/leveldb"
)

// internal check table for guid collisions
var fileGuids map[string]bool

// internal flag for data quality
var unfit bool

func IngestResultsFile(resultsFilePath string) {

	fileGuids = make(map[string]bool)
	db := GetDB()
	ge := GobEncoder{}

	// open the data file for streaming read
	xmlFile, err := OpenResultsFile(resultsFilePath)
	if err != nil {
		log.Fatalln("unable to open results file")
	}

	log.Printf("Reading data file [%s]", resultsFilePath)

	batch := new(leveldb.Batch)

	decoder := goxml.NewDecoder(xmlFile)
	totalTests := 0
	totalTestlets := 0
	totalTestItems := 0
	totalTestScoreSummarys := 0
	totalEvents := 0
	totalResponses := 0
	totalCodeFrames := 0
	totalSchools := 0
	totalStudents := 0
	var inElement string
	for {
		// Read tokens from the XML document in a stream.
		t, _ := decoder.Token()
		if t == nil {
			break
		}
		// Inspect the type of the token just read.
		switch se := t.(type) {
		case goxml.StartElement:
			// If we just read a StartElement token
			inElement = se.Name.Local
			// ...handle by type
			switch inElement {
			case "NAPTest":
				var t xml.NAPTest
				decoder.DecodeElement(&t, &se)
				gt, err := ge.Encode(t)
				if err != nil {
					log.Println("Unable to gob-encode nap test: ", err)
				}

				if isGuidCollision(db, t.TestID) {
					continue
				}

				// {NAPTest} = object
				batch.Put([]byte(t.TestID), gt)

				// NAPTest-type:{id} = id
				key := []byte("NAPTest:" + t.TestID)
				batch.Put(key, []byte(t.TestID))

				totalTests++

			case "NAPTestlet":
				var tl xml.NAPTestlet
				decoder.DecodeElement(&tl, &se)
				gtl, err := ge.Encode(tl)
				if err != nil {
					log.Println("Unable to gob-encode nap testlet: ", err)
				}

				if isGuidCollision(db, tl.TestletID) {
					continue
				}

				// {NAPTestlet} = object
				batch.Put([]byte(tl.TestletID), gtl)

				// NAPTestlet-type:{id} = {id}
				key := []byte("NAPTestlet:" + tl.TestletID)
				batch.Put(key, []byte(tl.TestletID))

				totalTestlets++

			case "NAPTestItem":
				var ti xml.NAPTestItem
				decoder.DecodeElement(&ti, &se)
				gti, err := ge.Encode(ti)
				if err != nil {
					log.Println("Unable to gob-encode nap test item: ", err)
				}

				if isGuidCollision(db, ti.ItemID) {
					continue
				}

				// {NAPTestItem} = object
				batch.Put([]byte(ti.ItemID), gti)

				// NapTestItem-type:{id} = {id}
				key := []byte("NAPTestItem:" + ti.ItemID)
				batch.Put(key, []byte(ti.ItemID))

				totalTestItems++

			case "NAPTestScoreSummary":
				var tss xml.NAPTestScoreSummary
				decoder.DecodeElement(&tss, &se)
				gtss, err := ge.Encode(tss)
				if err != nil {
					log.Println("Unable to gob-encode nap test-score-summary: ", err)
				}

				if isGuidCollision(db, tss.SummaryID) {
					continue
				}

				// {NAPTestScoreSummary} = object
				batch.Put([]byte(tss.SummaryID), gtss)

				// NAPTestScoreSummary-type:{id} = {id}
				key := []byte("NAPTestScoreSummary:" + tss.SummaryID)
				batch.Put(key, []byte(tss.SummaryID))

				// {school}:NAPTestScoreSummary-type:{id} = {id}
				key = []byte(tss.SchoolInfoRefId + ":NAPTestScoreSummary:" + tss.SummaryID)
				batch.Put(key, []byte(tss.SummaryID))

				// {test}:NAPTestScoreSummary-type:{school}:{id} = {id}
				key = []byte(tss.NAPTestRefId + ":NAPTestScoreSummary:" + tss.SchoolInfoRefId + ":" + tss.SummaryID)
				batch.Put(key, []byte(tss.SummaryID))

				totalTestScoreSummarys++

			case "NAPEventStudentLink":
				var e xml.NAPEvent
				decoder.DecodeElement(&e, &se)
				gev, err := ge.Encode(e)
				if err != nil {
					log.Println("Unable to gob-encode nap event link: ", err)
				}

				if isGuidCollision(db, e.EventID) {
					continue
				}

				// {NAPEventStudentLink} = object
				batch.Put([]byte(e.EventID), gev)

				// NAPEventStudentLink-type:{id} = {id}
				key := []byte("NAPEventStudentLink:" + e.EventID)
				batch.Put(key, []byte(e.EventID))

				// {school}:NAPEventStudentLink-type:{id} = {id}
				key = []byte(e.SchoolRefID + ":NAPEventStudentLink:" + e.EventID)
				batch.Put(key, []byte(e.EventID))

				// {student}:NAPEventStudentLink-type:{id} = {id}
				key = []byte(e.SPRefID + ":NAPEventStudentLink:" + e.EventID)
				batch.Put(key, []byte(e.EventID))

				// {test}:NAPEventStudentLink-type:{school}:{id} = {id}
				key = []byte(e.TestID + ":NAPEventStudentLink:" + e.SchoolRefID + ":" + e.EventID)
				batch.Put(key, []byte(e.EventID))

				totalEvents++

			case "NAPStudentResponseSet":
				var r xml.NAPResponseSet
				decoder.DecodeElement(&r, &se)
				gr, err := ge.Encode(r)
				if err != nil {
					log.Println("Unable to gob-encode student response set: ", err)
				}

				if isGuidCollision(db, r.ResponseID) {
					continue
				}

				// {response-id} = object
				batch.Put([]byte(r.ResponseID), gr)

				// response-type:{id} = {id}
				key := []byte("NAPStudentResponseSet:" + r.ResponseID)
				batch.Put(key, []byte(r.ResponseID))

				// {test}:NAPStudentResponseSet-type:{student} = {id}
				key = []byte(r.TestID + ":NAPStudentResponseSet:" + r.StudentID)
				batch.Put(key, []byte(r.ResponseID))

				// responseset_by_student:{sprefid}:{id} = {id}
				key = []byte("responseset_by_student:" + r.StudentID + ":" + r.ResponseID)
				batch.Put(key, []byte(r.ResponseID))

				totalResponses++

			case "NAPCodeFrame":
				var cf xml.NAPCodeFrame
				decoder.DecodeElement(&cf, &se)
				gcf, err := ge.Encode(cf)
				if err != nil {
					log.Println("Unable to gob-encode nap codeframe: ", err)
				}

				if isGuidCollision(db, cf.RefId) {
					continue
				}

				// {NAPCodeFrame-id} = object
				batch.Put([]byte(cf.RefId), gcf)

				// NAPCodeFrame-type:{id} = {id}
				key := []byte("NAPCodeFrame:" + cf.RefId)
				batch.Put(key, []byte(cf.RefId))

				totalCodeFrames++

			case "SchoolInfo":
				var si xml.SchoolInfo
				decoder.DecodeElement(&si, &se)
				gsi, err := ge.Encode(si)
				if err != nil {
					log.Println("Unable to gob-encode schoolinfo: ", err)
				}

				if isGuidCollision(db, si.RefId) {
					continue
				}

				// {SchoolInfo-id} = object
				batch.Put([]byte(si.RefId), gsi)

				// SchoolInfo-type:{id} = {id}
				key := []byte("SchoolInfo:" + si.RefId)
				batch.Put(key, []byte(si.RefId))

				// ASL lookup
				// {acara-id} = {refid}
				key = []byte(si.ACARAId + ":")
				batch.Put(key, []byte(si.RefId))

				// SchoolDetails lookup object
				// not a sif object so needs a guid
				sd_id := nuid.Next()
				key = []byte(sd_id)
				sd := SchoolDetails{
					SchoolName: si.SchoolName,
					ACARAId:    si.ACARAId,
				}
				gsd, err := ge.Encode(sd)
				if err != nil {
					log.Println("Unable to gob-encode schooldetails: ", err)
				}
				// {SchoolDetails-id} = object
				batch.Put(key, gsd)

				// SchoolDetails-type:{id} = {id}
				key = []byte("SchoolDetails:" + sd_id)
				batch.Put(key, []byte(sd_id))

				totalSchools++

			case "StudentPersonal":
				var sp xml.RegistrationRecord
				decoder.DecodeElement(&sp, &se)
				gsp, err := ge.Encode(sp)
				if err != nil {
					log.Println("Unable to gob-encode studentpersonal: ", err)
				}

				if isGuidCollision(db, sp.RefId) {
					continue
				}

				// {StudentPersonal-id} = object
				batch.Put([]byte(sp.RefId), gsp)

				// StudentPersonal-type:{id} = {id}
				key := []byte("StudentPersonal:" + sp.RefId)
				batch.Put(key, []byte(sp.RefId))

				// student_by_acaraid:{asl-id}:{studentpersonal-id} = {id}
				key = []byte("student_by_acaraid:" + sp.ASLSchoolId + ":" + sp.RefId)
				batch.Put(key, []byte(sp.RefId))

				totalStudents++

			}
			// write the batch out regularly to prevent
			// memory exhaustion for large inputs
			if (batch.Len() > 0) && (batch.Len()%20000) == 0 {
				batcherr := db.Write(batch, nil)
				if batcherr != nil {
					log.Fatalln("batch error: ", batcherr)
				}
				batch.Reset()
			}
		default:
		}

	}

	// write any remaining batch entries
	// since last flush
	batcherr := db.Write(batch, nil)
	if batcherr != nil {
		log.Fatalln("batch error: ", batcherr)
	}
	batch.Reset()

	log.Println("Data file read complete...")
	log.Printf("Total tests: %d \n", totalTests)
	log.Printf("Total codeframes: %d \n", totalCodeFrames)
	log.Printf("Total testlets: %d \n", totalTestlets)
	log.Printf("Total test items: %d \n", totalTestItems)
	log.Printf("Total test score summaries: %d \n", totalTestScoreSummarys)
	log.Printf("Total events: %d \n", totalEvents)
	log.Printf("Total responses: %d \n", totalResponses)
	log.Printf("Total schools: %d \n", totalSchools)
	log.Printf("Total students: %d \n", totalStudents)

	log.Printf("ingestion complete for [%s]", resultsFilePath)

}

//
// guids should only ever exist once in the db, if they are already present
// flag a warning to the console
//
func isGuidCollision(db *leveldb.DB, guid string) bool {
	_, exists := fileGuids[guid]
	if exists {
		log.Printf("Illegal attempt to assign guid {%s} to more than one object", guid)
		unfit = true // flag this data set has issues
		return exists
	}
	fileGuids[guid] = true
	return exists
}

//
// pick up any quality errors found during ingest
//
func DataUnfit() bool {

	return unfit

}
<|endoftext|>


<commit_before>package models_test

import (
	"encoding/json"

	"github.com/cloudfoundry-incubator/candiedyaml"
	. "github.com/cloudfoundry-incubator/runtime-schema/models"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("StagingMessages", func() {
	Describe("StagingRequestFromCC", func() {
		ccJSON := `{
           "app_id" : "fake-app_id",
           "task_id" : "fake-task_id",
           "memory_mb" : 1024,
           "disk_mb" : 10000,
           "file_descriptors" : 3,
           "environment" : [{"key": "FOO", "value":"BAR"}],
           "stack" : "fake-stack",
           "app_bits_download_uri" : "http://fake-download_uri",
           "build_artifacts_cache_download_uri" : "http://a-nice-place-to-get-valuable-artifacts.com",
           "buildpacks" : [{"name":"fake-buildpack-name", "key":"fake-buildpack-key" ,"url":"fake-buildpack-url"}]
        }`

		It("should be mapped to the CC's staging request JSON", func() {
			var stagingRequest StagingRequestFromCC
			err := json.Unmarshal([]byte(ccJSON), &stagingRequest)
			Ω(err).ShouldNot(HaveOccurred())

			Ω(stagingRequest).Should(Equal(StagingRequestFromCC{
				AppId:                          "fake-app_id",
				TaskId:                         "fake-task_id",
				Stack:                          "fake-stack",
				AppBitsDownloadUri:             "http://fake-download_uri",
				BuildArtifactsCacheDownloadUri: "http://a-nice-place-to-get-valuable-artifacts.com",
				MemoryMB:                       1024,
				FileDescriptors:                3,
				DiskMB:                         10000,
				Buildpacks: []Buildpack{
					{
						Name: "fake-buildpack-name",
						Key:  "fake-buildpack-key",
						Url:  "fake-buildpack-url",
					},
				},
				Environment: []EnvironmentVariable{
					{Key: "FOO", Value: "BAR"},
				},
			}))
		})
	})

	Describe("Buildpack", func() {
		ccJSONFragment := `{
						"name": "ocaml-buildpack",
            "key": "ocaml-buildpack-guid",
            "url": "http://ocaml.org/buildpack.zip"
          }`

		It("extracts key and url", func() {
			var buildpack Buildpack

			err := json.Unmarshal([]byte(ccJSONFragment), &buildpack)
			Ω(err).ShouldNot(HaveOccurred())

			Ω(buildpack).To(Equal(Buildpack{
				Name: "ocaml-buildpack",
				Key:  "ocaml-buildpack-guid",
				Url:  "http://ocaml.org/buildpack.zip",
			}))
		})
	})

	Describe("StagingInfo", func() {
		Context("when json", func() {
			stagingJSON := `{
				"buildpack_key": "buildpack-key",
				"detected_buildpack": "ocaml-buildpack",
				"start_command": "ocaml-my-camel"
			}`

			It("does not extract the `start_command` property", func() {
				var stagingInfo StagingInfo

				err := json.Unmarshal([]byte(stagingJSON), &stagingInfo)
				Ω(err).ShouldNot(HaveOccurred())

				Ω(stagingInfo).Should(Equal(StagingInfo{
					BuildpackKey:      "buildpack-key",
					DetectedBuildpack: "ocaml-buildpack",
				}))
			})
		})

		Context("when yaml", func() {
			stagingYAML := `---
detected_buildpack: yaml-buildpack
start_command: yaml-ize -d`

			It("exposes an extracted `detected_buildpack` property", func() {
				var stagingInfo StagingInfo

				err := candiedyaml.Unmarshal([]byte(stagingYAML), &stagingInfo)
				Ω(err).ShouldNot(HaveOccurred())

				Ω(stagingInfo).Should(Equal(StagingInfo{
					DetectedBuildpack: "yaml-buildpack",
					StartCommand:      "yaml-ize -d",
				}))
			})
		})
	})

	Describe("StagingResponseForCC", func() {
		Context("with a detected buildpack", func() {
			It("generates valid JSON with the buildpack", func() {
				stagingResponseForCC := StagingResponseForCC{
					DetectedBuildpack: "ocaml-buildpack",
				}

				Ω(json.Marshal(stagingResponseForCC)).Should(MatchJSON(`{"detected_buildpack": "ocaml-buildpack"}`))
			})
		})

		Context("with an admin buildpack key", func() {
			It("generates valid JSON with the buildpack key", func() {
				stagingResponseForCC := StagingResponseForCC{
					BuildpackKey: "admin-buildpack-key",
				}

				Ω(json.Marshal(stagingResponseForCC)).Should(MatchJSON(`{"buildpack_key": "admin-buildpack-key"}`))
			})
		})

		Context("without an admin buildpack key", func() {
			It("generates valid JSON and omits the buildpack key", func() {
				stagingResponseForCC := StagingResponseForCC{}

				Ω(json.Marshal(stagingResponseForCC)).Should(MatchJSON(`{}`))
			})
		})

		Context("with an error", func() {
			It("generates valid JSON with the error", func() {
				stagingResponseForCC := StagingResponseForCC{
					Error: "FAIL, missing camels!",
				}

				Ω(json.Marshal(stagingResponseForCC)).Should(MatchJSON(`{"error": "FAIL, missing camels!"}`))
			})
		})
	})
})
<commit_msg>remove test that now asserts the opposite of reality<commit_after>package models_test

import (
	"encoding/json"

	"github.com/cloudfoundry-incubator/candiedyaml"
	. "github.com/cloudfoundry-incubator/runtime-schema/models"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("StagingMessages", func() {
	Describe("StagingRequestFromCC", func() {
		ccJSON := `{
           "app_id" : "fake-app_id",
           "task_id" : "fake-task_id",
           "memory_mb" : 1024,
           "disk_mb" : 10000,
           "file_descriptors" : 3,
           "environment" : [{"key": "FOO", "value":"BAR"}],
           "stack" : "fake-stack",
           "app_bits_download_uri" : "http://fake-download_uri",
           "build_artifacts_cache_download_uri" : "http://a-nice-place-to-get-valuable-artifacts.com",
           "buildpacks" : [{"name":"fake-buildpack-name", "key":"fake-buildpack-key" ,"url":"fake-buildpack-url"}]
        }`

		It("should be mapped to the CC's staging request JSON", func() {
			var stagingRequest StagingRequestFromCC
			err := json.Unmarshal([]byte(ccJSON), &stagingRequest)
			Ω(err).ShouldNot(HaveOccurred())

			Ω(stagingRequest).Should(Equal(StagingRequestFromCC{
				AppId:                          "fake-app_id",
				TaskId:                         "fake-task_id",
				Stack:                          "fake-stack",
				AppBitsDownloadUri:             "http://fake-download_uri",
				BuildArtifactsCacheDownloadUri: "http://a-nice-place-to-get-valuable-artifacts.com",
				MemoryMB:                       1024,
				FileDescriptors:                3,
				DiskMB:                         10000,
				Buildpacks: []Buildpack{
					{
						Name: "fake-buildpack-name",
						Key:  "fake-buildpack-key",
						Url:  "fake-buildpack-url",
					},
				},
				Environment: []EnvironmentVariable{
					{Key: "FOO", Value: "BAR"},
				},
			}))
		})
	})

	Describe("Buildpack", func() {
		ccJSONFragment := `{
						"name": "ocaml-buildpack",
            "key": "ocaml-buildpack-guid",
            "url": "http://ocaml.org/buildpack.zip"
          }`

		It("extracts key and url", func() {
			var buildpack Buildpack

			err := json.Unmarshal([]byte(ccJSONFragment), &buildpack)
			Ω(err).ShouldNot(HaveOccurred())

			Ω(buildpack).To(Equal(Buildpack{
				Name: "ocaml-buildpack",
				Key:  "ocaml-buildpack-guid",
				Url:  "http://ocaml.org/buildpack.zip",
			}))
		})
	})

	Describe("StagingInfo", func() {
		Context("when yaml", func() {
			stagingYAML := `---
detected_buildpack: yaml-buildpack
start_command: yaml-ize -d`

			It("exposes an extracted `detected_buildpack` property", func() {
				var stagingInfo StagingInfo

				err := candiedyaml.Unmarshal([]byte(stagingYAML), &stagingInfo)
				Ω(err).ShouldNot(HaveOccurred())

				Ω(stagingInfo).Should(Equal(StagingInfo{
					DetectedBuildpack: "yaml-buildpack",
					StartCommand:      "yaml-ize -d",
				}))
			})
		})
	})

	Describe("StagingResponseForCC", func() {
		Context("with a detected buildpack", func() {
			It("generates valid JSON with the buildpack", func() {
				stagingResponseForCC := StagingResponseForCC{
					DetectedBuildpack: "ocaml-buildpack",
				}

				Ω(json.Marshal(stagingResponseForCC)).Should(MatchJSON(`{"detected_buildpack": "ocaml-buildpack"}`))
			})
		})

		Context("with an admin buildpack key", func() {
			It("generates valid JSON with the buildpack key", func() {
				stagingResponseForCC := StagingResponseForCC{
					BuildpackKey: "admin-buildpack-key",
				}

				Ω(json.Marshal(stagingResponseForCC)).Should(MatchJSON(`{"buildpack_key": "admin-buildpack-key"}`))
			})
		})

		Context("without an admin buildpack key", func() {
			It("generates valid JSON and omits the buildpack key", func() {
				stagingResponseForCC := StagingResponseForCC{}

				Ω(json.Marshal(stagingResponseForCC)).Should(MatchJSON(`{}`))
			})
		})

		Context("with an error", func() {
			It("generates valid JSON with the error", func() {
				stagingResponseForCC := StagingResponseForCC{
					Error: "FAIL, missing camels!",
				}

				Ω(json.Marshal(stagingResponseForCC)).Should(MatchJSON(`{"error": "FAIL, missing camels!"}`))
			})
		})
	})
})
<|endoftext|>


<commit_before>package context

import (
	"os"
	"path/filepath"
	"strings"

	"github.com/kardianos/govendor/internal/pathos"
)

type licenseSearchType byte

const (
	licensePrefix licenseSearchType = iota
	licenseSubstring
	licenseSuffix
)

type license struct {
	Text   string
	Search licenseSearchType
}

func (t licenseSearchType) Test(filename, test string) bool {
	switch t {
	case licensePrefix:
		return strings.HasPrefix(filename, test)
	case licenseSubstring:
		return strings.Contains(filename, test)
	case licenseSuffix:
		return strings.HasSuffix(filename, test)
	}
	return false
}

type licenseTest interface {
	Test(filename, test string) bool
}

// licenses lists the filenames to copy over to the vendor folder.
var licenses = []license{
	{Text: "license", Search: licensePrefix},
	{Text: "unlicense", Search: licensePrefix},
	{Text: "copying", Search: licensePrefix},
	{Text: "copyright", Search: licensePrefix},
	{Text: "copyright", Search: licensePrefix},
	{Text: "legal", Search: licenseSubstring},
	{Text: "notice", Search: licenseSubstring},
	{Text: "disclaimer", Search: licenseSubstring},
	{Text: "patent", Search: licenseSubstring},
	{Text: "third-party", Search: licenseSubstring},
	{Text: "thirdparty", Search: licenseSubstring},
}

func isLicenseFile(name string) bool {
	cname := strings.ToLower(name)
	for _, L := range licenses {
		if L.Search.Test(cname, L.Text) {
			return true
		}
	}
	return false
}

// licenseCopy starts the search in the parent of "startIn" folder.
// Looks in all sub-folders until root is reached. The root itself is not
// searched.
func licenseCopy(root, startIn, vendorRoot string) error {
	folder := filepath.Clean(filepath.Join(startIn, ".."))
	for i := 0; i <= looplimit; i++ {
		dir, err := os.Open(folder)
		if err != nil {
			return err
		}

		fl, err := dir.Readdir(-1)
		dir.Close()
		if err != nil {
			return err
		}
		for _, fi := range fl {
			name := fi.Name()
			if name[0] == '.' {
				continue
			}
			if fi.IsDir() {
				continue
			}
			if !isLicenseFile(name) {
				continue
			}

			srcPath := filepath.Join(folder, name)
			destPath := filepath.Join(vendorRoot, pathos.FileTrimPrefix(folder, root), name)

			// Only copy if file does not exist.
			_, err := os.Stat(destPath)
			if err == nil {
				continue
			}

			err = copyFile(destPath, srcPath)
			if err != nil {
				return err
			}
		}

		nextFolder := filepath.Clean(filepath.Join(folder, ".."))

		if nextFolder == folder {
			return nil
		}
		if pathos.FileStringEquals(root, nextFolder) {
			return nil
		}
		folder = nextFolder
	}
	panic("copyLicense loop limit")
}
<commit_msg>context: add missing header comment<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package context

import (
	"os"
	"path/filepath"
	"strings"

	"github.com/kardianos/govendor/internal/pathos"
)

type licenseSearchType byte

const (
	licensePrefix licenseSearchType = iota
	licenseSubstring
	licenseSuffix
)

type license struct {
	Text   string
	Search licenseSearchType
}

func (t licenseSearchType) Test(filename, test string) bool {
	switch t {
	case licensePrefix:
		return strings.HasPrefix(filename, test)
	case licenseSubstring:
		return strings.Contains(filename, test)
	case licenseSuffix:
		return strings.HasSuffix(filename, test)
	}
	return false
}

type licenseTest interface {
	Test(filename, test string) bool
}

// licenses lists the filenames to copy over to the vendor folder.
var licenses = []license{
	{Text: "license", Search: licensePrefix},
	{Text: "unlicense", Search: licensePrefix},
	{Text: "copying", Search: licensePrefix},
	{Text: "copyright", Search: licensePrefix},
	{Text: "copyright", Search: licensePrefix},
	{Text: "legal", Search: licenseSubstring},
	{Text: "notice", Search: licenseSubstring},
	{Text: "disclaimer", Search: licenseSubstring},
	{Text: "patent", Search: licenseSubstring},
	{Text: "third-party", Search: licenseSubstring},
	{Text: "thirdparty", Search: licenseSubstring},
}

func isLicenseFile(name string) bool {
	cname := strings.ToLower(name)
	for _, L := range licenses {
		if L.Search.Test(cname, L.Text) {
			return true
		}
	}
	return false
}

// licenseCopy starts the search in the parent of "startIn" folder.
// Looks in all sub-folders until root is reached. The root itself is not
// searched.
func licenseCopy(root, startIn, vendorRoot string) error {
	folder := filepath.Clean(filepath.Join(startIn, ".."))
	for i := 0; i <= looplimit; i++ {
		dir, err := os.Open(folder)
		if err != nil {
			return err
		}

		fl, err := dir.Readdir(-1)
		dir.Close()
		if err != nil {
			return err
		}
		for _, fi := range fl {
			name := fi.Name()
			if name[0] == '.' {
				continue
			}
			if fi.IsDir() {
				continue
			}
			if !isLicenseFile(name) {
				continue
			}

			srcPath := filepath.Join(folder, name)
			destPath := filepath.Join(vendorRoot, pathos.FileTrimPrefix(folder, root), name)

			// Only copy if file does not exist.
			_, err := os.Stat(destPath)
			if err == nil {
				continue
			}

			err = copyFile(destPath, srcPath)
			if err != nil {
				return err
			}
		}

		nextFolder := filepath.Clean(filepath.Join(folder, ".."))

		if nextFolder == folder {
			return nil
		}
		if pathos.FileStringEquals(root, nextFolder) {
			return nil
		}
		folder = nextFolder
	}
	panic("copyLicense loop limit")
}
<|endoftext|>