<commit_before>package engine

import (
	"api"
	"bytes"
	"engine/game_engine"
	"strconv"
)

type requestHandler struct {
	sessionGame *game_engine.Game
	gameRequest map[string]func([]byte, int, *game_engine.Game) []byte
}

func newRequestHandler() *requestHandler {
	return &requestHandler{
		sessionGame: nil,
		gameRequest: map[string]func([]byte, int, *game_engine.Game) []byte{
			api.COMMAND_EXIT:         exitRequest,
			api.COMMAND_MOVE:         moveRequest,
			api.COMMAND_TURN:         endTurnRequest,
			api.COMMAND_ATTACK:       attackRequest,
			api.COMMAND_VIEW_WORLD:   viewWorldRequest,
			api.COMMAND_VIEW_TERRAIN: viewTerrainRequest,
			api.COMMAND_VIEW_UNITS:   viewUnitsRequest,
			api.COMMAND_VIEW_PLAYERS: viewPlayersRequest}}
}

func (handler *requestHandler) handleRequest(request []byte) []byte {
	command, requestJson := splitOnce(request)
	if handler.sessionGame == nil {
		if string(command) == api.COMMAND_NEW {
			response, game := newRequest(requestJson)
			handler.sessionGame = game
			return buildResponse(command, response)
		} else {
			return buildResponse(command, respondUnknownRequest("Need new game request"))
		}
	} else {
		fun, ok := handler.gameRequest[string(command)]
		if ok {
			playerId, requestJsonNoPlayerId := splitOnce(requestJson)
			playerIdInt, err := strconv.Atoi(string(playerId))
			if err != nil {
				return buildResponse(command, respondMalformed("playerId not an int"))
			}
			response := fun(requestJsonNoPlayerId, playerIdInt, handler.sessionGame)
			return buildResponse(command, response)
		} else {
			return buildResponse(command, respondUnknownRequest("Unknown command"))
		}
	}
}

func buildResponse(command []byte, response []byte) []byte {
	return append(append(command, []byte(":")...), response...)
}

func splitOnce(input []byte) ([]byte, []byte) {
	pieces := bytes.SplitN(input, []byte(":"), 2)
	if len(pieces) == 1 {
		return pieces[0], []byte{}
	} else if len(pieces) == 2 {
		return pieces[0], pieces[1]
	} else {
		return []byte{}, []byte{}
	}
}
<commit_msg>Convert single char to byte over string to []byte<commit_after>package engine

import (
	"api"
	"bytes"
	"engine/game_engine"
	"strconv"
)

type requestHandler struct {
	sessionGame *game_engine.Game
	gameRequest map[string]func([]byte, int, *game_engine.Game) []byte
}

func newRequestHandler() *requestHandler {
	return &requestHandler{
		sessionGame: nil,
		gameRequest: map[string]func([]byte, int, *game_engine.Game) []byte{
			api.COMMAND_EXIT:         exitRequest,
			api.COMMAND_MOVE:         moveRequest,
			api.COMMAND_TURN:         endTurnRequest,
			api.COMMAND_ATTACK:       attackRequest,
			api.COMMAND_VIEW_WORLD:   viewWorldRequest,
			api.COMMAND_VIEW_TERRAIN: viewTerrainRequest,
			api.COMMAND_VIEW_UNITS:   viewUnitsRequest,
			api.COMMAND_VIEW_PLAYERS: viewPlayersRequest}}
}

func (handler *requestHandler) handleRequest(request []byte) []byte {
	command, requestJson := splitOnce(request)
	if handler.sessionGame == nil {
		if string(command) == api.COMMAND_NEW {
			response, game := newRequest(requestJson)
			handler.sessionGame = game
			return buildResponse(command, response)
		} else {
			return buildResponse(command, respondUnknownRequest("Need new game request"))
		}
	} else {
		fun, ok := handler.gameRequest[string(command)]
		if ok {
			playerId, requestJsonNoPlayerId := splitOnce(requestJson)
			playerIdInt, err := strconv.Atoi(string(playerId))
			if err != nil {
				return buildResponse(command, respondMalformed("playerId not an int"))
			}
			response := fun(requestJsonNoPlayerId, playerIdInt, handler.sessionGame)
			return buildResponse(command, response)
		} else {
			return buildResponse(command, respondUnknownRequest("Unknown command"))
		}
	}
}

func buildResponse(command []byte, response []byte) []byte {
	return append(append(command, byte(':')), response...)
}

func splitOnce(input []byte) ([]byte, []byte) {
	pieces := bytes.SplitN(input, []byte(":"), 2)
	if len(pieces) == 1 {
		return pieces[0], []byte{}
	} else if len(pieces) == 2 {
		return pieces[0], pieces[1]
	} else {
		return []byte{}, []byte{}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	log "github.com/Sirupsen/logrus"
)

type patchFile struct {
	path    string
	version string
	blocks  []patchBlock
}

func writePatchFile(patch patchFile) error {
	log.Infof("Writing %s", patch.path)

	err := os.Remove(patch.path)
	check(err)

	f, err := os.Create(patch.path)
	check(err)
	defer f.Close()

	w := bufio.NewWriter(f)

	_, err = w.WriteString(fmt.Sprintf("> %s\n", patch.version))
	check(err)

	for _, block := range patch.blocks {
		_, err = w.WriteString("> BEGIN STRING\n")
		check(err)

		_, err = w.WriteString(block.original)
		check(err)

		for _, context := range block.contexts {
			context = fmt.Sprintf("> CONTEXT: %s", context)

			if !block.translated {
				context += " < UNTRANSLATED\n"
			} else {
				context += "\n"
			}
			_, err = w.WriteString(context)
			check(err)
		}

		var trans string

		if block.translated {
			trans = breakLines(block.translation)
		} else {
			trans = "\n"
		}
		_, err = w.WriteString(trans)
		check(err)

		_, err = w.WriteString("> END STRING\n\n")
		check(err)
	}

	err = w.Flush()
	check(err)

	log.Infof("Done writing %s", patch.path)

	return nil
}

func parsePatchFile(path string) (patchFile, error) {
	log.Info("Parsing", path)

	f, err := os.Open(path)
	defer f.Close()

	file := patchFile{path: path}

	s := bufio.NewScanner(f)
	s.Split(bufio.ScanLines)

	original := false
	translation := false

	var orig string
	var trans string
	var contexts []string

	var block patchBlock

	for s.Scan() {
		l := s.Text()

		if strings.HasPrefix(l, "> ") {
			l = l[2:]

			switch {
			case strings.HasPrefix(l, "RPGMAKER TRANS PATCH FILE VERSION"):
				file.version = l
			case strings.HasPrefix(l, "BEGIN STRING"):
				original = true
			case strings.HasPrefix(l, "CONTEXT: "):
				if len(l) > len("CONTEXT: ")+1 {
					start := len("CONTEXT: ")
					end := strings.Index(l, " < UNTRANSLATED")
					if end == -1 {
						contexts = append(contexts, l[start:])
					} else {
						contexts = append(contexts, l[start:end])
					}
				} else {
					log.Warn("Empty context?", l)
				}

				original = false
				translation = true
			case strings.HasPrefix(l, "END STRING"):
				translation = false

				block.original = orig
				block.contexts = contexts

				if len(strings.TrimRight(trans, "\n")) < 1 {
					block.translation = ""
				} else {
					block.translated = true
					block.translation = trans
				}

				//log.Info(spew.Sdump(block))

				file.blocks = append(file.blocks, block)

				orig = ""
				trans = ""
				contexts = nil
			default:
				log.Warn("Unknown input:", l)
			}

			continue
		}

		if !strings.HasSuffix(l, "\n") && (original || translation) {
			l += "\n"
		}

		if original {
			orig += l
		} else if translation {
			trans += l
		}
	}

	return file, err
}

func translatePatch(patch patchFile) (patchFile, error) {
	var err error

	if strings.HasSuffix(patch.path, "Scripts.txt") {
		return patch, err
	}

	for i, block := range patch.blocks {
		patch.blocks[i] = parseBlock(block)
	}

	return patch, err
}
<commit_msg>Make block parsing and translating concurrent<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"runtime"
	"strings"

	log "github.com/Sirupsen/logrus"
)

type patchFile struct {
	path    string
	version string
	blocks  []patchBlock
}

func writePatchFile(patch patchFile) error {
	log.Infof("Writing %s", patch.path)

	err := os.Remove(patch.path)
	check(err)

	f, err := os.Create(patch.path)
	check(err)
	defer f.Close()

	w := bufio.NewWriter(f)

	_, err = w.WriteString(fmt.Sprintf("> %s\n", patch.version))
	check(err)

	for _, block := range patch.blocks {
		_, err = w.WriteString("> BEGIN STRING\n")
		check(err)

		_, err = w.WriteString(block.original)
		check(err)

		for _, context := range block.contexts {
			context = fmt.Sprintf("> CONTEXT: %s", context)

			if !block.translated {
				context += " < UNTRANSLATED\n"
			} else {
				context += "\n"
			}
			_, err = w.WriteString(context)
			check(err)
		}

		var trans string

		if block.translated {
			trans = breakLines(block.translation)
		} else {
			trans = "\n"
		}
		_, err = w.WriteString(trans)
		check(err)

		_, err = w.WriteString("> END STRING\n\n")
		check(err)
	}

	err = w.Flush()
	check(err)

	log.Infof("Done writing %s", patch.path)

	return nil
}

func parsePatchFile(path string) (patchFile, error) {
	log.Info("Parsing", path)

	f, err := os.Open(path)
	defer f.Close()

	file := patchFile{path: path}

	s := bufio.NewScanner(f)
	s.Split(bufio.ScanLines)

	original := false
	translation := false

	var orig string
	var trans string
	var contexts []string

	var block patchBlock

	for s.Scan() {
		l := s.Text()

		if strings.HasPrefix(l, "> ") {
			l = l[2:]

			switch {
			case strings.HasPrefix(l, "RPGMAKER TRANS PATCH FILE VERSION"):
				file.version = l
			case strings.HasPrefix(l, "BEGIN STRING"):
				original = true
			case strings.HasPrefix(l, "CONTEXT: "):
				if len(l) > len("CONTEXT: ")+1 {
					start := len("CONTEXT: ")
					end := strings.Index(l, " < UNTRANSLATED")
					if end == -1 {
						contexts = append(contexts, l[start:])
					} else {
						contexts = append(contexts, l[start:end])
					}
				} else {
					log.Warn("Empty context?", l)
				}

				original = false
				translation = true
			case strings.HasPrefix(l, "END STRING"):
				translation = false

				block.original = orig
				block.contexts = contexts

				if len(strings.TrimRight(trans, "\n")) < 1 {
					block.translation = ""
				} else {
					block.translated = true
					block.translation = trans
				}

				//log.Info(spew.Sdump(block))

				file.blocks = append(file.blocks, block)

				orig = ""
				trans = ""
				contexts = nil
			default:
				log.Warn("Unknown input:", l)
			}

			continue
		}

		if !strings.HasSuffix(l, "\n") && (original || translation) {
			l += "\n"
		}

		if original {
			orig += l
		} else if translation {
			trans += l
		}
	}

	return file, err
}

func translatePatch(patch patchFile) (patchFile, error) {
	var err error

	if strings.HasSuffix(patch.path, "Scripts.txt") {
		return patch, err
	}

	// Only needed to preserve order in patch file
	type blockWork struct {
		id    int
		block patchBlock
	}

	jobs := make(chan blockWork, runtime.NumCPU()*2)
	results := make(chan blockWork, runtime.NumCPU()*2)

	// Start workers
	for w := 1; w <= runtime.NumCPU(); w++ {
		go func(jobs <-chan blockWork, results chan<- blockWork) {
			for j := range jobs {
				j.block = parseBlock(j.block)
				results <- j
			}
		}(jobs, results)
	}

	// Add blocks in background to job queue
	go func() {
		for i, block := range patch.blocks {
			//patch.blocks[i] = parseBlock(block)
			w := blockWork{i, block}
			jobs <- w
		}
		close(jobs)
	}()

	// Start reading results, will block if there are none
	for a := len(patch.blocks); a > 0; a-- {

		j := <-results

		patch.blocks[j.id] = j.block
	}

	return patch, err
}
<|endoftext|>


<commit_before>package cruncy

import (
	"bufio"
	"fmt"
	"io"
	"path/filepath"
	"syscall"

	"os"
	"os/exec"
	"strings"
)

// EnsureFileSave creates directory unless exists for a given file
func EnsureFileSave(fileName string) {
	pt := filepath.Dir(fileName)
	CreateDirUnlessExists(pt)
}

// Exists returns true if file/path exists
func Exists(path string) bool {
	_, err := os.Stat(path)
	return err == nil || os.IsExist(err)
}

// CreateDirUnlessExists creates a directory if to do not exist
func CreateDirUnlessExists(path string) {
	if !Exists(path) {
		os.MkdirAll(path, os.ModeDir|0755)
	}
}

// Decompress decompresses a file using os utility gzip
func Decompress(fileName string) (string, error) {
	err := DoCmd("gzip", "-d", fileName)
	if err != nil {
		return "", err
	}
	return strings.Replace(fileName, ".gz", "", -1), nil
}

// DoCmd Runs a os command
func DoCmd(command string, arg ...string) error {
	cmd := exec.Command(command, arg...) // no need to call Output method here
	cmd.Stdout = os.Stdout               // instead use Stdout
	cmd.Stderr = os.Stderr               // attach Stderr as well

	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("Unable to execute %s with error %s", command, err)
	}
	return nil
}

// FileToReader gets a buffered reader from a fileName
func FileToReader(fileName string) (io.Reader, error) {
	if _, err := os.Stat(fileName); os.IsNotExist(err) {
		return nil, fmt.Errorf("File %s do not exist", fileName)
	}

	f, err := os.Open(fileName)
	if err != nil {
		return nil, fmt.Errorf("Unable to open %s with error: %s", fileName, err)
	}

	return bufio.NewReader(f), nil
}

// DiskUsage disk usage of path/disk
func DiskUsage(path string, checkSize uint64) float64 {

	fs := syscall.Statfs_t{}
	err := syscall.Statfs(path, &fs)
	if err != nil {
		return 0.0
	}
	all := fs.Blocks * uint64(fs.Bsize)
	free := fs.Bfree * uint64(fs.Bsize)

	// Adding download size to the calculation.
	used := all - free - checkSize

	return (float64(used) / float64(all)) * 100.0

}
<commit_msg>Added more common file functions<commit_after>package cruncy

import (
	"bufio"
	"fmt"
	"io"
	"net/http"
	"path/filepath"
	"syscall"

	"os"
	"os/exec"
	"strings"

	"github.com/pkg/errors"
)

// EnsureFileSave creates directory unless exists for a given file
func EnsureFileSave(fileName string) {
	pt := filepath.Dir(fileName)
	CreateDirUnlessExists(pt)
}

// Exists returns true if file/path exists
func Exists(path string) bool {
	_, err := os.Stat(path)
	return err == nil || os.IsExist(err)
}

// CreateDirUnlessExists creates a directory if to do not exist
func CreateDirUnlessExists(path string) {
	if !Exists(path) {
		os.MkdirAll(path, os.ModeDir|0755)
	}
}

// Decompress decompresses a file using os utility gzip
func Decompress(fileName string) (string, error) {
	err := DoCmd("gzip", "-d", fileName)
	if err != nil {
		return "", err
	}
	return strings.Replace(fileName, ".gz", "", -1), nil
}

// DoCmd Runs a os command
func DoCmd(command string, arg ...string) error {
	cmd := exec.Command(command, arg...) // no need to call Output method here
	cmd.Stdout = os.Stdout               // instead use Stdout
	cmd.Stderr = os.Stderr               // attach Stderr as well

	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("Unable to execute %s with error %s", command, err)
	}
	return nil
}

// FileToReader gets a buffered reader from a fileName
func FileToReader(fileName string) (io.Reader, error) {
	if _, err := os.Stat(fileName); os.IsNotExist(err) {
		return nil, fmt.Errorf("File %s do not exist", fileName)
	}

	f, err := os.Open(fileName)
	if err != nil {
		return nil, fmt.Errorf("Unable to open %s with error: %s", fileName, err)
	}

	return bufio.NewReader(f), nil
}

// DiskUsage disk usage of path/disk
func DiskUsage(path string, checkSize uint64) float64 {

	fs := syscall.Statfs_t{}
	err := syscall.Statfs(path, &fs)
	if err != nil {
		return 0.0
	}
	all := fs.Blocks * uint64(fs.Bsize)
	free := fs.Bfree * uint64(fs.Bsize)

	// Adding download size to the calculation.
	used := all - free - checkSize

	return (float64(used) / float64(all)) * 100.0
}

// IsDirectory is path a directory
func IsDirectory(path string) bool {
	fileInfo, err := os.Stat(path)
	if err != nil {
		return false
	}
	return fileInfo.IsDir()
}

// DeleteFile deletes a file
func DeleteFile(fileName string) error {
	err := os.Remove(fileName)
	if err != nil {
		return fmt.Errorf("unable to delete %s with error: %s", fileName, err)
	}

	return nil
}

// IsTextFile tells if a file is a text file
func IsTextFile(fileName string) (bool, error) {
	peekLen := 64

	f, err := os.Open(fileName)
	if err != nil {
		return false, errors.Wrap(err, fmt.Sprintf("isTextFile Unable to open  %s with error %s", fileName, err))
	}
	defer f.Close()
	b := make([]byte, peekLen)
	_, err = f.Read(b)
	if err != nil && err != io.EOF {
		return false, errors.Wrap(err, fmt.Sprintf("isTextFile Unable to open  %s with error %s", fileName, err))
	}
	return IsTextData(b), nil
}

// IsTextData checks if a byte stream is text
func IsTextData(data []byte) bool {
	if len(data) == 0 {
		return true
	}
	return strings.Contains(http.DetectContentType(data), "text/")
}
<|endoftext|>


<commit_before>package xlsx

import (
	"archive/zip"
	"encoding/xml"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
)

// File is a high level structure providing a slice of Sheet structs
// to the user.
type File struct {
	worksheets     map[string]*zip.File
	referenceTable *RefTable
	Date1904       bool
	styles         *xlsxStyleSheet
	Sheets         []*Sheet
	Sheet          map[string]*Sheet
	theme          *theme
}

// Create a new File
func NewFile() (file *File) {
	file = &File{}
	file.Sheet = make(map[string]*Sheet)
	file.Sheets = make([]*Sheet, 0)
	return
}

// OpenFile() take the name of an XLSX file and returns a populated
// xlsx.File struct for it.
func OpenFile(filename string) (file *File, err error) {
	var f *zip.ReadCloser
	f, err = zip.OpenReader(filename)
	if err != nil {
		return nil, err
	}
	file, err = ReadZip(f)
	return
}

// A convenient wrapper around File.ToSlice, FileToSlice will
// return the raw data contained in an Excel XLSX file as three
// dimensional slice.  The first index represents the sheet number,
// the second the row number, and the third the cell number.
//
// For example:
//
//    var mySlice [][][]string
//    var value string
//    mySlice = xlsx.FileToSlice("myXLSX.xlsx")
//    value = mySlice[0][0][0]
//
// Here, value would be set to the raw value of the cell A1 in the
// first sheet in the XLSX file.
func FileToSlice(path string) ([][][]string, error) {
	f, err := OpenFile(path)
	if err != nil {
		return nil, err
	}
	return f.ToSlice()
}

// Save the File to an xlsx file at the provided path.
func (f *File) Save(path string) (err error) {
	var target *os.File

	target, err = os.Create(path)
	if err != nil {
		return
	}

	err = f.Write(target)
	if err != nil {
		return
	}

	return target.Close()
}

// Write the File to io.Writer as xlsx
func (f *File) Write(writer io.Writer) (err error) {
	var parts map[string]string
	var zipWriter *zip.Writer

	parts, err = f.MarshallParts()
	if err != nil {
		return
	}

	zipWriter = zip.NewWriter(writer)

	for partName, part := range parts {
		var writer io.Writer
		writer, err = zipWriter.Create(partName)
		if err != nil {
			return
		}
		_, err = writer.Write([]byte(part))
		if err != nil {
			return
		}
	}

	err = zipWriter.Close()

	return
}

// Add a new Sheet, with the provided name, to a File
func (f *File) AddSheet(sheetName string) (sheet *Sheet) {
	sheet = &Sheet{Name: sheetName, File: f}
	if len(f.Sheets) == 0 {
		sheet.Selected = true
	}
	f.Sheet[sheetName] = sheet
	f.Sheets = append(f.Sheets, sheet)
	return sheet
}

func (f *File) makeWorkbook() xlsxWorkbook {
	var workbook xlsxWorkbook
	workbook = xlsxWorkbook{}
	workbook.FileVersion = xlsxFileVersion{}
	workbook.FileVersion.AppName = "Go XLSX"
	workbook.WorkbookPr = xlsxWorkbookPr{
		BackupFile:  false,
		ShowObjects: "all"}
	workbook.BookViews = xlsxBookViews{}
	workbook.BookViews.WorkBookView = make([]xlsxWorkBookView, 1)
	workbook.BookViews.WorkBookView[0] = xlsxWorkBookView{
		ActiveTab:            0,
		FirstSheet:           0,
		ShowHorizontalScroll: true,
		ShowSheetTabs:        true,
		ShowVerticalScroll:   true,
		TabRatio:             204,
		WindowHeight:         8192,
		WindowWidth:          16384,
		XWindow:              "0",
		YWindow:              "0"}
	workbook.Sheets = xlsxSheets{}
	workbook.Sheets.Sheet = make([]xlsxSheet, len(f.Sheets))
	workbook.CalcPr.IterateCount = 100
	workbook.CalcPr.RefMode = "A1"
	workbook.CalcPr.Iterate = false
	workbook.CalcPr.IterateDelta = 0.001
	return workbook
}

// Some tools that read XLSX files have very strict requirements about
// the structure of the input XML.  In particular both Numbers on the Mac
// and SAS dislike inline XML namespace declarations, or namespace
// prefixes that don't match the ones that Excel itself uses.  This is a
// problem because the Go XML library doesn't multiple namespace
// declarations in a single element of a document.  This function is a
// horrible hack to fix that after the XML marshalling is completed.
func replaceRelationshipsNameSpace(workbookMarshal string) string {
	newWorkbook := strings.Replace(workbookMarshal, `xmlns:relationships="http://schemas.openxmlformats.org/officeDocument/2006/relationships" relationships:id`, `r:id`, -1)
	// Dirty hack to fix issues #63 and #91; encoding/xml currently
	// "doesn't allow for additional namespaces to be defined in the
	// root element of the document," as described by @tealeg in the
	// comments for #63.
	oldXmlns := `<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">`
	newXmlns := `<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">`
	return strings.Replace(newWorkbook, oldXmlns, newXmlns, 1)
}

// Construct a map of file name to XML content representing the file
// in terms of the structure of an XLSX file.
func (f *File) MarshallParts() (map[string]string, error) {
	var parts map[string]string
	var refTable *RefTable = NewSharedStringRefTable()
	refTable.isWrite = true
	var workbookRels WorkBookRels = make(WorkBookRels)
	var err error
	var workbook xlsxWorkbook
	var types xlsxTypes = MakeDefaultContentTypes()

	marshal := func(thing interface{}) (string, error) {
		body, err := xml.Marshal(thing)
		if err != nil {
			return "", err
		}
		return xml.Header + string(body), nil
	}

	parts = make(map[string]string)
	workbook = f.makeWorkbook()
	sheetIndex := 1

	if f.styles == nil {
		f.styles = newXlsxStyleSheet(f.theme)
	}
	f.styles.reset()
	for _, sheet := range f.Sheets {
		xSheet := sheet.makeXLSXSheet(refTable, f.styles)
		rId := fmt.Sprintf("rId%d", sheetIndex)
		sheetId := strconv.Itoa(sheetIndex)
		sheetPath := fmt.Sprintf("worksheets/sheet%d.xml", sheetIndex)
		partName := "xl/" + sheetPath
		types.Overrides = append(
			types.Overrides,
			xlsxOverride{
				PartName:    "/" + partName,
				ContentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"})
		workbookRels[rId] = sheetPath
		workbook.Sheets.Sheet[sheetIndex-1] = xlsxSheet{
			Name:    sheet.Name,
			SheetId: sheetId,
			Id:      rId,
			State:   "visible"}
		parts[partName], err = marshal(xSheet)
		if err != nil {
			return parts, err
		}
		sheetIndex++
	}

	workbookMarshal, err := marshal(workbook)
	if err != nil {
		return parts, err
	}
	workbookMarshal = replaceRelationshipsNameSpace(workbookMarshal)
	parts["xl/workbook.xml"] = workbookMarshal
	if err != nil {
		return parts, err
	}

	parts["_rels/.rels"] = TEMPLATE__RELS_DOT_RELS
	parts["docProps/app.xml"] = TEMPLATE_DOCPROPS_APP
	// TODO - do this properly, modification and revision information
	parts["docProps/core.xml"] = TEMPLATE_DOCPROPS_CORE
	parts["xl/theme/theme1.xml"] = TEMPLATE_XL_THEME_THEME

	xSST := refTable.makeXLSXSST()
	parts["xl/sharedStrings.xml"], err = marshal(xSST)
	if err != nil {
		return parts, err
	}

	xWRel := workbookRels.MakeXLSXWorkbookRels()

	parts["xl/_rels/workbook.xml.rels"], err = marshal(xWRel)
	if err != nil {
		return parts, err
	}

	parts["[Content_Types].xml"], err = marshal(types)
	if err != nil {
		return parts, err
	}

	parts["xl/styles.xml"], err = f.styles.Marshal()
	if err != nil {
		return parts, err
	}

	return parts, nil
}

// Return the raw data contained in the File as three
// dimensional slice.  The first index represents the sheet number,
// the second the row number, and the third the cell number.
//
// For example:
//
//    var mySlice [][][]string
//    var value string
//    mySlice = xlsx.FileToSlice("myXLSX.xlsx")
//    value = mySlice[0][0][0]
//
// Here, value would be set to the raw value of the cell A1 in the
// first sheet in the XLSX file.
func (file *File) ToSlice() (output [][][]string, err error) {
	output = [][][]string{}
	for _, sheet := range file.Sheets {
		s := [][]string{}
		for _, row := range sheet.Rows {
			if row == nil {
				continue
			}
			r := []string{}
			for _, cell := range row.Cells {
				r = append(r, cell.String())
			}
			s = append(s, r)
		}
		output = append(output, s)
	}
	return output, nil
}
<commit_msg>add OpenBinary & OpenReaderAt<commit_after>package xlsx

import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
)

// File is a high level structure providing a slice of Sheet structs
// to the user.
type File struct {
	worksheets     map[string]*zip.File
	referenceTable *RefTable
	Date1904       bool
	styles         *xlsxStyleSheet
	Sheets         []*Sheet
	Sheet          map[string]*Sheet
	theme          *theme
}

// Create a new File
func NewFile() (file *File) {
	file = &File{}
	file.Sheet = make(map[string]*Sheet)
	file.Sheets = make([]*Sheet, 0)
	return
}

// OpenFile() take the name of an XLSX file and returns a populated
// xlsx.File struct for it.
func OpenFile(filename string) (file *File, err error) {
	var f *zip.ReadCloser
	f, err = zip.OpenReader(filename)
	if err != nil {
		return nil, err
	}
	file, err = ReadZip(f)
	return
}

// OpenBinary() take bytes of an XLSX file and returns a populated
// xlsx.File struct for it.
func OpenBinary(bs []byte) (file *File, err error) {
	r := bytes.NewReader(bs)
	file, err = OpenReaderAt(r, int64(r.Len()))
	return
}

// OpenReaderAt() take io.ReaderAt of an XLSX file and returns a populated
// xlsx.File struct for it.
func OpenReaderAt(r io.ReaderAt, size int64) (file *File, err error) {
	var f *zip.Reader
	f, err = zip.NewReader(r, size)
	if err != nil {
		return nil, err
	}
	file, err = ReadZipReader(f)
	return
}

// A convenient wrapper around File.ToSlice, FileToSlice will
// return the raw data contained in an Excel XLSX file as three
// dimensional slice.  The first index represents the sheet number,
// the second the row number, and the third the cell number.
//
// For example:
//
//    var mySlice [][][]string
//    var value string
//    mySlice = xlsx.FileToSlice("myXLSX.xlsx")
//    value = mySlice[0][0][0]
//
// Here, value would be set to the raw value of the cell A1 in the
// first sheet in the XLSX file.
func FileToSlice(path string) ([][][]string, error) {
	f, err := OpenFile(path)
	if err != nil {
		return nil, err
	}
	return f.ToSlice()
}

// Save the File to an xlsx file at the provided path.
func (f *File) Save(path string) (err error) {
	var target *os.File

	target, err = os.Create(path)
	if err != nil {
		return
	}

	err = f.Write(target)
	if err != nil {
		return
	}

	return target.Close()
}

// Write the File to io.Writer as xlsx
func (f *File) Write(writer io.Writer) (err error) {
	var parts map[string]string
	var zipWriter *zip.Writer

	parts, err = f.MarshallParts()
	if err != nil {
		return
	}

	zipWriter = zip.NewWriter(writer)

	for partName, part := range parts {
		var writer io.Writer
		writer, err = zipWriter.Create(partName)
		if err != nil {
			return
		}
		_, err = writer.Write([]byte(part))
		if err != nil {
			return
		}
	}

	err = zipWriter.Close()

	return
}

// Add a new Sheet, with the provided name, to a File
func (f *File) AddSheet(sheetName string) (sheet *Sheet) {
	sheet = &Sheet{Name: sheetName, File: f}
	if len(f.Sheets) == 0 {
		sheet.Selected = true
	}
	f.Sheet[sheetName] = sheet
	f.Sheets = append(f.Sheets, sheet)
	return sheet
}

func (f *File) makeWorkbook() xlsxWorkbook {
	var workbook xlsxWorkbook
	workbook = xlsxWorkbook{}
	workbook.FileVersion = xlsxFileVersion{}
	workbook.FileVersion.AppName = "Go XLSX"
	workbook.WorkbookPr = xlsxWorkbookPr{
		BackupFile:  false,
		ShowObjects: "all"}
	workbook.BookViews = xlsxBookViews{}
	workbook.BookViews.WorkBookView = make([]xlsxWorkBookView, 1)
	workbook.BookViews.WorkBookView[0] = xlsxWorkBookView{
		ActiveTab:            0,
		FirstSheet:           0,
		ShowHorizontalScroll: true,
		ShowSheetTabs:        true,
		ShowVerticalScroll:   true,
		TabRatio:             204,
		WindowHeight:         8192,
		WindowWidth:          16384,
		XWindow:              "0",
		YWindow:              "0"}
	workbook.Sheets = xlsxSheets{}
	workbook.Sheets.Sheet = make([]xlsxSheet, len(f.Sheets))
	workbook.CalcPr.IterateCount = 100
	workbook.CalcPr.RefMode = "A1"
	workbook.CalcPr.Iterate = false
	workbook.CalcPr.IterateDelta = 0.001
	return workbook
}

// Some tools that read XLSX files have very strict requirements about
// the structure of the input XML.  In particular both Numbers on the Mac
// and SAS dislike inline XML namespace declarations, or namespace
// prefixes that don't match the ones that Excel itself uses.  This is a
// problem because the Go XML library doesn't multiple namespace
// declarations in a single element of a document.  This function is a
// horrible hack to fix that after the XML marshalling is completed.
func replaceRelationshipsNameSpace(workbookMarshal string) string {
	newWorkbook := strings.Replace(workbookMarshal, `xmlns:relationships="http://schemas.openxmlformats.org/officeDocument/2006/relationships" relationships:id`, `r:id`, -1)
	// Dirty hack to fix issues #63 and #91; encoding/xml currently
	// "doesn't allow for additional namespaces to be defined in the
	// root element of the document," as described by @tealeg in the
	// comments for #63.
	oldXmlns := `<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">`
	newXmlns := `<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">`
	return strings.Replace(newWorkbook, oldXmlns, newXmlns, 1)
}

// Construct a map of file name to XML content representing the file
// in terms of the structure of an XLSX file.
func (f *File) MarshallParts() (map[string]string, error) {
	var parts map[string]string
	var refTable *RefTable = NewSharedStringRefTable()
	refTable.isWrite = true
	var workbookRels WorkBookRels = make(WorkBookRels)
	var err error
	var workbook xlsxWorkbook
	var types xlsxTypes = MakeDefaultContentTypes()

	marshal := func(thing interface{}) (string, error) {
		body, err := xml.Marshal(thing)
		if err != nil {
			return "", err
		}
		return xml.Header + string(body), nil
	}

	parts = make(map[string]string)
	workbook = f.makeWorkbook()
	sheetIndex := 1

	if f.styles == nil {
		f.styles = newXlsxStyleSheet(f.theme)
	}
	f.styles.reset()
	for _, sheet := range f.Sheets {
		xSheet := sheet.makeXLSXSheet(refTable, f.styles)
		rId := fmt.Sprintf("rId%d", sheetIndex)
		sheetId := strconv.Itoa(sheetIndex)
		sheetPath := fmt.Sprintf("worksheets/sheet%d.xml", sheetIndex)
		partName := "xl/" + sheetPath
		types.Overrides = append(
			types.Overrides,
			xlsxOverride{
				PartName:    "/" + partName,
				ContentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"})
		workbookRels[rId] = sheetPath
		workbook.Sheets.Sheet[sheetIndex-1] = xlsxSheet{
			Name:    sheet.Name,
			SheetId: sheetId,
			Id:      rId,
			State:   "visible"}
		parts[partName], err = marshal(xSheet)
		if err != nil {
			return parts, err
		}
		sheetIndex++
	}

	workbookMarshal, err := marshal(workbook)
	if err != nil {
		return parts, err
	}
	workbookMarshal = replaceRelationshipsNameSpace(workbookMarshal)
	parts["xl/workbook.xml"] = workbookMarshal
	if err != nil {
		return parts, err
	}

	parts["_rels/.rels"] = TEMPLATE__RELS_DOT_RELS
	parts["docProps/app.xml"] = TEMPLATE_DOCPROPS_APP
	// TODO - do this properly, modification and revision information
	parts["docProps/core.xml"] = TEMPLATE_DOCPROPS_CORE
	parts["xl/theme/theme1.xml"] = TEMPLATE_XL_THEME_THEME

	xSST := refTable.makeXLSXSST()
	parts["xl/sharedStrings.xml"], err = marshal(xSST)
	if err != nil {
		return parts, err
	}

	xWRel := workbookRels.MakeXLSXWorkbookRels()

	parts["xl/_rels/workbook.xml.rels"], err = marshal(xWRel)
	if err != nil {
		return parts, err
	}

	parts["[Content_Types].xml"], err = marshal(types)
	if err != nil {
		return parts, err
	}

	parts["xl/styles.xml"], err = f.styles.Marshal()
	if err != nil {
		return parts, err
	}

	return parts, nil
}

// Return the raw data contained in the File as three
// dimensional slice.  The first index represents the sheet number,
// the second the row number, and the third the cell number.
//
// For example:
//
//    var mySlice [][][]string
//    var value string
//    mySlice = xlsx.FileToSlice("myXLSX.xlsx")
//    value = mySlice[0][0][0]
//
// Here, value would be set to the raw value of the cell A1 in the
// first sheet in the XLSX file.
func (file *File) ToSlice() (output [][][]string, err error) {
	output = [][][]string{}
	for _, sheet := range file.Sheets {
		s := [][]string{}
		for _, row := range sheet.Rows {
			if row == nil {
				continue
			}
			r := []string{}
			for _, cell := range row.Cells {
				r = append(r, cell.String())
			}
			s = append(s, r)
		}
		output = append(output, s)
	}
	return output, nil
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package atom

import (
	"context"

	"github.com/google/gapid/core/data/protoconv"
	"github.com/google/gapid/core/log"
	"github.com/google/gapid/gapis/atom/atom_pb"
)

// ProtoToAtom returns a function that converts all the storage atoms it is
// handed, passing the generated live atoms to the handler.
// You must call this with a nil to flush the final atom.
func ProtoToAtom(handler func(a Atom)) func(context.Context, atom_pb.Atom) error {
	var (
		last         Atom
		observations *Observations
		invoked      bool
		count        int
	)
	return func(ctx context.Context, in atom_pb.Atom) error {
		count++
		if in == nil {
			if last != nil {
				handler(last)
			}
			last = nil
			return nil
		}
		out, err := protoconv.ToObject(ctx, in)
		if err != nil {
			return nil
		}
		switch out := out.(type) {
		case Atom:
			if last != nil {
				handler(last)
			}
			last = out
			invoked = false
			observations = nil
		case Observation:
			if observations == nil {
				observations = &Observations{}
				e := last.Extras()
				if e == nil {
					return log.Errf(ctx, nil, "Not allowed extras %T:%v", last, last)
				}
				*e = append(*e, observations)
			}
			if !invoked {
				observations.Reads = append(observations.Reads, out)
			} else {
				observations.Writes = append(observations.Writes, out)
			}
		case invokeMarker:
			invoked = true
		case Extra:
			e := last.Extras()
			if e == nil {
				return log.Errf(ctx, nil, "Not allowed extras %T:%v", last, last)
			}
			*e = append(*e, out)
		default:
			return log.Errf(ctx, nil, "Unhandled type during conversion %T:%v", out, out)
		}
		return nil
	}
}

// AtomToProto returns a function that converts all the atoms it is handed,
// passing the generated proto atoms to the handler.
func AtomToProto(handler func(a atom_pb.Atom)) func(context.Context, Atom) error {
	return func(ctx context.Context, in Atom) error {
		out, err := protoconv.ToProto(ctx, in)
		if err != nil {
			return err
		}
		handler(out)

		for _, e := range in.Extras().All() {
			switch e := e.(type) {
			case Observations:
				for _, o := range e.Reads {
					p, err := protoconv.ToProto(ctx, o)
					if err != nil {
						return nil
					}
					handler(p)
				}
				handler(atom_pb.InvokeMarker)
				for _, o := range e.Writes {
					p, err := protoconv.ToProto(ctx, o)
					if err != nil {
						return nil
					}
					handler(p)
				}
			default:
				p, err := protoconv.ToProto(ctx, e)
				if err != nil {
					return nil
				}
				handler(p)
			}
		}

		return nil
	}
}

type invokeMarker struct{}

func init() {
	protoconv.Register(
		func(ctx context.Context, a *invokeMarker) (*atom_pb.Invoke, error) {
			return &atom_pb.Invoke{}, nil
		},
		func(ctx context.Context, a *atom_pb.Invoke) (*invokeMarker, error) {
			return &invokeMarker{}, nil
		},
	)
}
<commit_msg>gapis/atom: Assign thread ids to deserialized atoms.<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package atom

import (
	"context"

	"github.com/google/gapid/core/data/protoconv"
	"github.com/google/gapid/core/log"
	"github.com/google/gapid/gapis/atom/atom_pb"
	"github.com/google/gapid/gapis/gfxapi/core/core_pb"
)

// ProtoToAtom returns a function that converts all the storage atoms it is
// handed, passing the generated live atoms to the handler.
// You must call this with a nil to flush the final atom.
func ProtoToAtom(handler func(a Atom)) func(context.Context, atom_pb.Atom) error {
	var (
		last         Atom
		observations *Observations
		invoked      bool
		count        int
	)
	var threadID uint64
	return func(ctx context.Context, in atom_pb.Atom) error {
		count++
		if in == nil {
			if last != nil {
				handler(last)
			}
			last = nil
			return nil
		}

		if in, ok := in.(*core_pb.SwitchThread); ok {
			threadID = in.ThreadID
		}

		out, err := protoconv.ToObject(ctx, in)
		if err != nil {
			return nil
		}
		switch out := out.(type) {
		case Atom:
			if last != nil {
				handler(last)
			}
			last = out
			invoked = false
			observations = nil
			out.SetThread(threadID)

		case Observation:
			if observations == nil {
				observations = &Observations{}
				e := last.Extras()
				if e == nil {
					return log.Errf(ctx, nil, "Not allowed extras %T:%v", last, last)
				}
				*e = append(*e, observations)
			}
			if !invoked {
				observations.Reads = append(observations.Reads, out)
			} else {
				observations.Writes = append(observations.Writes, out)
			}
		case invokeMarker:
			invoked = true
		case Extra:
			e := last.Extras()
			if e == nil {
				return log.Errf(ctx, nil, "Not allowed extras %T:%v", last, last)
			}
			*e = append(*e, out)
		default:
			return log.Errf(ctx, nil, "Unhandled type during conversion %T:%v", out, out)
		}
		return nil
	}
}

// AtomToProto returns a function that converts all the atoms it is handed,
// passing the generated proto atoms to the handler.
func AtomToProto(handler func(a atom_pb.Atom)) func(context.Context, Atom) error {
	return func(ctx context.Context, in Atom) error {
		out, err := protoconv.ToProto(ctx, in)
		if err != nil {
			return err
		}
		handler(out)

		for _, e := range in.Extras().All() {
			switch e := e.(type) {
			case Observations:
				for _, o := range e.Reads {
					p, err := protoconv.ToProto(ctx, o)
					if err != nil {
						return nil
					}
					handler(p)
				}
				handler(atom_pb.InvokeMarker)
				for _, o := range e.Writes {
					p, err := protoconv.ToProto(ctx, o)
					if err != nil {
						return nil
					}
					handler(p)
				}
			default:
				p, err := protoconv.ToProto(ctx, e)
				if err != nil {
					return nil
				}
				handler(p)
			}
		}

		return nil
	}
}

type invokeMarker struct{}

func init() {
	protoconv.Register(
		func(ctx context.Context, a *invokeMarker) (*atom_pb.Invoke, error) {
			return &atom_pb.Invoke{}, nil
		},
		func(ctx context.Context, a *atom_pb.Invoke) (*invokeMarker, error) {
			return &invokeMarker{}, nil
		},
	)
}
<|endoftext|>


<commit_before>package taggolib

import (
	"encoding/binary"
	"fmt"
	"io"
	"strconv"
	"strings"
	"time"

	"github.com/eaburns/bit"
)

const (
	// flacStreamInfo denotes a STREAMINFO metadata block
	flacStreamInfo = 0
	// flacVorbisComment denotes a VORBISCOMMENT metadata block
	flacVorbisComment = 4
)

var (
	// flacMagicNumber is the magic number used to identify a FLAC audio stream
	flacMagicNumber = []byte("fLaC")
)

// flacParser represents a FLAC audio metadata tag parser
type flacParser struct {
	encoder    string
	endPos     int64
	properties *flacStreamInfoBlock
	reader     io.ReadSeeker
	tags       map[string]string

	// Shared buffer stored as field to prevent unneeded allocations
	buffer []byte
}

// Album returns the Album tag for this stream
func (f flacParser) Album() string {
	return f.tags[tagAlbum]
}

// AlbumArtist returns the AlbumArtist tag for this stream
func (f flacParser) AlbumArtist() string {
	return f.tags[tagAlbumArtist]
}

// Artist returns the Artist tag for this stream
func (f flacParser) Artist() string {
	return f.tags[tagArtist]
}

// BitDepth returns the bits-per-sample of this stream
func (f flacParser) BitDepth() int {
	return int(f.properties.BitsPerSample)
}

// Bitrate calculates the audio bitrate for this stream
func (f flacParser) Bitrate() int {
	return int(((f.endPos * 8) / int64(f.Duration().Seconds())) / 1024)
}

// Channels returns the number of channels for this stream
func (f flacParser) Channels() int {
	return int(f.properties.ChannelCount)
}

// Checksum returns the checksum for this stream
func (f flacParser) Checksum() string {
	return f.properties.MD5Checksum
}

// Comment returns the Comment tag for this stream
func (f flacParser) Comment() string {
	return f.tags[tagComment]
}

// Date returns the Date tag for this stream
func (f flacParser) Date() string {
	return f.tags[tagDate]
}

// DiscNumber returns the DiscNumber tag for this stream
func (f flacParser) DiscNumber() int {
	disc, err := strconv.Atoi(f.tags[tagDiscNumber])
	if err != nil {
		return 0
	}

	return disc
}

// Duration returns the time duration for this stream
func (f flacParser) Duration() time.Duration {
	return time.Duration(int64(f.properties.SampleCount)/int64(f.SampleRate())) * time.Second
}

// Encoder returns the encoder for this stream
func (f flacParser) Encoder() string {
	return f.encoder
}

// Format returns the name of the FLAC format
func (f flacParser) Format() string {
	return "FLAC"
}

// Genre returns the Genre tag for this stream
func (f flacParser) Genre() string {
	return f.tags[tagGenre]
}

// SampleRate returns the sample rate in Hertz for this stream
func (f flacParser) SampleRate() int {
	return int(f.properties.SampleRate)
}

// Tag attempts to return the raw, unprocessed tag with the specified name for this stream
func (f flacParser) Tag(name string) string {
	return f.tags[strings.ToUpper(name)]
}

// Title returns the Title tag for this stream
func (f flacParser) Title() string {
	return f.tags[tagTitle]
}

// TrackNumber returns the TrackNumber tag for this stream
func (f flacParser) TrackNumber() int {
	track, err := strconv.Atoi(f.tags[tagTrackNumber])
	if err != nil {
		return 0
	}

	return track
}

// newFLACParser creates a parser for FLAC audio streams
func newFLACParser(reader io.ReadSeeker) (*flacParser, error) {
	// Create FLAC parser
	parser := &flacParser{
		buffer: make([]byte, 128),
		reader: reader,
	}

	// Begin parsing properties
	if err := parser.parseProperties(); err != nil {
		return nil, err
	}

	// Seek through the file and attempt to parse tags
	if err := parser.parseTags(); err != nil {
		return nil, err
	}

	// Seek to end of file to grab the final position, used to calculate bitrate
	n, err := parser.reader.Seek(0, 2)
	if err != nil {
		return nil, err
	}
	parser.endPos = n

	// Return parser
	return parser, nil
}

// flacMetadataHeader represents the header for a FLAC metadata block
type flacMetadataHeader struct {
	LastBlock   bool
	BlockType   uint8
	BlockLength uint32
}

// flacStreamInfoBlock represents the metadata from a FLAC STREAMINFO block
type flacStreamInfoBlock struct {
	SampleRate    uint16
	ChannelCount  uint8
	BitsPerSample uint16
	SampleCount   uint64
	MD5Checksum   string
}

// parseMetadataHeader retrieves metadata header information from a FLAC stream
func (f *flacParser) parseMetadataHeader() (*flacMetadataHeader, error) {
	// Create and use a bit reader to parse the following fields:
	//    1 - Last metadata block before audio (boolean)
	//    7 - Metadata block type (should be 0, for streaminfo)
	//   24 - Length of following metadata (in bytes)
	fields, err := bit.NewReader(f.reader).ReadFields(1, 7, 24)
	if err != nil {
		return nil, err
	}

	// Generate metadata header
	return &flacMetadataHeader{
		LastBlock:   fields[0] == 1,
		BlockType:   uint8(fields[1]),
		BlockLength: uint32(fields[2]),
	}, nil
}

// parseTags retrieves metadata tags from a FLAC VORBISCOMMENT block
func (f *flacParser) parseTags() error {
	// Continuously parse and seek through blocks until we discover the VORBISCOMMENT block
	for {
		header, err := f.parseMetadataHeader()
		if err != nil {
			return err
		}

		// Check for VORBISCOMMENT block, break so we can begin parsing tags
		if header.BlockType == flacVorbisComment {
			break
		}

		// If last block and no VORBISCOMMENT block found, no tags
		if header.LastBlock {
			return nil
		}

		// If nothing found and not last block, seek forward in stream
		if _, err := f.reader.Seek(int64(header.BlockLength), 1); err != nil {
			return err
		}
	}

	// Parse length fields
	var length uint32

	// Read vendor string length
	if err := binary.Read(f.reader, binary.LittleEndian, &length); err != nil {
		return err
	}

	// Read vendor string
	if _, err := f.reader.Read(f.buffer[:length]); err != nil {
		return err
	}
	f.encoder = string(f.buffer[:length])

	// Read comment length (new allocation so we can use it as loop counter)
	var commentLength uint32
	if err := binary.Read(f.reader, binary.LittleEndian, &commentLength); err != nil {
		return err
	}

	// Begin iterating tags, and building tag map
	tagMap := map[string]string{}
	for i := 0; i < int(commentLength); i++ {
		// Read tag string length
		if err := binary.Read(f.reader, binary.LittleEndian, &length); err != nil {
			return err
		}

		// Read tag string
		n, err := f.reader.Read(f.buffer[:length])
		if err != nil {
			return err
		}

		// Split tag name and data, store in map
		pair := strings.Split(string(f.buffer[:n]), "=")
		tagMap[strings.ToUpper(pair[0])] = pair[1]
	}

	// Store tags
	f.tags = tagMap
	return nil
}

// parseProperties retrieves stream properties from a FLAC STREAMINFO block
func (f *flacParser) parseProperties() error {
	// Read the metadata header for STREAMINFO block
	header, err := f.parseMetadataHeader()
	if err != nil {
		return err
	}

	// Ensure that the metadata block type is STREAMINFO
	if header.BlockType != flacStreamInfo {
		return TagError{
			Err:     errInvalidStream,
			Format:  f.Format(),
			Details: "first metadata block is not type STREAMINFO",
		}
	}

	// Ensure that STREAMINFO is not the last block
	if header.LastBlock {
		return TagError{
			Err:     errInvalidStream,
			Format:  f.Format(),
			Details: "STREAMINFO block is marked as last metadata block in stream",
		}
	}

	// Seek forward past frame information, to sample rate
	if _, err := f.reader.Seek(10, 1); err != nil {
		return err
	}

	// Create and use a bit reader to parse the following fields:
	//   20 - Sample rate
	//    3 - Channel count (+1)
	//    5 - Bits per sample (+1)
	//   36 - Sample count
	fields, err := bit.NewReader(f.reader).ReadFields(20, 3, 5, 36)
	if err != nil {
		return err
	}

	// Read the MD5 checksum of the stream
	if _, err := f.reader.Read(f.buffer[:16]); err != nil {
		return err
	}

	// Store properties
	f.properties = &flacStreamInfoBlock{
		SampleRate:    uint16(fields[0]),
		ChannelCount:  uint8(fields[1]) + 1,
		BitsPerSample: uint16(fields[2]) + 1,
		SampleCount:   uint64(fields[3]),
		MD5Checksum:   fmt.Sprintf("%x", f.buffer[:16]),
	}

	return nil
}
<commit_msg>FLAC: increase buffer length to prevent panics<commit_after>package taggolib

import (
	"encoding/binary"
	"fmt"
	"io"
	"strconv"
	"strings"
	"time"

	"github.com/eaburns/bit"
)

const (
	// flacStreamInfo denotes a STREAMINFO metadata block
	flacStreamInfo = 0
	// flacVorbisComment denotes a VORBISCOMMENT metadata block
	flacVorbisComment = 4
)

var (
	// flacMagicNumber is the magic number used to identify a FLAC audio stream
	flacMagicNumber = []byte("fLaC")
)

// flacParser represents a FLAC audio metadata tag parser
type flacParser struct {
	encoder    string
	endPos     int64
	properties *flacStreamInfoBlock
	reader     io.ReadSeeker
	tags       map[string]string

	// Shared buffer stored as field to prevent unneeded allocations
	buffer []byte
}

// Album returns the Album tag for this stream
func (f flacParser) Album() string {
	return f.tags[tagAlbum]
}

// AlbumArtist returns the AlbumArtist tag for this stream
func (f flacParser) AlbumArtist() string {
	return f.tags[tagAlbumArtist]
}

// Artist returns the Artist tag for this stream
func (f flacParser) Artist() string {
	return f.tags[tagArtist]
}

// BitDepth returns the bits-per-sample of this stream
func (f flacParser) BitDepth() int {
	return int(f.properties.BitsPerSample)
}

// Bitrate calculates the audio bitrate for this stream
func (f flacParser) Bitrate() int {
	return int(((f.endPos * 8) / int64(f.Duration().Seconds())) / 1024)
}

// Channels returns the number of channels for this stream
func (f flacParser) Channels() int {
	return int(f.properties.ChannelCount)
}

// Checksum returns the checksum for this stream
func (f flacParser) Checksum() string {
	return f.properties.MD5Checksum
}

// Comment returns the Comment tag for this stream
func (f flacParser) Comment() string {
	return f.tags[tagComment]
}

// Date returns the Date tag for this stream
func (f flacParser) Date() string {
	return f.tags[tagDate]
}

// DiscNumber returns the DiscNumber tag for this stream
func (f flacParser) DiscNumber() int {
	disc, err := strconv.Atoi(f.tags[tagDiscNumber])
	if err != nil {
		return 0
	}

	return disc
}

// Duration returns the time duration for this stream
func (f flacParser) Duration() time.Duration {
	return time.Duration(int64(f.properties.SampleCount)/int64(f.SampleRate())) * time.Second
}

// Encoder returns the encoder for this stream
func (f flacParser) Encoder() string {
	return f.encoder
}

// Format returns the name of the FLAC format
func (f flacParser) Format() string {
	return "FLAC"
}

// Genre returns the Genre tag for this stream
func (f flacParser) Genre() string {
	return f.tags[tagGenre]
}

// SampleRate returns the sample rate in Hertz for this stream
func (f flacParser) SampleRate() int {
	return int(f.properties.SampleRate)
}

// Tag attempts to return the raw, unprocessed tag with the specified name for this stream
func (f flacParser) Tag(name string) string {
	return f.tags[strings.ToUpper(name)]
}

// Title returns the Title tag for this stream
func (f flacParser) Title() string {
	return f.tags[tagTitle]
}

// TrackNumber returns the TrackNumber tag for this stream
func (f flacParser) TrackNumber() int {
	track, err := strconv.Atoi(f.tags[tagTrackNumber])
	if err != nil {
		return 0
	}

	return track
}

// newFLACParser creates a parser for FLAC audio streams
func newFLACParser(reader io.ReadSeeker) (*flacParser, error) {
	// Create FLAC parser
	parser := &flacParser{
		buffer: make([]byte, 256),
		reader: reader,
	}

	// Begin parsing properties
	if err := parser.parseProperties(); err != nil {
		return nil, err
	}

	// Seek through the file and attempt to parse tags
	if err := parser.parseTags(); err != nil {
		return nil, err
	}

	// Seek to end of file to grab the final position, used to calculate bitrate
	n, err := parser.reader.Seek(0, 2)
	if err != nil {
		return nil, err
	}
	parser.endPos = n

	// Return parser
	return parser, nil
}

// flacMetadataHeader represents the header for a FLAC metadata block
type flacMetadataHeader struct {
	LastBlock   bool
	BlockType   uint8
	BlockLength uint32
}

// flacStreamInfoBlock represents the metadata from a FLAC STREAMINFO block
type flacStreamInfoBlock struct {
	SampleRate    uint16
	ChannelCount  uint8
	BitsPerSample uint16
	SampleCount   uint64
	MD5Checksum   string
}

// parseMetadataHeader retrieves metadata header information from a FLAC stream
func (f *flacParser) parseMetadataHeader() (*flacMetadataHeader, error) {
	// Create and use a bit reader to parse the following fields:
	//    1 - Last metadata block before audio (boolean)
	//    7 - Metadata block type (should be 0, for streaminfo)
	//   24 - Length of following metadata (in bytes)
	fields, err := bit.NewReader(f.reader).ReadFields(1, 7, 24)
	if err != nil {
		return nil, err
	}

	// Generate metadata header
	return &flacMetadataHeader{
		LastBlock:   fields[0] == 1,
		BlockType:   uint8(fields[1]),
		BlockLength: uint32(fields[2]),
	}, nil
}

// parseTags retrieves metadata tags from a FLAC VORBISCOMMENT block
func (f *flacParser) parseTags() error {
	// Continuously parse and seek through blocks until we discover the VORBISCOMMENT block
	for {
		header, err := f.parseMetadataHeader()
		if err != nil {
			return err
		}

		// Check for VORBISCOMMENT block, break so we can begin parsing tags
		if header.BlockType == flacVorbisComment {
			break
		}

		// If last block and no VORBISCOMMENT block found, no tags
		if header.LastBlock {
			return nil
		}

		// If nothing found and not last block, seek forward in stream
		if _, err := f.reader.Seek(int64(header.BlockLength), 1); err != nil {
			return err
		}
	}

	// Parse length fields
	var length uint32

	// Read vendor string length
	if err := binary.Read(f.reader, binary.LittleEndian, &length); err != nil {
		return err
	}

	// Read vendor string
	if _, err := f.reader.Read(f.buffer[:length]); err != nil {
		return err
	}
	f.encoder = string(f.buffer[:length])

	// Read comment length (new allocation so we can use it as loop counter)
	var commentLength uint32
	if err := binary.Read(f.reader, binary.LittleEndian, &commentLength); err != nil {
		return err
	}

	// Begin iterating tags, and building tag map
	tagMap := map[string]string{}
	for i := 0; i < int(commentLength); i++ {
		// Read tag string length
		if err := binary.Read(f.reader, binary.LittleEndian, &length); err != nil {
			return err
		}

		// Read tag string
		n, err := f.reader.Read(f.buffer[:length])
		if err != nil {
			return err
		}

		// Split tag name and data, store in map
		pair := strings.Split(string(f.buffer[:n]), "=")
		tagMap[strings.ToUpper(pair[0])] = pair[1]
	}

	// Store tags
	f.tags = tagMap
	return nil
}

// parseProperties retrieves stream properties from a FLAC STREAMINFO block
func (f *flacParser) parseProperties() error {
	// Read the metadata header for STREAMINFO block
	header, err := f.parseMetadataHeader()
	if err != nil {
		return err
	}

	// Ensure that the metadata block type is STREAMINFO
	if header.BlockType != flacStreamInfo {
		return TagError{
			Err:     errInvalidStream,
			Format:  f.Format(),
			Details: "first metadata block is not type STREAMINFO",
		}
	}

	// Ensure that STREAMINFO is not the last block
	if header.LastBlock {
		return TagError{
			Err:     errInvalidStream,
			Format:  f.Format(),
			Details: "STREAMINFO block is marked as last metadata block in stream",
		}
	}

	// Seek forward past frame information, to sample rate
	if _, err := f.reader.Seek(10, 1); err != nil {
		return err
	}

	// Create and use a bit reader to parse the following fields:
	//   20 - Sample rate
	//    3 - Channel count (+1)
	//    5 - Bits per sample (+1)
	//   36 - Sample count
	fields, err := bit.NewReader(f.reader).ReadFields(20, 3, 5, 36)
	if err != nil {
		return err
	}

	// Read the MD5 checksum of the stream
	if _, err := f.reader.Read(f.buffer[:16]); err != nil {
		return err
	}

	// Store properties
	f.properties = &flacStreamInfoBlock{
		SampleRate:    uint16(fields[0]),
		ChannelCount:  uint8(fields[1]) + 1,
		BitsPerSample: uint16(fields[2]) + 1,
		SampleCount:   uint64(fields[3]),
		MD5Checksum:   fmt.Sprintf("%x", f.buffer[:16]),
	}

	return nil
}
<|endoftext|>


<commit_before>package gitManip

import (
	"fmt"

	"github.com/fatih/color"

	git "gopkg.in/libgit2/git2go.v25"
)

/*Map to match the RepositoryState enum type with a string
 */
var repositoryStateToString = map[git.RepositoryState]string{
	git.RepositoryStateNone:                 "None",
	git.RepositoryStateMerge:                "Merge",
	git.RepositoryStateRevert:               "Revert",
	git.RepositoryStateCherrypick:           "Cherrypick",
	git.RepositoryStateBisect:               "Bisect",
	git.RepositoryStateRebase:               "Rebase",
	git.RepositoryStateRebaseInteractive:    "Rebase Interactive",
	git.RepositoryStateRebaseMerge:          "Rebase Merge",
	git.RepositoryStateApplyMailbox:         "Apply Mailbox",
	git.RepositoryStateApplyMailboxOrRebase: "Apply Mailbox or Rebase",
}

var fileStateToString = map[git.Status]string{
	git.StatusIndexNew:     "You forgot to commit a new file!",
	git.StatusIgnored:      "Ignored",
	git.StatusConflicted:   "Conflicted",
	git.StatusWtNew:        "New file in your working tree!",
	git.StatusWtModified:   "Modified file in your working tree!",
	git.StatusWtDeleted:    "Deleted file in your working tree!",
	git.StatusWtTypeChange: "Type change detected in your working tree!",
	git.StatusWtRenamed:    "Renamed file in your working tree!",
}

/*Global variable to set the StatusOption parameter, in order to list each file status
 */
var statusOption = git.StatusOptions{
	Show:     git.StatusShowIndexAndWorkdir,
	Flags:    git.StatusOptIncludeUntracked,
	Pathspec: []string{},
}

/*GitObject contains informations about the current git repository
 *
 *The structure is:
 *  accessible:
 *		Is the repository still exists in the hard drive?
 *	path:
 *		The path file.
 *	repository:
 *		The object repository.
 */
type GitObject struct {
	accessible error
	path       string
	repository git.Repository
}

/*New is a constructor for GitObject
 *
 * It neeeds:
 *	path:
 *		The path of the current repository.
 */
func New(path string) *GitObject {
	r, err := git.OpenRepository(path)
	return &GitObject{accessible: err, path: path, repository: *r}
}

/*isAccesible returns the information that is the current git repository is existing or not.
 *This method returns a boolean value: true if the git repository is still accesible (still exists), or false if not.
 */
func (g *GitObject) isAccessible() bool {
	return g.accessible == nil
}

/*Status prints the current status of the repository, accessible via the structure path field.
 *This method works only if the repository is accessible.
 */
func (g *GitObject) Status() {
	if g.isAccessible() {
		if err := g.printChanges(); err != nil {
			color.RedString("Impossible to get stats from %s, due to error %s", g.path, err)
		}
	} else {
		color.RedString("Repository %s not found!", g.path)
	}
}

/*getDiffWithWT returns the difference between the working tree and the index, for the current git repository.
 *If there is an error processing the request, it returns an error.
 */
func (g *GitObject) getDiffWithWT() (*git.Diff, error) {
	// Get the index of the repository
	currentIndex, err := g.repository.Index()
	if err != nil {
		return nil, err
	}
	// Get the default diff options, and add it custom flags
	defaultDiffOptions, err := git.DefaultDiffOptions()
	if err != nil {
		return nil, err
	}
	defaultDiffOptions.Flags = defaultDiffOptions.Flags | git.DiffIncludeUntracked | git.DiffIncludeTypeChange
	// Check the difference between the working directory and the index
	diff, err := g.repository.DiffIndexToWorkdir(currentIndex, &defaultDiffOptions)
	if err != nil {
		return nil, err
	}
	return diff, nil
}

/*printChanges prints out all changes for the current git repository.
 *If there is an error processing the request, it returns this one.
 */
func (g *GitObject) printChanges() error {
	diff, err := g.getDiffWithWT()
	if err != nil {
		return err
	}
	numDeltas, err := diff.NumDeltas()
	if err != nil {
		return err
	}
	if numDeltas > 0 {
		fmt.Printf("%s %s\t[%d modification(s)]\n", color.RedString("✘"), g.path, numDeltas)
		for i := 0; i < numDeltas; i++ {
			delta, _ := diff.GetDelta(i)
			currentStatus := delta.Status
			newFile := delta.NewFile.Path
			oldFile := delta.OldFile.Path
			switch currentStatus {
			case git.DeltaAdded:
				fmt.Printf("\t===> %s has been added!\n", color.MagentaString(newFile))
			case git.DeltaDeleted:
				fmt.Printf("\t===> %s has been deleted!\n", color.MagentaString(newFile))
			case git.DeltaModified:
				fmt.Printf("\t===> %s has been modified!\n", color.MagentaString(newFile))
			case git.DeltaRenamed:
				fmt.Printf("\t===> %s has been renamed to %s!\n", color.MagentaString(oldFile), color.MagentaString(newFile))
			case git.DeltaUntracked:
				fmt.Printf("\t===> %s is untracked - please to add it or update the gitignore file!\n", color.MagentaString(newFile))
			case git.DeltaTypeChange:
				fmt.Printf("\t===> the type of %s has been changed from %d to %d!", color.MagentaString(newFile), delta.OldFile.Mode, delta.NewFile.Mode)
			}
		}
	} else {
		fmt.Printf("%s %s\n", color.GreenString("✔"), g.path)
	}
	return nil
}

/*List lists the path and the accessibility of a list of git repositories
 */
func List(repositories *[]GitObject) {
	for _, object := range *repositories {
		fmt.Printf("* %s ", object.path)
		if object.isAccessible() {
			fmt.Println(color.GreenString(" [accessible]"))
		} else {
			fmt.Println(color.RedString(" [not accessible]"))
		}
	}
}
<commit_msg>Remove useless code<commit_after>package gitManip

import (
	"fmt"

	"github.com/fatih/color"

	git "gopkg.in/libgit2/git2go.v25"
)

/*Map to match the RepositoryState enum type with a string
 */
var repositoryStateToString = map[git.RepositoryState]string{
	git.RepositoryStateNone:                 "None",
	git.RepositoryStateMerge:                "Merge",
	git.RepositoryStateRevert:               "Revert",
	git.RepositoryStateCherrypick:           "Cherrypick",
	git.RepositoryStateBisect:               "Bisect",
	git.RepositoryStateRebase:               "Rebase",
	git.RepositoryStateRebaseInteractive:    "Rebase Interactive",
	git.RepositoryStateRebaseMerge:          "Rebase Merge",
	git.RepositoryStateApplyMailbox:         "Apply Mailbox",
	git.RepositoryStateApplyMailboxOrRebase: "Apply Mailbox or Rebase",
}

/*Global variable to set the StatusOption parameter, in order to list each file status
 */
var statusOption = git.StatusOptions{
	Show:     git.StatusShowIndexAndWorkdir,
	Flags:    git.StatusOptIncludeUntracked,
	Pathspec: []string{},
}

/*GitObject contains informations about the current git repository
 *
 *The structure is:
 *  accessible:
 *		Is the repository still exists in the hard drive?
 *	path:
 *		The path file.
 *	repository:
 *		The object repository.
 */
type GitObject struct {
	accessible error
	path       string
	repository git.Repository
}

/*New is a constructor for GitObject
 *
 * It neeeds:
 *	path:
 *		The path of the current repository.
 */
func New(path string) *GitObject {
	r, err := git.OpenRepository(path)
	return &GitObject{accessible: err, path: path, repository: *r}
}

/*isAccesible returns the information that is the current git repository is existing or not.
 *This method returns a boolean value: true if the git repository is still accesible (still exists), or false if not.
 */
func (g *GitObject) isAccessible() bool {
	return g.accessible == nil
}

/*Status prints the current status of the repository, accessible via the structure path field.
 *This method works only if the repository is accessible.
 */
func (g *GitObject) Status() {
	if g.isAccessible() {
		if err := g.printChanges(); err != nil {
			color.RedString("Impossible to get stats from %s, due to error %s", g.path, err)
		}
	} else {
		color.RedString("Repository %s not found!", g.path)
	}
}

/*getDiffWithWT returns the difference between the working tree and the index, for the current git repository.
 *If there is an error processing the request, it returns an error.
 */
func (g *GitObject) getDiffWithWT() (*git.Diff, error) {
	// Get the index of the repository
	currentIndex, err := g.repository.Index()
	if err != nil {
		return nil, err
	}
	// Get the default diff options, and add it custom flags
	defaultDiffOptions, err := git.DefaultDiffOptions()
	if err != nil {
		return nil, err
	}
	defaultDiffOptions.Flags = defaultDiffOptions.Flags | git.DiffIncludeUntracked | git.DiffIncludeTypeChange
	// Check the difference between the working directory and the index
	diff, err := g.repository.DiffIndexToWorkdir(currentIndex, &defaultDiffOptions)
	if err != nil {
		return nil, err
	}
	return diff, nil
}

/*printChanges prints out all changes for the current git repository.
 *If there is an error processing the request, it returns this one.
 */
func (g *GitObject) printChanges() error {
	diff, err := g.getDiffWithWT()
	if err != nil {
		return err
	}
	numDeltas, err := diff.NumDeltas()
	if err != nil {
		return err
	}
	if numDeltas > 0 {
		fmt.Printf("%s %s\t[%d modification(s)]\n", color.RedString("✘"), g.path, numDeltas)
		for i := 0; i < numDeltas; i++ {
			delta, _ := diff.GetDelta(i)
			currentStatus := delta.Status
			newFile := delta.NewFile.Path
			oldFile := delta.OldFile.Path
			switch currentStatus {
			case git.DeltaAdded:
				fmt.Printf("\t===> %s has been added!\n", color.MagentaString(newFile))
			case git.DeltaDeleted:
				fmt.Printf("\t===> %s has been deleted!\n", color.MagentaString(newFile))
			case git.DeltaModified:
				fmt.Printf("\t===> %s has been modified!\n", color.MagentaString(newFile))
			case git.DeltaRenamed:
				fmt.Printf("\t===> %s has been renamed to %s!\n", color.MagentaString(oldFile), color.MagentaString(newFile))
			case git.DeltaUntracked:
				fmt.Printf("\t===> %s is untracked - please to add it or update the gitignore file!\n", color.MagentaString(newFile))
			case git.DeltaTypeChange:
				fmt.Printf("\t===> the type of %s has been changed from %d to %d!", color.MagentaString(newFile), delta.OldFile.Mode, delta.NewFile.Mode)
			}
		}
	} else {
		fmt.Printf("%s %s\n", color.GreenString("✔"), g.path)
	}
	return nil
}

/*List lists the path and the accessibility of a list of git repositories
 */
func List(repositories *[]GitObject) {
	for _, object := range *repositories {
		fmt.Printf("* %s ", object.path)
		if object.isAccessible() {
			fmt.Println(color.GreenString(" [accessible]"))
		} else {
			fmt.Println(color.RedString(" [not accessible]"))
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	debug                        bool
	verbose                      bool
	info                         bool
	quiet                        bool
	force                        bool
	usemove                      bool
	usecacheFallback             bool
	retryGitCommands             bool
	pfMode                       bool
	pfLocation                   string
	dryRun                       bool
	check4update                 bool
	checkSum                     bool
	gitObjectSyntaxNotSupported  bool
	moduleDirParam               string
	cacheDirParam                string
	branchParam                  string
	tags                         bool
	outputNameParam              string
	moduleParam                  string
	configFile                   string
	config                       ConfigSettings
	mutex                        sync.Mutex
	empty                        struct{}
	syncGitCount                 int
	syncForgeCount               int
	needSyncGitCount             int
	needSyncForgeCount           int
	syncGitTime                  float64
	syncForgeTime                float64
	ioGitTime                    float64
	ioForgeTime                  float64
	forgeJsonParseTime           float64
	metadataJsonParseTime        float64
	gmetadataJsonParseTime       float64
	buildtime                    string
	uniqueForgeModules           map[string]ForgeModule
	latestForgeModules           LatestForgeModules
	maxworker                    int
	maxExtractworker             int
	forgeModuleDeprecationNotice string
)

type LatestForgeModules struct {
	sync.RWMutex
	m map[string]string
}

// ConfigSettings contains the key value pairs from the g10k config file
type ConfigSettings struct {
	CacheDir                    string `yaml:"cachedir"`
	ForgeCacheDir               string
	ModulesCacheDir             string
	EnvCacheDir                 string
	Git                         Git
	Forge                       Forge
	Sources                     map[string]Source
	Timeout                     int  `yaml:"timeout"`
	IgnoreUnreachableModules    bool `yaml:"ignore_unreachable_modules"`
	Maxworker                   int  `yaml:"maxworker"`
	MaxExtractworker            int  `yaml:"maxextractworker"`
	UseCacheFallback            bool `yaml:"use_cache_fallback"`
	RetryGitCommands            bool `yaml:"retry_git_commands"`
	GitObjectSyntaxNotSupported bool `yaml:"git_object_syntax_not_supported"`
}

type Forge struct {
	Baseurl string `yaml:"baseurl"`
}

type Git struct {
	privateKey string `yaml:"private_key"`
	username   string
}

// Source contains basic information about a Puppet environment repository
type Source struct {
	Remote                      string
	Basedir                     string
	Prefix                      string
	PrivateKey                  string `yaml:"private_key"`
	ForceForgeVersions          bool   `yaml:"force_forge_versions"`
	WarnMissingBranch           bool   `yaml:"warn_if_branch_is_missing"`
	ExitIfUnreachable           bool   `yaml:"exit_if_unreachable"`
	AutoCorrectEnvironmentNames string `yaml:"invalid_branches"`
}

// Puppetfile contains the key value pairs from the Puppetfile
type Puppetfile struct {
	moduleDir     string
	forgeBaseURL  string
	forgeCacheTtl time.Duration
	forgeModules  map[string]ForgeModule
	gitModules    map[string]GitModule
	privateKey    string
	source        string
	workDir       string
}

// ForgeModule contains information (Version, Name, Author, md5 checksum, file size of the tar.gz archive, Forge BaseURL if custom) about a Puppetlabs Forge module
type ForgeModule struct {
	version   string
	name      string
	author    string
	md5sum    string
	fileSize  int64
	baseUrl   string
	cacheTtl  time.Duration
	sha256sum string
}

// GitModule contains information about a Git Puppet module
type GitModule struct {
	privateKey        string
	git               string
	branch            string
	tag               string
	commit            string
	ref               string
	link              bool
	ignoreUnreachable bool
	fallback          []string
	installPath       string
	local             bool
}

// ForgeResult is returned by queryForgeAPI and contains if and which version of the Puppetlabs Forge module needs to be downloaded
type ForgeResult struct {
	needToGet     bool
	versionNumber string
	md5sum        string
	fileSize      int64
}

// ExecResult contains the exit code and output of an external command (e.g. git)
type ExecResult struct {
	returnCode int
	output     string
}

func main() {

	var (
		configFileFlag = flag.String("config", "", "which config file to use")
		versionFlag    = flag.Bool("version", false, "show build time and version number")
	)
	flag.StringVar(&branchParam, "branch", "", "which git branch of the Puppet environment to update, e.g. core_foobar")
	flag.BoolVar(&tags, "tags", false, "to pull tags as well as branches")
	flag.StringVar(&outputNameParam, "outputname", "", "overwrite the environment name if -branch is specified")
	flag.StringVar(&moduleParam, "module", "", "which module of the Puppet environment to update, e.g. stdlib")
	flag.StringVar(&moduleDirParam, "moduledir", "", "allows overriding of Puppetfile specific moduledir setting, the folder in which Puppet modules will be extracted")
	flag.StringVar(&cacheDirParam, "cachedir", "", "allows overriding of the g10k config file cachedir setting, the folder in which g10k will download git repositories and Forge modules")
	flag.IntVar(&maxworker, "maxworker", 50, "how many Goroutines are allowed to run in parallel for Git and Forge module resolving")
	flag.IntVar(&maxExtractworker, "maxextractworker", 20, "how many Goroutines are allowed to run in parallel for local Git and Forge module extracting processes (git clone, untar and gunzip)")
	flag.BoolVar(&pfMode, "puppetfile", false, "install all modules from Puppetfile in cwd")
	flag.StringVar(&pfLocation, "puppetfilelocation", "./Puppetfile", "which Puppetfile to use in -puppetfile mode")
	flag.BoolVar(&force, "force", false, "purge the Puppet environment directory and do a full sync")
	flag.BoolVar(&dryRun, "dryrun", false, "do not modify anything, just print what would be changed")
	flag.BoolVar(&usemove, "usemove", false, "do not use hardlinks to populate your Puppet environments with Puppetlabs Forge modules. Instead uses simple move commands and purges the Forge cache directory after each run! (Useful for g10k runs inside a Docker container)")
	flag.BoolVar(&check4update, "check4update", false, "only check if the is newer version of the Puppet module avaialable. Does implicitly set dryrun to true")
	flag.BoolVar(&checkSum, "checksum", false, "get the md5 check sum for each Puppetlabs Forge module and verify the integrity of the downloaded archive. Increases g10k run time!")
	flag.BoolVar(&debug, "debug", false, "log debug output, defaults to false")
	flag.BoolVar(&verbose, "verbose", false, "log verbose output, defaults to false")
	flag.BoolVar(&info, "info", false, "log info output, defaults to false")
	flag.BoolVar(&quiet, "quiet", false, "no output, defaults to false")
	flag.BoolVar(&usecacheFallback, "usecachefallback", false, "if g10k should try to use its cache for sources and modules instead of failing")
	flag.BoolVar(&retryGitCommands, "retrygitcommands", false, "if g10k should purge the local repository and retry a failed git command (clone or remote update) instead of failing")
	flag.BoolVar(&gitObjectSyntaxNotSupported, "gitobjectsyntaxnotsupported", false, "if your git version is too old to support reference syntax like master^{object} use this setting to revert to the older syntax")
	flag.Parse()

	configFile = *configFileFlag
	version := *versionFlag

	if version {
		fmt.Println("g10k version 0.4.8 Build time:", buildtime, "UTC")
		os.Exit(0)
	}

	if check4update {
		dryRun = true
	}

	// check for git executable dependency
	if _, err := exec.LookPath("git"); err != nil {
		Fatalf("Error: could not find 'git' executable in PATH")
	}

	target := ""
	before := time.Now()
	if len(configFile) > 0 {
		if usemove {
			Fatalf("Error: -usemove parameter is only allowed in -puppetfile mode!")
		}
		if pfMode {
			Fatalf("Error: -puppetfile parameter is not allowed with -config parameter!")
		}
		if (len(outputNameParam) > 0) && (len(branchParam) == 0) {
			Fatalf("Error: -outputname specified without -branch!")
		}
		if usecacheFallback {
			config.UseCacheFallback = true
		}
		Debugf("Using as config file: " + configFile)
		config = readConfigfile(configFile)
		target = configFile
		if len(branchParam) > 0 {
			resolvePuppetEnvironment(branchParam, tags, outputNameParam)
			target += " with branch " + branchParam
		} else {
			resolvePuppetEnvironment("", tags, "")
		}
	} else {
		if pfMode {
			Debugf("Trying to use as Puppetfile: " + pfLocation)
			sm := make(map[string]Source)
			sm["cmdlineparam"] = Source{Basedir: "."}
			cachedir := "/tmp/g10k"
			if len(os.Getenv("g10k_cachedir")) > 0 {
				cachedir = os.Getenv("g10k_cachedir")
				cachedir = checkDirAndCreate(cachedir, "cachedir environment variable g10k_cachedir")
				Debugf("Found environment variable g10k_cachedir set to: " + cachedir)
			} else if len(cacheDirParam) > 0 {
				Debugf("Using -cachedir parameter set to : " + cacheDirParam)
				cachedir = checkDirAndCreate(cacheDirParam, "cachedir CLI param")
			} else {
				cachedir = checkDirAndCreate(cachedir, "cachedir default value")
			}
			forgeDefaultSettings := Forge{Baseurl: "https://forgeapi.puppetlabs.com"}
			config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: cachedir, EnvCacheDir: cachedir, Sources: sm, Forge: forgeDefaultSettings, Maxworker: maxworker, UseCacheFallback: usecacheFallback, MaxExtractworker: maxExtractworker, RetryGitCommands: retryGitCommands, GitObjectSyntaxNotSupported: gitObjectSyntaxNotSupported}
			target = pfLocation
			puppetfile := readPuppetfile(target, "", "cmdlineparam", false)
			puppetfile.workDir = "."
			pfm := make(map[string]Puppetfile)
			pfm["cmdlineparam"] = puppetfile
			resolvePuppetfile(pfm)
		} else {
			Fatalf("Error: you need to specify at least a config file or use the Puppetfile mode\nExample call: " + os.Args[0] + " -config test.yaml or " + os.Args[0] + " -puppetfile\n")
		}
	}

	if usemove {
		// we can not reuse the Forge cache at all when -usemove gets used, because we can not delete the -latest link for some reason
		defer purgeDir(config.ForgeCacheDir, "main() -puppetfile mode with -usemove parameter")
	}

	Debugf("Forge response JSON parsing took " + strconv.FormatFloat(forgeJsonParseTime, 'f', 4, 64) + " seconds")
	Debugf("Forge modules metadata.json parsing took " + strconv.FormatFloat(metadataJsonParseTime, 'f', 4, 64) + " seconds")

	if !check4update && !quiet {
		if len(forgeModuleDeprecationNotice) > 0 {
			Warnf(strings.TrimSuffix(forgeModuleDeprecationNotice, "\n"))
		}
		fmt.Println("Synced", target, "with", syncGitCount, "git repositories and", syncForgeCount, "Forge modules in "+strconv.FormatFloat(time.Since(before).Seconds(), 'f', 1, 64)+"s with git ("+strconv.FormatFloat(syncGitTime, 'f', 1, 64)+"s sync, I/O", strconv.FormatFloat(ioGitTime, 'f', 1, 64)+"s) and Forge ("+strconv.FormatFloat(syncForgeTime, 'f', 1, 64)+"s query+download, I/O", strconv.FormatFloat(ioForgeTime, 'f', 1, 64)+"s) using", strconv.Itoa(config.Maxworker), "resolv and", strconv.Itoa(config.MaxExtractworker), "extract workers")
	}
	if dryRun && (needSyncForgeCount > 0 || needSyncGitCount > 0) {
		os.Exit(1)
	}
}
<commit_msg>bump version to v0.4.9<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	debug                        bool
	verbose                      bool
	info                         bool
	quiet                        bool
	force                        bool
	usemove                      bool
	usecacheFallback             bool
	retryGitCommands             bool
	pfMode                       bool
	pfLocation                   string
	dryRun                       bool
	check4update                 bool
	checkSum                     bool
	gitObjectSyntaxNotSupported  bool
	moduleDirParam               string
	cacheDirParam                string
	branchParam                  string
	tags                         bool
	outputNameParam              string
	moduleParam                  string
	configFile                   string
	config                       ConfigSettings
	mutex                        sync.Mutex
	empty                        struct{}
	syncGitCount                 int
	syncForgeCount               int
	needSyncGitCount             int
	needSyncForgeCount           int
	syncGitTime                  float64
	syncForgeTime                float64
	ioGitTime                    float64
	ioForgeTime                  float64
	forgeJsonParseTime           float64
	metadataJsonParseTime        float64
	gmetadataJsonParseTime       float64
	buildtime                    string
	uniqueForgeModules           map[string]ForgeModule
	latestForgeModules           LatestForgeModules
	maxworker                    int
	maxExtractworker             int
	forgeModuleDeprecationNotice string
)

type LatestForgeModules struct {
	sync.RWMutex
	m map[string]string
}

// ConfigSettings contains the key value pairs from the g10k config file
type ConfigSettings struct {
	CacheDir                    string `yaml:"cachedir"`
	ForgeCacheDir               string
	ModulesCacheDir             string
	EnvCacheDir                 string
	Git                         Git
	Forge                       Forge
	Sources                     map[string]Source
	Timeout                     int  `yaml:"timeout"`
	IgnoreUnreachableModules    bool `yaml:"ignore_unreachable_modules"`
	Maxworker                   int  `yaml:"maxworker"`
	MaxExtractworker            int  `yaml:"maxextractworker"`
	UseCacheFallback            bool `yaml:"use_cache_fallback"`
	RetryGitCommands            bool `yaml:"retry_git_commands"`
	GitObjectSyntaxNotSupported bool `yaml:"git_object_syntax_not_supported"`
}

type Forge struct {
	Baseurl string `yaml:"baseurl"`
}

type Git struct {
	privateKey string `yaml:"private_key"`
	username   string
}

// Source contains basic information about a Puppet environment repository
type Source struct {
	Remote                      string
	Basedir                     string
	Prefix                      string
	PrivateKey                  string `yaml:"private_key"`
	ForceForgeVersions          bool   `yaml:"force_forge_versions"`
	WarnMissingBranch           bool   `yaml:"warn_if_branch_is_missing"`
	ExitIfUnreachable           bool   `yaml:"exit_if_unreachable"`
	AutoCorrectEnvironmentNames string `yaml:"invalid_branches"`
}

// Puppetfile contains the key value pairs from the Puppetfile
type Puppetfile struct {
	moduleDir     string
	forgeBaseURL  string
	forgeCacheTtl time.Duration
	forgeModules  map[string]ForgeModule
	gitModules    map[string]GitModule
	privateKey    string
	source        string
	workDir       string
}

// ForgeModule contains information (Version, Name, Author, md5 checksum, file size of the tar.gz archive, Forge BaseURL if custom) about a Puppetlabs Forge module
type ForgeModule struct {
	version   string
	name      string
	author    string
	md5sum    string
	fileSize  int64
	baseUrl   string
	cacheTtl  time.Duration
	sha256sum string
}

// GitModule contains information about a Git Puppet module
type GitModule struct {
	privateKey        string
	git               string
	branch            string
	tag               string
	commit            string
	ref               string
	link              bool
	ignoreUnreachable bool
	fallback          []string
	installPath       string
	local             bool
}

// ForgeResult is returned by queryForgeAPI and contains if and which version of the Puppetlabs Forge module needs to be downloaded
type ForgeResult struct {
	needToGet     bool
	versionNumber string
	md5sum        string
	fileSize      int64
}

// ExecResult contains the exit code and output of an external command (e.g. git)
type ExecResult struct {
	returnCode int
	output     string
}

func main() {

	var (
		configFileFlag = flag.String("config", "", "which config file to use")
		versionFlag    = flag.Bool("version", false, "show build time and version number")
	)
	flag.StringVar(&branchParam, "branch", "", "which git branch of the Puppet environment to update, e.g. core_foobar")
	flag.BoolVar(&tags, "tags", false, "to pull tags as well as branches")
	flag.StringVar(&outputNameParam, "outputname", "", "overwrite the environment name if -branch is specified")
	flag.StringVar(&moduleParam, "module", "", "which module of the Puppet environment to update, e.g. stdlib")
	flag.StringVar(&moduleDirParam, "moduledir", "", "allows overriding of Puppetfile specific moduledir setting, the folder in which Puppet modules will be extracted")
	flag.StringVar(&cacheDirParam, "cachedir", "", "allows overriding of the g10k config file cachedir setting, the folder in which g10k will download git repositories and Forge modules")
	flag.IntVar(&maxworker, "maxworker", 50, "how many Goroutines are allowed to run in parallel for Git and Forge module resolving")
	flag.IntVar(&maxExtractworker, "maxextractworker", 20, "how many Goroutines are allowed to run in parallel for local Git and Forge module extracting processes (git clone, untar and gunzip)")
	flag.BoolVar(&pfMode, "puppetfile", false, "install all modules from Puppetfile in cwd")
	flag.StringVar(&pfLocation, "puppetfilelocation", "./Puppetfile", "which Puppetfile to use in -puppetfile mode")
	flag.BoolVar(&force, "force", false, "purge the Puppet environment directory and do a full sync")
	flag.BoolVar(&dryRun, "dryrun", false, "do not modify anything, just print what would be changed")
	flag.BoolVar(&usemove, "usemove", false, "do not use hardlinks to populate your Puppet environments with Puppetlabs Forge modules. Instead uses simple move commands and purges the Forge cache directory after each run! (Useful for g10k runs inside a Docker container)")
	flag.BoolVar(&check4update, "check4update", false, "only check if the is newer version of the Puppet module avaialable. Does implicitly set dryrun to true")
	flag.BoolVar(&checkSum, "checksum", false, "get the md5 check sum for each Puppetlabs Forge module and verify the integrity of the downloaded archive. Increases g10k run time!")
	flag.BoolVar(&debug, "debug", false, "log debug output, defaults to false")
	flag.BoolVar(&verbose, "verbose", false, "log verbose output, defaults to false")
	flag.BoolVar(&info, "info", false, "log info output, defaults to false")
	flag.BoolVar(&quiet, "quiet", false, "no output, defaults to false")
	flag.BoolVar(&usecacheFallback, "usecachefallback", false, "if g10k should try to use its cache for sources and modules instead of failing")
	flag.BoolVar(&retryGitCommands, "retrygitcommands", false, "if g10k should purge the local repository and retry a failed git command (clone or remote update) instead of failing")
	flag.BoolVar(&gitObjectSyntaxNotSupported, "gitobjectsyntaxnotsupported", false, "if your git version is too old to support reference syntax like master^{object} use this setting to revert to the older syntax")
	flag.Parse()

	configFile = *configFileFlag
	version := *versionFlag

	if version {
		fmt.Println("g10k version 0.4.9 Build time:", buildtime, "UTC")
		os.Exit(0)
	}

	if check4update {
		dryRun = true
	}

	// check for git executable dependency
	if _, err := exec.LookPath("git"); err != nil {
		Fatalf("Error: could not find 'git' executable in PATH")
	}

	target := ""
	before := time.Now()
	if len(configFile) > 0 {
		if usemove {
			Fatalf("Error: -usemove parameter is only allowed in -puppetfile mode!")
		}
		if pfMode {
			Fatalf("Error: -puppetfile parameter is not allowed with -config parameter!")
		}
		if (len(outputNameParam) > 0) && (len(branchParam) == 0) {
			Fatalf("Error: -outputname specified without -branch!")
		}
		if usecacheFallback {
			config.UseCacheFallback = true
		}
		Debugf("Using as config file: " + configFile)
		config = readConfigfile(configFile)
		target = configFile
		if len(branchParam) > 0 {
			resolvePuppetEnvironment(branchParam, tags, outputNameParam)
			target += " with branch " + branchParam
		} else {
			resolvePuppetEnvironment("", tags, "")
		}
	} else {
		if pfMode {
			Debugf("Trying to use as Puppetfile: " + pfLocation)
			sm := make(map[string]Source)
			sm["cmdlineparam"] = Source{Basedir: "."}
			cachedir := "/tmp/g10k"
			if len(os.Getenv("g10k_cachedir")) > 0 {
				cachedir = os.Getenv("g10k_cachedir")
				cachedir = checkDirAndCreate(cachedir, "cachedir environment variable g10k_cachedir")
				Debugf("Found environment variable g10k_cachedir set to: " + cachedir)
			} else if len(cacheDirParam) > 0 {
				Debugf("Using -cachedir parameter set to : " + cacheDirParam)
				cachedir = checkDirAndCreate(cacheDirParam, "cachedir CLI param")
			} else {
				cachedir = checkDirAndCreate(cachedir, "cachedir default value")
			}
			forgeDefaultSettings := Forge{Baseurl: "https://forgeapi.puppetlabs.com"}
			config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: cachedir, EnvCacheDir: cachedir, Sources: sm, Forge: forgeDefaultSettings, Maxworker: maxworker, UseCacheFallback: usecacheFallback, MaxExtractworker: maxExtractworker, RetryGitCommands: retryGitCommands, GitObjectSyntaxNotSupported: gitObjectSyntaxNotSupported}
			target = pfLocation
			puppetfile := readPuppetfile(target, "", "cmdlineparam", false)
			puppetfile.workDir = "."
			pfm := make(map[string]Puppetfile)
			pfm["cmdlineparam"] = puppetfile
			resolvePuppetfile(pfm)
		} else {
			Fatalf("Error: you need to specify at least a config file or use the Puppetfile mode\nExample call: " + os.Args[0] + " -config test.yaml or " + os.Args[0] + " -puppetfile\n")
		}
	}

	if usemove {
		// we can not reuse the Forge cache at all when -usemove gets used, because we can not delete the -latest link for some reason
		defer purgeDir(config.ForgeCacheDir, "main() -puppetfile mode with -usemove parameter")
	}

	Debugf("Forge response JSON parsing took " + strconv.FormatFloat(forgeJsonParseTime, 'f', 4, 64) + " seconds")
	Debugf("Forge modules metadata.json parsing took " + strconv.FormatFloat(metadataJsonParseTime, 'f', 4, 64) + " seconds")

	if !check4update && !quiet {
		if len(forgeModuleDeprecationNotice) > 0 {
			Warnf(strings.TrimSuffix(forgeModuleDeprecationNotice, "\n"))
		}
		fmt.Println("Synced", target, "with", syncGitCount, "git repositories and", syncForgeCount, "Forge modules in "+strconv.FormatFloat(time.Since(before).Seconds(), 'f', 1, 64)+"s with git ("+strconv.FormatFloat(syncGitTime, 'f', 1, 64)+"s sync, I/O", strconv.FormatFloat(ioGitTime, 'f', 1, 64)+"s) and Forge ("+strconv.FormatFloat(syncForgeTime, 'f', 1, 64)+"s query+download, I/O", strconv.FormatFloat(ioForgeTime, 'f', 1, 64)+"s) using", strconv.Itoa(config.Maxworker), "resolv and", strconv.Itoa(config.MaxExtractworker), "extract workers")
	}
	if dryRun && (needSyncForgeCount > 0 || needSyncGitCount > 0) {
		os.Exit(1)
	}
}
<|endoftext|>