<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/phrase/phraseapp-go/phraseapp"
)

type PushCommand struct {
	Credentials
}

func (cmd *PushCommand) Run() error {
	if cmd.Debug {
		// suppresses content output
		cmd.Debug = false
		Debug = true
	}

	err := func() error {
		client, err := ClientFromCmdCredentials(cmd.Credentials)
		if err != nil {
			return err
		}

		sources, err := SourcesFromConfig(cmd)
		if err != nil {
			return err
		}

		for _, source := range sources {
			err := source.Push(client)
			if err != nil {
				return err
			}
		}
		return nil
	}()

	if err != nil {
		ReportError("Push Error", err.Error())
	}

	return err
}

type Sources []*Source

type Source struct {
	File          string      `yaml:"file,omitempty"`
	ProjectID     string      `yaml:"project_id,omitempty"`
	AccessToken   string      `yaml:"access_token,omitempty"`
	FileFormat    string      `yaml:"file_format,omitempty"`
	Params        *phraseapp.UploadParams `yaml:"params"`

	RemoteLocales []*phraseapp.Locale
	Extension     string
}

var separator = string(os.PathSeparator)

func (source *Source) CheckPreconditions() error {
	if err := ValidPath(source.File, source.FileFormat, ""); err != nil {
		return err
	}

	duplicatedPlaceholders := []string{}
	for _, name := range []string{"<locale_name>", "<locale_code>", "<tag>"} {
		if strings.Count(source.File, name) > 1 {
			duplicatedPlaceholders = append(duplicatedPlaceholders, name)
		}
	}

	starCount := strings.Count(source.File, "*")
	recCount := strings.Count(source.File, "**")

	if recCount == 0 && starCount > 1 || starCount-(recCount*2) > 1 {
		duplicatedPlaceholders = append(duplicatedPlaceholders, "*")
	}

	if recCount > 1 {
		duplicatedPlaceholders = append(duplicatedPlaceholders, "**")
	}

	if len(duplicatedPlaceholders) > 0 {
		dups := strings.Join(duplicatedPlaceholders, ", ")
		return fmt.Errorf(fmt.Sprintf("%s can only occur once in a file pattern!", dups))
	}

	return nil
}

func (source *Source) Push(client *phraseapp.Client) error {
	if err := source.CheckPreconditions(); err != nil {
		return err
	}

	source.Extension = filepath.Ext(source.File)

	remoteLocales, err := RemoteLocales(client, source.ProjectID)
	if err != nil {
		return err
	}
	source.RemoteLocales = remoteLocales

	localeFiles, err := source.LocaleFiles()
	if err != nil {
		return err
	}

	for _, localeFile := range localeFiles {
		fmt.Println("Uploading", localeFile.RelPath())

		if !localeFile.ExistsRemote {
			localeDetails, err := source.createLocale(client, localeFile)
			if err == nil {
				localeFile.ID = localeDetails.ID
				localeFile.RFC = localeDetails.Code
				localeFile.Name = localeDetails.Name
			} else {
				fmt.Printf("failed to create locale: %s\n", err)
				continue
			}
		}

		err = source.uploadFile(client, localeFile)
		if err != nil {
			return err
		} else {
			sharedMessage("push", localeFile)
		}

		if Debug {
			fmt.Fprintln(os.Stderr, strings.Repeat("-", 10))
		}

	}

	return nil
}

func (source *Source) createLocale(client *phraseapp.Client, localeFile *LocaleFile) (*phraseapp.LocaleDetails, error) {
	if localeFile.RFC == "" {
		return nil, fmt.Errorf("no locale code specified")
	}

	localeParams := new(phraseapp.LocaleParams)

	if localeFile.Name != "" {
		localeParams.Name = &localeFile.Name
	} else if localeFile.RFC != "" {
		localeParams.Name = &localeFile.RFC
	}

	localeName := source.replacePlaceholderInParams(localeFile)
	if localeName != "" && localeName != localeFile.RFC {
		localeParams.Name = &localeName
	}

	if localeFile.RFC != "" {
		localeParams.Code = &localeFile.RFC
	}

	localeDetails, err := client.LocaleCreate(source.ProjectID, localeParams)
	if err != nil {
		return nil, err
	}
	return localeDetails, nil
}

func (source *Source) replacePlaceholderInParams(localeFile *LocaleFile) string {
	if localeFile.RFC != "" && strings.Contains(source.GetLocaleID(), "<locale_code>") {
		return strings.Replace(source.GetLocaleID(), "<locale_code>", localeFile.RFC, 1)
	}
	return ""
}

func (source *Source) uploadFile(client *phraseapp.Client, localeFile *LocaleFile) error {
	if Debug {
		fmt.Fprintln(os.Stdout, "Source file pattern:", source.File)
		fmt.Fprintln(os.Stdout, "Actual file location:", localeFile.Path)
	}

	params := new(phraseapp.UploadParams)
	*params = *source.Params

	params.File = &localeFile.Path

	if params.LocaleID == nil {
		switch {
		case localeFile.ID != "":
			params.LocaleID = &localeFile.ID
		case localeFile.RFC != "":
			params.LocaleID = &localeFile.RFC
		}
	}

	aUpload, err := client.UploadCreate(source.ProjectID, params)
	if err != nil {
		return err
	}

	return nil
}

func (source *Source) SystemFiles() ([]string, error) {
	if strings.Contains(source.File, "**") {
		return source.recurse()
	}

	return source.glob()
}

func (source *Source) glob() ([]string, error) {
	withoutPlaceholder := placeholderRegexp.ReplaceAllString(source.File, "*")
	tokens := splitPathToTokens(withoutPlaceholder)

	fileHead := tokens[len(tokens)-1]
	if strings.HasPrefix(fileHead, ".") {
		tokens[len(tokens)-1] = "*" + fileHead
	}
	pattern := strings.Join(tokens, separator)

	files, err := filepath.Glob(pattern)
	if err != nil {
		return nil, err
	}

	if Debug {
		fmt.Fprintln(os.Stderr, "Found", len(files), "files matching the source pattern", pattern)
	}

	return files, nil
}

func (source *Source) recurse() ([]string, error) {
	files := []string{}
	err := filepath.Walk(source.root(), func(path string, f os.FileInfo, err error) error {
		if err != nil {
			errmsg := fmt.Sprintf("%s for pattern: %s", err, source.File)
			ReportError("Push Error", errmsg)
			return fmt.Errorf(errmsg)
		}
		if !f.Mode().IsDir() && strings.HasSuffix(f.Name(), source.Extension) {
			files = append(files, path)
		}
		return nil
	})

	return files, err
}

func (source *Source) root() string {
	parts := splitPathToTokens(source.File)
	rootParts := TakeWhile(parts, func(x string) bool {
		return x != "**"
	})
	root := strings.Join(rootParts, separator)
	if root == "" {
		root = "."
	}
	return root
}

// Return all locale files from disk that match the source pattern.
func (source *Source) LocaleFiles() (LocaleFiles, error) {
	filePaths, err := source.SystemFiles()
	if err != nil {
		return nil, err
	}

	tokens := splitPathToTokens(source.File)

	var localeFiles LocaleFiles
	for _, path := range filePaths {

		pathTokens := splitPathToTokens(path)
		if len(pathTokens) < len(tokens) {
			continue
		}
		localeFile := Reduce(tokens, pathTokens)

		absolutePath, err := filepath.Abs(path)
		if err != nil {
			return nil, err
		}
		localeFile.Path = absolutePath

		locale := source.getRemoteLocaleForLocaleFile(localeFile)
		if locale != nil {
			localeFile.ExistsRemote = true
			localeFile.RFC = locale.Code
			localeFile.Name = locale.Name
			localeFile.ID = locale.ID
		}

		if Debug {
			fmt.Println(fmt.Sprintf(
				"RFC:'%s', Name:'%s', Tag;'%s', Pattern:'%s'",
				localeFile.RFC, localeFile.Name, localeFile.Tag,
			))
		}

		localeFiles = append(localeFiles, localeFile)
	}

	if len(localeFiles) <= 0 {
		abs, err := filepath.Abs(source.File)
		if err != nil {
			abs = source.File
		}
		return nil, fmt.Errorf("Could not find any files on your system that matches: '%s'", abs)
	}
	return localeFiles, nil
}

func (source *Source) getRemoteLocaleForLocaleFile(localeFile *LocaleFile) *phraseapp.Locale {
	for _, remote := range source.RemoteLocales {
		if remote.Name == source.GetLocaleID() || remote.ID == source.GetLocaleID() {
			return remote
		}

		localeName := source.replacePlaceholderInParams(localeFile)
		if localeName != "" && strings.Contains(remote.Name, localeName) {
			return remote
		}

		if remote.Name == localeFile.Name {
			return remote
		}

		if remote.Name == localeFile.RFC {
			return remote
		}
	}
	return nil
}

func splitPathToTokens(s string) []string {
	tokens := []string{}
	for _, token := range strings.Split(s, separator) {
		if token == "." || token == "" {
			continue
		}
		tokens = append(tokens, token)
	}
	return tokens
}

func Reduce(tokens, pathTokens []string) *LocaleFile {
	tagged := map[string]string{}

	for idx, token := range tokens {
		pathToken := pathTokens[idx]
		if token == "*" {
			continue
		}
		if token == "**" {
			break
		}
		tagged = tag(tagged, token, pathToken)
	}

	if Contains(tokens, "**") {
		offset := 1
		for idx := len(tokens) - 1; idx >= 0; idx-- {
			token := tokens[idx]
			pathToken := pathTokens[len(pathTokens)-offset]
			offset += 1

			if token == "*" {
				continue
			}
			if token == "**" {
				break
			}

			tagged = tag(tagged, token, pathToken)
		}
	}

	return &LocaleFile{
		Name: tagged["locale_name"],
		RFC:  tagged["locale_code"],
		Tag:  tagged["tag"],
	}
}

func tag(tagged map[string]string, token, pathToken string) map[string]string {
	groups := placeholderRegexp.FindAllString(token, -1)
	if len(groups) <= 0 {
		return tagged
	}

	match := strings.Replace(token, ".", "[.]", -1)
	if strings.HasPrefix(match, "*") {
		match = strings.Replace(match, "*", ".*", -1)
	}

	for _, group := range groups {
		replacer := fmt.Sprintf("(?P%s.+)", group)
		match = strings.Replace(match, group, replacer, 1)
	}

	if match == "" {
		return tagged
	}

	tmpRegexp, err := regexp.Compile(match)
	if err != nil {
		return tagged
	}

	namedMatches := tmpRegexp.SubexpNames()
	subMatches := tmpRegexp.FindStringSubmatch(pathToken)
	for i, subMatch := range subMatches {
		if subMatch != "" {
			tagged[namedMatches[i]] = strings.Trim(subMatch, separator)
		}
	}

	return tagged
}

// Configuration
type PushConfig struct {
	Phraseapp struct {
		AccessToken string `yaml:"access_token"`
		ProjectID   string `yaml:"project_id"`
		FileFormat  string `yaml:"file_format,omitempty"`
		Push        struct {
			Sources Sources
		}
	}
}

func SourcesFromConfig(cmd *PushCommand) (Sources, error) {
	content, err := ConfigContent()
	if err != nil {
		return nil, err
	}

	var config *PushConfig

	err = yaml.Unmarshal([]byte(content), &config)
	if err != nil {
		return nil, err
	}

	token := config.Phraseapp.AccessToken
	if cmd.Token != "" {
		token = cmd.Token
	}
	projectId := config.Phraseapp.ProjectID
	fileFormat := config.Phraseapp.FileFormat

	if &config.Phraseapp.Push == nil || config.Phraseapp.Push.Sources == nil {
		return nil, fmt.Errorf("no sources for upload specified")
	}

	sources := config.Phraseapp.Push.Sources

	validSources := []*Source{}
	for _, source := range sources {
		if source == nil {
			continue
		}
		if source.ProjectID == "" {
			source.ProjectID = projectId
		}
		if source.AccessToken == "" {
			source.AccessToken = token
		}
		if source.Params == nil {
			source.Params = new(phraseapp.UploadParams)
		}

		if source.Params.FileFormat == nil {
			switch {
			case source.FileFormat != "":
				source.Params.FileFormat = &source.FileFormat
			case fileFormat != "":
				source.Params.FileFormat = &fileFormat
			}
		}
		validSources = append(validSources, source)
	}

	if len(validSources) <= 0 {
		return nil, fmt.Errorf("no sources could be identified! Refine the sources list in your config")
	}

	return validSources, nil
}

func (source *Source) GetLocaleID() string {
	if source.Params != nil && source.Params.LocaleID != nil {
		return *source.Params.LocaleID
	}
	return ""
}
<commit_msg>renamed some functions and variables<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/phrase/phraseapp-go/phraseapp"
)

type PushCommand struct {
	Credentials
}

func (cmd *PushCommand) Run() error {
	if cmd.Debug {
		// suppresses content output
		cmd.Debug = false
		Debug = true
	}

	err := func() error {
		client, err := ClientFromCmdCredentials(cmd.Credentials)
		if err != nil {
			return err
		}

		sources, err := SourcesFromConfig(cmd)
		if err != nil {
			return err
		}

		for _, source := range sources {
			err := source.Push(client)
			if err != nil {
				return err
			}
		}
		return nil
	}()

	if err != nil {
		ReportError("Push Error", err.Error())
	}

	return err
}

type Sources []*Source

type Source struct {
	File          string      `yaml:"file,omitempty"`
	ProjectID     string      `yaml:"project_id,omitempty"`
	AccessToken   string      `yaml:"access_token,omitempty"`
	FileFormat    string      `yaml:"file_format,omitempty"`
	Params        *phraseapp.UploadParams `yaml:"params"`

	RemoteLocales []*phraseapp.Locale
	Extension     string
}

var separator = string(os.PathSeparator)

func (source *Source) CheckPreconditions() error {
	if err := ValidPath(source.File, source.FileFormat, ""); err != nil {
		return err
	}

	duplicatedPlaceholders := []string{}
	for _, name := range []string{"<locale_name>", "<locale_code>", "<tag>"} {
		if strings.Count(source.File, name) > 1 {
			duplicatedPlaceholders = append(duplicatedPlaceholders, name)
		}
	}

	starCount := strings.Count(source.File, "*")
	recCount := strings.Count(source.File, "**")

	if recCount == 0 && starCount > 1 || starCount-(recCount*2) > 1 {
		duplicatedPlaceholders = append(duplicatedPlaceholders, "*")
	}

	if recCount > 1 {
		duplicatedPlaceholders = append(duplicatedPlaceholders, "**")
	}

	if len(duplicatedPlaceholders) > 0 {
		dups := strings.Join(duplicatedPlaceholders, ", ")
		return fmt.Errorf(fmt.Sprintf("%s can only occur once in a file pattern!", dups))
	}

	return nil
}

func (source *Source) Push(client *phraseapp.Client) error {
	if err := source.CheckPreconditions(); err != nil {
		return err
	}

	source.Extension = filepath.Ext(source.File)

	remoteLocales, err := RemoteLocales(client, source.ProjectID)
	if err != nil {
		return err
	}
	source.RemoteLocales = remoteLocales

	localeFiles, err := source.LocaleFiles()
	if err != nil {
		return err
	}

	for _, localeFile := range localeFiles {
		fmt.Println("Uploading", localeFile.RelPath())

		if !localeFile.ExistsRemote {
			localeDetails, err := source.createLocale(client, localeFile)
			if err == nil {
				localeFile.ID = localeDetails.ID
				localeFile.RFC = localeDetails.Code
				localeFile.Name = localeDetails.Name
			} else {
				fmt.Printf("failed to create locale: %s\n", err)
				continue
			}
		}

		err = source.uploadFile(client, localeFile)
		if err != nil {
			return err
		} else {
			sharedMessage("push", localeFile)
		}

		if Debug {
			fmt.Fprintln(os.Stderr, strings.Repeat("-", 10))
		}

	}

	return nil
}

func (source *Source) createLocale(client *phraseapp.Client, localeFile *LocaleFile) (*phraseapp.LocaleDetails, error) {
	if localeFile.RFC == "" {
		return nil, fmt.Errorf("no locale code specified")
	}

	localeParams := new(phraseapp.LocaleParams)

	if localeFile.Name != "" {
		localeParams.Name = &localeFile.Name
	} else if localeFile.RFC != "" {
		localeParams.Name = &localeFile.RFC
	}

	localeName := source.replacePlaceholderInParams(localeFile)
	if localeName != "" && localeName != localeFile.RFC {
		localeParams.Name = &localeName
	}

	if localeFile.RFC != "" {
		localeParams.Code = &localeFile.RFC
	}

	localeDetails, err := client.LocaleCreate(source.ProjectID, localeParams)
	if err != nil {
		return nil, err
	}
	return localeDetails, nil
}

func (source *Source) replacePlaceholderInParams(localeFile *LocaleFile) string {
	if localeFile.RFC != "" && strings.Contains(source.GetLocaleID(), "<locale_code>") {
		return strings.Replace(source.GetLocaleID(), "<locale_code>", localeFile.RFC, 1)
	}
	return ""
}

func (source *Source) uploadFile(client *phraseapp.Client, localeFile *LocaleFile) error {
	if Debug {
		fmt.Fprintln(os.Stdout, "Source file pattern:", source.File)
		fmt.Fprintln(os.Stdout, "Actual file location:", localeFile.Path)
	}

	params := new(phraseapp.UploadParams)
	*params = *source.Params

	params.File = &localeFile.Path

	if params.LocaleID == nil {
		switch {
		case localeFile.ID != "":
			params.LocaleID = &localeFile.ID
		case localeFile.RFC != "":
			params.LocaleID = &localeFile.RFC
		}
	}

	aUpload, err := client.UploadCreate(source.ProjectID, params)
	if err != nil {
		return err
	}

	return nil
}

func (source *Source) SystemFiles() ([]string, error) {
	if strings.Contains(source.File, "**") {
		return source.recurse()
	}

	return source.glob()
}

func (source *Source) glob() ([]string, error) {
	withoutPlaceholder := placeholderRegexp.ReplaceAllString(source.File, "*")
	tokens := splitPathToTokens(withoutPlaceholder)

	fileHead := tokens[len(tokens)-1]
	if strings.HasPrefix(fileHead, ".") {
		tokens[len(tokens)-1] = "*" + fileHead
	}
	pattern := strings.Join(tokens, separator)

	files, err := filepath.Glob(pattern)
	if err != nil {
		return nil, err
	}

	if Debug {
		fmt.Fprintln(os.Stderr, "Found", len(files), "files matching the source pattern", pattern)
	}

	return files, nil
}

func (source *Source) recurse() ([]string, error) {
	files := []string{}
	err := filepath.Walk(source.root(), func(path string, f os.FileInfo, err error) error {
		if err != nil {
			errmsg := fmt.Sprintf("%s for pattern: %s", err, source.File)
			ReportError("Push Error", errmsg)
			return fmt.Errorf(errmsg)
		}
		if !f.Mode().IsDir() && strings.HasSuffix(f.Name(), source.Extension) {
			files = append(files, path)
		}
		return nil
	})

	return files, err
}

func (source *Source) root() string {
	parts := splitPathToTokens(source.File)
	rootParts := TakeWhile(parts, func(x string) bool {
		return x != "**"
	})
	root := strings.Join(rootParts, separator)
	if root == "" {
		root = "."
	}
	return root
}

// Return all locale files from disk that match the source pattern.
func (source *Source) LocaleFiles() (LocaleFiles, error) {
	filePaths, err := source.SystemFiles()
	if err != nil {
		return nil, err
	}

	tokens := splitPathToTokens(source.File)

	var localeFiles LocaleFiles
	for _, path := range filePaths {

		pathTokens := splitPathToTokens(path)
		if len(pathTokens) < len(tokens) {
			continue
		}
		localeFile := extractParamsFromPathTokens(tokens, pathTokens)

		absolutePath, err := filepath.Abs(path)
		if err != nil {
			return nil, err
		}
		localeFile.Path = absolutePath

		locale := source.getRemoteLocaleForLocaleFile(localeFile)
		if locale != nil {
			localeFile.ExistsRemote = true
			localeFile.RFC = locale.Code
			localeFile.Name = locale.Name
			localeFile.ID = locale.ID
		}

		if Debug {
			fmt.Println(fmt.Sprintf(
				"RFC:'%s', Name:'%s', Tag;'%s', Pattern:'%s'",
				localeFile.RFC, localeFile.Name, localeFile.Tag,
			))
		}

		localeFiles = append(localeFiles, localeFile)
	}

	if len(localeFiles) <= 0 {
		abs, err := filepath.Abs(source.File)
		if err != nil {
			abs = source.File
		}
		return nil, fmt.Errorf("Could not find any files on your system that matches: '%s'", abs)
	}
	return localeFiles, nil
}

func (source *Source) getRemoteLocaleForLocaleFile(localeFile *LocaleFile) *phraseapp.Locale {
	for _, remote := range source.RemoteLocales {
		if remote.Name == source.GetLocaleID() || remote.ID == source.GetLocaleID() {
			return remote
		}

		localeName := source.replacePlaceholderInParams(localeFile)
		if localeName != "" && strings.Contains(remote.Name, localeName) {
			return remote
		}

		if remote.Name == localeFile.Name {
			return remote
		}

		if remote.Name == localeFile.RFC {
			return remote
		}
	}
	return nil
}

func splitPathToTokens(s string) []string {
	tokens := []string{}
	for _, token := range strings.Split(s, separator) {
		if token == "." || token == "" {
			continue
		}
		tokens = append(tokens, token)
	}
	return tokens
}

func extractParamsFromPathTokens(srcTokens, pathTokens []string) *LocaleFile {
	localeFile := new(LocaleFile)

	for idx, token := range srcTokens {
		pathToken := pathTokens[idx]
		if token == "*" {
			continue
		}
		if token == "**" {
			break
		}
		extractParamFromPathToken(localeFile, token, pathToken)
	}

	if Contains(srcTokens, "**") {
		offset := 1
		for idx := len(srcTokens) - 1; idx >= 0; idx-- {
			token := srcTokens[idx]
			pathToken := pathTokens[len(pathTokens)-offset]
			offset += 1

			if token == "*" {
				continue
			}
			if token == "**" {
				break
			}

			extractParamFromPathToken(localeFile, token, pathToken)
		}
	}

	return localeFile
}

func extractParamFromPathToken(localeFile *LocaleFile, srcToken, pathToken string) {
	groups := placeholderRegexp.FindAllString(srcToken, -1)
	if len(groups) <= 0 {
		return
	}

	match := strings.Replace(srcToken, ".", "[.]", -1)
	if strings.HasPrefix(match, "*") {
		match = strings.Replace(match, "*", ".*", -1)
	}

	for _, group := range groups {
		replacer := fmt.Sprintf("(?P%s.+)", group)
		match = strings.Replace(match, group, replacer, 1)
	}

	if match == "" {
		return
	}

	tmpRegexp, err := regexp.Compile(match)
	if err != nil {
		return
	}

	namedMatches := tmpRegexp.SubexpNames()
	subMatches := tmpRegexp.FindStringSubmatch(pathToken)
	for i, subMatch := range subMatches {
		value := strings.Trim(subMatch, separator)
		switch namedMatches[i] {
		case "locale_code":
			localeFile.RFC = value
		case "locale_name":
			localeFile.Name = value
		case "tag":
			localeFile.Tag = value
		default:
			// ignore
		}
	}
}

// Configuration
type PushConfig struct {
	Phraseapp struct {
		AccessToken string `yaml:"access_token"`
		ProjectID   string `yaml:"project_id"`
		FileFormat  string `yaml:"file_format,omitempty"`
		Push        struct {
			Sources Sources
		}
	}
}

func SourcesFromConfig(cmd *PushCommand) (Sources, error) {
	content, err := ConfigContent()
	if err != nil {
		return nil, err
	}

	var config *PushConfig

	err = yaml.Unmarshal([]byte(content), &config)
	if err != nil {
		return nil, err
	}

	token := config.Phraseapp.AccessToken
	if cmd.Token != "" {
		token = cmd.Token
	}
	projectId := config.Phraseapp.ProjectID
	fileFormat := config.Phraseapp.FileFormat

	if &config.Phraseapp.Push == nil || config.Phraseapp.Push.Sources == nil {
		return nil, fmt.Errorf("no sources for upload specified")
	}

	sources := config.Phraseapp.Push.Sources

	validSources := []*Source{}
	for _, source := range sources {
		if source == nil {
			continue
		}
		if source.ProjectID == "" {
			source.ProjectID = projectId
		}
		if source.AccessToken == "" {
			source.AccessToken = token
		}
		if source.Params == nil {
			source.Params = new(phraseapp.UploadParams)
		}

		if source.Params.FileFormat == nil {
			switch {
			case source.FileFormat != "":
				source.Params.FileFormat = &source.FileFormat
			case fileFormat != "":
				source.Params.FileFormat = &fileFormat
			}
		}
		validSources = append(validSources, source)
	}

	if len(validSources) <= 0 {
		return nil, fmt.Errorf("no sources could be identified! Refine the sources list in your config")
	}

	return validSources, nil
}

func (source *Source) GetLocaleID() string {
	if source.Params != nil && source.Params.LocaleID != nil {
		return *source.Params.LocaleID
	}
	return ""
}
<|endoftext|>


<commit_before>package proxmox

import (
	"errors"
	"net/url"
	"strconv"
	"strings"
	"time"

	_ "github.com/davecgh/go-spew/spew"
)

type QemuVM struct {
	Mem       float64 `json:"mem"`
	CPUs      float64 `json:"cpus"`
	NetOut    float64 `json:"netout"`
	PID       string  `json:"pid"`
	Disk      float64 `json:"disk"`
	MaxMem    float64 `json:"maxmem"`
	Status    string  `json:"status"`
	Template  float64 `json:"template"`
	NetIn     float64 `json:"netin"`
	MaxDisk   float64 `json:"maxdisk"`
	Name      string  `json:"name"`
	DiskWrite float64 `json:"diskwrite"`
	CPU       float64 `json:"cpu"`
	VMId      float64 `json:"vmid"`
	DiskRead  float64 `json:"diskread"`
	Uptime    float64 `json:"uptime"`
	Node      Node
}

type QemuList map[string]QemuVM

type QemuNet map[string]string

type QemuConfig struct {
	Bootdisk    string  `json:"bootdisk"`
	Cores       float64 `json:"cores"`
	Digest      string  `json:"digest"`
	Memory      float64 `json:"memory"`
	Net         map[string]QemuNet
	SMBios1     string            `json:"smbios1"`
	Sockets     float64           `json:"sockets"`
	Disks       map[string]string `json:"disks"`
	Description string            `json:"description"`
}

type QemuStatus struct {
	CPU       float64 `json:"cpu"`
	CPUs      float64 `json:"cpus"`
	Mem       float64 `json:"mem"`
	MaxMem    float64 `json:"maxmem"`
	Disk      float64 `json:"disk"`
	MaxDisk   float64 `json:"maxdisk"`
	DiskWrite float64 `json:"diskwrite"`
	DiskRead  float64 `json:"diskread"`
	NetIn     float64 `json:"netin"`
	NetOut    float64 `json:"netout"`
	Uptime    float64 `json:"uptime"`
	QmpStatus string  `json:"qmpstatus"`
	Status    string  `json:"status"`
	Template  string  `json:"template"`
}

func (qemu QemuVM) Delete() (map[string]interface{}, error) {
	var target string
	var data map[string]interface{}
	var err error

	//fmt.Print("!QemuDelete ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64)
	data, err = qemu.Node.Proxmox.Delete(target)
	if err != nil {
		return nil, err
	}
	return data, nil
}

func stringToMap(data string, itemSeparator string, kvSeparator string) map[string]string {
	var result map[string]string

	result = make(map[string]string)
	list := strings.Split(data, itemSeparator)
	for _, item := range list {
		kv := strings.Split(item, kvSeparator)
		result[kv[0]] = kv[1]
	}
	return result
}

func (qemu QemuVM) Config() (QemuConfig, error) {
	var target string
	var data map[string]interface{}
	var results map[string]interface{}
	var config QemuConfig
	var err error

	//fmt.Print("!QemuConfig ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/config"
	data, err = qemu.Node.Proxmox.Get(target)
	results = data["data"].(map[string]interface{})
	if err != nil {
		return config, err
	}
	config = QemuConfig{
		Bootdisk:    results["bootdisk"].(string),
		Cores:       results["cores"].(float64),
		Digest:      results["digest"].(string),
		Memory:      results["memory"].(float64),
		Sockets:     results["sockets"].(float64),
		SMBios1:     results["smbios1"].(string),
		Description: results["description"].(string),
	}
	disktype := [3]string{"virtio", "sata", "ide"}
	disknum := [4]string{"0", "1", "2", "3"}
	config.Disks = make(map[string]string)
	for _, d := range disktype {
		for _, i := range disknum {
			id := d + i
			if disk, ok := results[id]; ok {
				config.Disks[id] = disk.(string)
			}
		}
	}
	config.Net = make(map[string]QemuNet)
	netnum := [4]string{"0", "1", "2", "3"}
	for _, n := range netnum {
		if net, ok := results["net"+n]; ok {
			config.Net["net"+n] = stringToMap(net.(string), ",", "=")
		}
	}

	return config, nil
}

func (qemu QemuVM) CurrentStatus() (QemuStatus, error) {
	var target string
	var err error
	var data map[string]interface{}
	var results map[string]interface{}
	var status QemuStatus

	//fmt.Println("!QemuStatus ", strconv.FormatFloat(qemu.VMId, 'f', 0, 64))

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/current"
	data, err = qemu.Node.Proxmox.Get(target)
	if err != nil {
		return status, err
	}
	results = data["data"].(map[string]interface{})
	status = QemuStatus{
		CPU:       results["cpu"].(float64),
		CPUs:      results["cpus"].(float64),
		Mem:       results["mem"].(float64),
		MaxMem:    results["maxmem"].(float64),
		Disk:      results["disk"].(float64),
		MaxDisk:   results["maxdisk"].(float64),
		DiskWrite: results["diskwrite"].(float64),
		DiskRead:  results["diskread"].(float64),
		NetIn:     results["netin"].(float64),
		NetOut:    results["netout"].(float64),
		Uptime:    results["uptime"].(float64),
		QmpStatus: results["qmpstatus"].(string),
		Status:    results["status"].(string),
		Template:  results["template"].(string),
	}
	return status, nil
}

func (qemu QemuVM) WaitForStatus(status string, timeout int) error {
	var i int
	var err error
	var qStatus QemuStatus
	for i = 0; i < timeout; i++ {
		qStatus, err = qemu.CurrentStatus()
		if err != nil {
			return err
		}

		if qStatus.Status == status {
			return nil
		}
		time.Sleep(time.Second * 1)
	}
	return errors.New("Timeout reached")
}

func (qemu QemuVM) Start() error {
	var target string
	var err error

	//fmt.Println("!QemuStart ", strconv.FormatFloat(qemu.VMId, 'f', 0, 64))

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/start"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Stop() error {
	var target string
	var err error

	//fmt.Print("!QemuStop ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/stop"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Shutdown() (Task, error) {
	var target string
	var err error

	//fmt.Print("!QemuShutdown ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/shutdown"
	data, err := qemu.Node.Proxmox.Post(target, "")

	if err != err {
		return Task{}, err
	}

	t := Task{
		UPid:    data["data"].(string),
		proxmox: qemu.Node.Proxmox,
	}

	return t, err
}

func (qemu QemuVM) Suspend() error {
	var target string
	var err error

	//fmt.Print("!QemuSuspend ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/suspend"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Resume() error {
	var target string
	var err error

	//fmt.Print("!QemuResume ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/resume"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Clone(newId float64, name string, targetName string) (Task, error) {
	var target string
	var err error

	newVMID := strconv.FormatFloat(newId, 'f', 0, 64)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/clone"

	form := url.Values{
		"newid":  {newVMID},
		"name":   {name},
		"target": {targetName},
		"full":   {"1"},
	}

	data, err := qemu.Node.Proxmox.PostForm(target, form)
	if err != err {
		return Task{}, err
	}

	t := Task{
		UPid:    data["data"].(string),
		proxmox: qemu.Node.Proxmox,
	}

	return t, nil
}

func (qemu QemuVM) SetDescription(description string) (error) {
	var target string
	var err error

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/config"

	form := url.Values{
		"description":  {description},
	}

	_, err = qemu.Node.Proxmox.PutForm(target, form)
	if err != err {
		return err
	}

	return nil
}

func (qemu QemuVM) SetMemory(memory int) (error) {
	var target string
	var err error

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/config"

	form := url.Values{
		"memory":  {memory},
	}

	_, err = qemu.Node.Proxmox.PutForm(target, form)
	if err != err {
		return err
	}

	return nil
}

func (qemu QemuVM) SetIPSet(ip string) error {
	var target string
	var err error

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/firewall/options"

	form := url.Values{
		"dhcp":          {"1"},
		"enable":        {"1"},
		"log_level_in":  {"nolog"},
		"log_level_out": {"nolog"},
		"macfilter":     {"1"},
		"ipfilter":      {"1"},
		"ndp":           {"1"},
		"policy_in":     {"ACCEPT"},
		"policy_out":    {"ACCEPT"},
	}

	_, err = qemu.Node.Proxmox.PutForm(target, form)
	if err != nil {
		return err
	}

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/firewall/ipset"

	form = url.Values{
		"name": {"ipfilter-net0"},
	}

	_, err = qemu.Node.Proxmox.PostForm(target, form)
	if err != nil {
		return err
	}

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/firewall/ipset/ipfilter-net0"

	form = url.Values{
		"cidr": {ip},
	}

	_, err = qemu.Node.Proxmox.PostForm(target, form)
	if err != nil {
		return err
	}

	config, err := qemu.Config()
	if err != nil {
		return err
	}

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/config"

	net := ""

	for k, v := range config.Net["net0"] {
		net += k + "=" + v + ","
	}

	form = url.Values{
		"net0": {net + ",firewall=1"},
	}

	_, err = qemu.Node.Proxmox.PutForm(target, form)
	if err != nil {
		return err
	}

	return nil
}

func (qemu QemuVM) ResizeDisk(size string) error {
	var target string
	var err error

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/resize"

	form := url.Values{
		"disk": {"scsi1"},
		"size": {size + "G"},
	}

	_, err = qemu.Node.Proxmox.PutForm(target, form)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>fixed memory func<commit_after>package proxmox

import (
	"errors"
	"net/url"
	"strconv"
	"strings"
	"time"

	_ "github.com/davecgh/go-spew/spew"
)

type QemuVM struct {
	Mem       float64 `json:"mem"`
	CPUs      float64 `json:"cpus"`
	NetOut    float64 `json:"netout"`
	PID       string  `json:"pid"`
	Disk      float64 `json:"disk"`
	MaxMem    float64 `json:"maxmem"`
	Status    string  `json:"status"`
	Template  float64 `json:"template"`
	NetIn     float64 `json:"netin"`
	MaxDisk   float64 `json:"maxdisk"`
	Name      string  `json:"name"`
	DiskWrite float64 `json:"diskwrite"`
	CPU       float64 `json:"cpu"`
	VMId      float64 `json:"vmid"`
	DiskRead  float64 `json:"diskread"`
	Uptime    float64 `json:"uptime"`
	Node      Node
}

type QemuList map[string]QemuVM

type QemuNet map[string]string

type QemuConfig struct {
	Bootdisk    string  `json:"bootdisk"`
	Cores       float64 `json:"cores"`
	Digest      string  `json:"digest"`
	Memory      float64 `json:"memory"`
	Net         map[string]QemuNet
	SMBios1     string            `json:"smbios1"`
	Sockets     float64           `json:"sockets"`
	Disks       map[string]string `json:"disks"`
	Description string            `json:"description"`
}

type QemuStatus struct {
	CPU       float64 `json:"cpu"`
	CPUs      float64 `json:"cpus"`
	Mem       float64 `json:"mem"`
	MaxMem    float64 `json:"maxmem"`
	Disk      float64 `json:"disk"`
	MaxDisk   float64 `json:"maxdisk"`
	DiskWrite float64 `json:"diskwrite"`
	DiskRead  float64 `json:"diskread"`
	NetIn     float64 `json:"netin"`
	NetOut    float64 `json:"netout"`
	Uptime    float64 `json:"uptime"`
	QmpStatus string  `json:"qmpstatus"`
	Status    string  `json:"status"`
	Template  string  `json:"template"`
}

func (qemu QemuVM) Delete() (map[string]interface{}, error) {
	var target string
	var data map[string]interface{}
	var err error

	//fmt.Print("!QemuDelete ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64)
	data, err = qemu.Node.Proxmox.Delete(target)
	if err != nil {
		return nil, err
	}
	return data, nil
}

func stringToMap(data string, itemSeparator string, kvSeparator string) map[string]string {
	var result map[string]string

	result = make(map[string]string)
	list := strings.Split(data, itemSeparator)
	for _, item := range list {
		kv := strings.Split(item, kvSeparator)
		result[kv[0]] = kv[1]
	}
	return result
}

func (qemu QemuVM) Config() (QemuConfig, error) {
	var target string
	var data map[string]interface{}
	var results map[string]interface{}
	var config QemuConfig
	var err error

	//fmt.Print("!QemuConfig ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/config"
	data, err = qemu.Node.Proxmox.Get(target)
	results = data["data"].(map[string]interface{})
	if err != nil {
		return config, err
	}
	config = QemuConfig{
		Bootdisk:    results["bootdisk"].(string),
		Cores:       results["cores"].(float64),
		Digest:      results["digest"].(string),
		Memory:      results["memory"].(float64),
		Sockets:     results["sockets"].(float64),
		SMBios1:     results["smbios1"].(string),
		Description: results["description"].(string),
	}
	disktype := [3]string{"virtio", "sata", "ide"}
	disknum := [4]string{"0", "1", "2", "3"}
	config.Disks = make(map[string]string)
	for _, d := range disktype {
		for _, i := range disknum {
			id := d + i
			if disk, ok := results[id]; ok {
				config.Disks[id] = disk.(string)
			}
		}
	}
	config.Net = make(map[string]QemuNet)
	netnum := [4]string{"0", "1", "2", "3"}
	for _, n := range netnum {
		if net, ok := results["net"+n]; ok {
			config.Net["net"+n] = stringToMap(net.(string), ",", "=")
		}
	}

	return config, nil
}

func (qemu QemuVM) CurrentStatus() (QemuStatus, error) {
	var target string
	var err error
	var data map[string]interface{}
	var results map[string]interface{}
	var status QemuStatus

	//fmt.Println("!QemuStatus ", strconv.FormatFloat(qemu.VMId, 'f', 0, 64))

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/current"
	data, err = qemu.Node.Proxmox.Get(target)
	if err != nil {
		return status, err
	}
	results = data["data"].(map[string]interface{})
	status = QemuStatus{
		CPU:       results["cpu"].(float64),
		CPUs:      results["cpus"].(float64),
		Mem:       results["mem"].(float64),
		MaxMem:    results["maxmem"].(float64),
		Disk:      results["disk"].(float64),
		MaxDisk:   results["maxdisk"].(float64),
		DiskWrite: results["diskwrite"].(float64),
		DiskRead:  results["diskread"].(float64),
		NetIn:     results["netin"].(float64),
		NetOut:    results["netout"].(float64),
		Uptime:    results["uptime"].(float64),
		QmpStatus: results["qmpstatus"].(string),
		Status:    results["status"].(string),
		Template:  results["template"].(string),
	}
	return status, nil
}

func (qemu QemuVM) WaitForStatus(status string, timeout int) error {
	var i int
	var err error
	var qStatus QemuStatus
	for i = 0; i < timeout; i++ {
		qStatus, err = qemu.CurrentStatus()
		if err != nil {
			return err
		}

		if qStatus.Status == status {
			return nil
		}
		time.Sleep(time.Second * 1)
	}
	return errors.New("Timeout reached")
}

func (qemu QemuVM) Start() error {
	var target string
	var err error

	//fmt.Println("!QemuStart ", strconv.FormatFloat(qemu.VMId, 'f', 0, 64))

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/start"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Stop() error {
	var target string
	var err error

	//fmt.Print("!QemuStop ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/stop"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Shutdown() (Task, error) {
	var target string
	var err error

	//fmt.Print("!QemuShutdown ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/shutdown"
	data, err := qemu.Node.Proxmox.Post(target, "")

	if err != err {
		return Task{}, err
	}

	t := Task{
		UPid:    data["data"].(string),
		proxmox: qemu.Node.Proxmox,
	}

	return t, err
}

func (qemu QemuVM) Suspend() error {
	var target string
	var err error

	//fmt.Print("!QemuSuspend ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/suspend"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Resume() error {
	var target string
	var err error

	//fmt.Print("!QemuResume ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/resume"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Clone(newId float64, name string, targetName string) (Task, error) {
	var target string
	var err error

	newVMID := strconv.FormatFloat(newId, 'f', 0, 64)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/clone"

	form := url.Values{
		"newid":  {newVMID},
		"name":   {name},
		"target": {targetName},
		"full":   {"1"},
	}

	data, err := qemu.Node.Proxmox.PostForm(target, form)
	if err != err {
		return Task{}, err
	}

	t := Task{
		UPid:    data["data"].(string),
		proxmox: qemu.Node.Proxmox,
	}

	return t, nil
}

func (qemu QemuVM) SetDescription(description string) (error) {
	var target string
	var err error

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/config"

	form := url.Values{
		"description":  {description},
	}

	_, err = qemu.Node.Proxmox.PutForm(target, form)
	if err != err {
		return err
	}

	return nil
}

func (qemu QemuVM) SetMemory(memory string) (error) {
	var target string
	var err error

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/config"

	form := url.Values{
		"memory":  {memory},
	}

	_, err = qemu.Node.Proxmox.PutForm(target, form)
	if err != err {
		return err
	}

	return nil
}

func (qemu QemuVM) SetIPSet(ip string) error {
	var target string
	var err error

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/firewall/options"

	form := url.Values{
		"dhcp":          {"1"},
		"enable":        {"1"},
		"log_level_in":  {"nolog"},
		"log_level_out": {"nolog"},
		"macfilter":     {"1"},
		"ipfilter":      {"1"},
		"ndp":           {"1"},
		"policy_in":     {"ACCEPT"},
		"policy_out":    {"ACCEPT"},
	}

	_, err = qemu.Node.Proxmox.PutForm(target, form)
	if err != nil {
		return err
	}

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/firewall/ipset"

	form = url.Values{
		"name": {"ipfilter-net0"},
	}

	_, err = qemu.Node.Proxmox.PostForm(target, form)
	if err != nil {
		return err
	}

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/firewall/ipset/ipfilter-net0"

	form = url.Values{
		"cidr": {ip},
	}

	_, err = qemu.Node.Proxmox.PostForm(target, form)
	if err != nil {
		return err
	}

	config, err := qemu.Config()
	if err != nil {
		return err
	}

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/config"

	net := ""

	for k, v := range config.Net["net0"] {
		net += k + "=" + v + ","
	}

	form = url.Values{
		"net0": {net + ",firewall=1"},
	}

	_, err = qemu.Node.Proxmox.PutForm(target, form)
	if err != nil {
		return err
	}

	return nil
}

func (qemu QemuVM) ResizeDisk(size string) error {
	var target string
	var err error

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/resize"

	form := url.Values{
		"disk": {"scsi1"},
		"size": {size + "G"},
	}

	_, err = qemu.Node.Proxmox.PutForm(target, form)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package raft

import (
	"sync/atomic"
	"time"
)

type ConsensusModule struct {
	passiveConsensusModule *passiveConsensusModule

	// -- State - these fields may be accessed concurrently
	stopped int32

	// -- Channels
	rpcChannel chan rpcTuple
	ticker     *time.Ticker

	// -- Control
	stopSignal chan struct{}
	stopError  *atomic.Value
}

// Initialize a consensus module with the given components and settings.
// A goroutine that handles consensus processing is created.
// All parameters are required and cannot be nil.
// Server ids are check using ValidateServerIds().
// Time settings is checked using ValidateTimeSettings().
func NewConsensusModule(
	persistentState PersistentState,
	log Log,
	rpcSender RpcSender,
	thisServerId ServerId,
	peerServerIds []ServerId,
	timeSettings TimeSettings,
) *ConsensusModule {
	pcm, _ := newPassiveConsensusModule(
		persistentState,
		log,
		rpcSender,
		thisServerId,
		peerServerIds,
		timeSettings,
	)

	rpcChannel := make(chan rpcTuple, RPC_CHANNEL_BUFFER_SIZE)
	ticker := time.NewTicker(timeSettings.TickerDuration)

	cm := &ConsensusModule{
		pcm,

		// -- State
		0,

		// -- Channels
		rpcChannel,
		ticker,

		// -- Control
		make(chan struct{}),
		&atomic.Value{},
	}

	// Start the go routine
	go cm.processor()

	return cm
}

// Check if the goroutine is stopped.
func (cm *ConsensusModule) IsStopped() bool {
	return atomic.LoadInt32(&cm.stopped) != 0
}

// Stop the consensus module asynchronously.
// This will stop the goroutine that does the processing.
// Safe to call even if the goroutine has stopped.
// Will panic if called more than once.
func (cm *ConsensusModule) StopAsync() {
	close(cm.stopSignal)
}

// Get the panic error value that stopped the goroutine.
// The value will be nil if the goroutine is not stopped, or stopped
// without an error, or  panicked with a nil value.
func (cm *ConsensusModule) GetStopError() interface{} {
	return cm.stopError.Load()
}

// Get the current server state
func (cm *ConsensusModule) GetServerState() ServerState {
	return cm.passiveConsensusModule.getServerState()
}

// Process the given RPC message from the given peer asynchronously.
// See comments in rpctypes.go for an explanation of the messaging model.
// This method sends the rpc to the ConsensusModule's goroutine.
// Sending an unknown or unexpected rpc message will cause the
// ConsensusModule goroutine to panic and stop.
func (cm *ConsensusModule) ProcessRpcAsync(from ServerId, rpc interface{}) {
	select {
	case cm.rpcChannel <- rpcTuple{from, rpc}:
	default:
		// FIXME
		panic("oops! rpcChannel is full!")
	}
}

// -- protected methods

func (cm *ConsensusModule) processor() {
	defer func() {
		// Recover & save the panic reason
		if r := recover(); r != nil {
			cm.stopError.Store(r)
		}
		// Mark the server as stopped
		atomic.StoreInt32(&cm.stopped, 1)
		// Clean up things
		close(cm.rpcChannel)
		cm.ticker.Stop()
		// TODO: call stop event listener(s)
	}()

loop:
	for {
		select {
		case rpc, ok := <-cm.rpcChannel:
			if !ok {
				panic("FATAL: rpcChannel closed")
			}
			cm.passiveConsensusModule.rpc(rpc.from, rpc.rpc)
		case now, ok := <-cm.ticker.C:
			if !ok {
				panic("FATAL: ticker channel closed")
			}
			cm.passiveConsensusModule.tick(now)
		case <-cm.stopSignal:
			break loop
		}
	}
}
<commit_msg>ProcessRpcAsync blocks when channel is full<commit_after>package raft

import (
	"sync/atomic"
	"time"
)

type ConsensusModule struct {
	passiveConsensusModule *passiveConsensusModule

	// -- State - these fields may be accessed concurrently
	stopped int32

	// -- Channels
	rpcChannel chan rpcTuple
	ticker     *time.Ticker

	// -- Control
	stopSignal chan struct{}
	stopError  *atomic.Value
}

// Initialize a consensus module with the given components and settings.
// A goroutine that handles consensus processing is created.
// All parameters are required and cannot be nil.
// Server ids are check using ValidateServerIds().
// Time settings is checked using ValidateTimeSettings().
func NewConsensusModule(
	persistentState PersistentState,
	log Log,
	rpcSender RpcSender,
	thisServerId ServerId,
	peerServerIds []ServerId,
	timeSettings TimeSettings,
) *ConsensusModule {
	pcm, _ := newPassiveConsensusModule(
		persistentState,
		log,
		rpcSender,
		thisServerId,
		peerServerIds,
		timeSettings,
	)

	rpcChannel := make(chan rpcTuple, RPC_CHANNEL_BUFFER_SIZE)
	ticker := time.NewTicker(timeSettings.TickerDuration)

	cm := &ConsensusModule{
		pcm,

		// -- State
		0,

		// -- Channels
		rpcChannel,
		ticker,

		// -- Control
		make(chan struct{}),
		&atomic.Value{},
	}

	// Start the go routine
	go cm.processor()

	return cm
}

// Check if the goroutine is stopped.
func (cm *ConsensusModule) IsStopped() bool {
	return atomic.LoadInt32(&cm.stopped) != 0
}

// Stop the consensus module asynchronously.
// This will stop the goroutine that does the processing.
// Safe to call even if the goroutine has stopped.
// Will panic if called more than once.
func (cm *ConsensusModule) StopAsync() {
	close(cm.stopSignal)
}

// Get the panic error value that stopped the goroutine.
// The value will be nil if the goroutine is not stopped, or stopped
// without an error, or  panicked with a nil value.
func (cm *ConsensusModule) GetStopError() interface{} {
	return cm.stopError.Load()
}

// Get the current server state
func (cm *ConsensusModule) GetServerState() ServerState {
	return cm.passiveConsensusModule.getServerState()
}

// Process the given RPC message from the given peer asynchronously.
// See comments in rpctypes.go for an explanation of the messaging model.
// This method sends the rpc to the ConsensusModule's goroutine.
// Sending an unknown or unexpected rpc message will cause the
// ConsensusModule goroutine to panic and stop.
// TODO: behavior when channel full?
func (cm *ConsensusModule) ProcessRpcAsync(from ServerId, rpc interface{}) {
	cm.rpcChannel <- rpcTuple{from, rpc}
}

// -- protected methods

func (cm *ConsensusModule) processor() {
	defer func() {
		// Recover & save the panic reason
		if r := recover(); r != nil {
			cm.stopError.Store(r)
		}
		// Mark the server as stopped
		atomic.StoreInt32(&cm.stopped, 1)
		// Clean up things
		close(cm.rpcChannel)
		cm.ticker.Stop()
		// TODO: call stop event listener(s)
	}()

loop:
	for {
		select {
		case rpc, ok := <-cm.rpcChannel:
			if !ok {
				panic("FATAL: rpcChannel closed")
			}
			cm.passiveConsensusModule.rpc(rpc.from, rpc.rpc)
		case now, ok := <-cm.ticker.C:
			if !ok {
				panic("FATAL: ticker channel closed")
			}
			cm.passiveConsensusModule.tick(now)
		case <-cm.stopSignal:
			break loop
		}
	}
}
<|endoftext|>


<commit_before>//统计出一个项目中每种编程语言所使用的数量。
package main

import (
	"flag"
	"fmt"
	"log"
	"path/filepath"
    "os"
)

var c_count int = 0
var cpp_count int = 0
var h_count int = 0
var java_count int = 0
var go_count int = 0
var dart_count int = 0

func visit(path string, fileInfo os.FileInfo, err error) error {
	if !fileInfo.IsDir() {
		counter(path,fileInfo)
	}
	return err
}

func counter(path string,file os.FileInfo) {
	extension := filepath.Ext(file.Name())
	switch extension {
	case ".c":
		c_count++
	case ".cpp" :
		cpp_count++
    case ".cc" :
		cpp_count++

	case ".java":
		java_count++
	case ".go":
		go_count++
	case ".dart":
		dart_count++
	}
}

func main() {
	flag.Parse()
	root := flag.Arg(0)
	err := filepath.Walk(root, visit)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("c : %d\ncpp : %d\njava : %d\ngo : %d\ndart : %d\n", c_count, cpp_count, java_count, go_count, dart_count)
}
<commit_msg>format<commit_after>//统计出一个项目中每种编程语言所使用的数量。
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
)

var c_count int = 0
var cpp_count int = 0
var h_count int = 0
var java_count int = 0
var go_count int = 0
var dart_count int = 0

func visit(path string, fileInfo os.FileInfo, err error) error {
	if !fileInfo.IsDir() {
		counter(path, fileInfo)
	}
	return err
}

func counter(path string, file os.FileInfo) {
	extension := filepath.Ext(file.Name())
	switch extension {
	case ".c":
		c_count++
	case ".cpp":
		cpp_count++
	case ".cc":
		cpp_count++

	case ".java":
		java_count++
	case ".go":
		go_count++
	case ".dart":
		dart_count++
	}
}

func main() {
	flag.Parse()
	root := flag.Arg(0)
	err := filepath.Walk(root, visit)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("c : %d\ncpp : %d\njava : %d\ngo : %d\ndart : %d\n", c_count, cpp_count, java_count, go_count, dart_count)
}
<|endoftext|>


<commit_before>package dual

import (
	"fmt"
	"math"
	"strings"
)

// A Real represents a dual real number as an ordered array of two float64
// values.
type Real [2]float64

// String returns the string version of a Real value. If z = a + bε, then the
// string is "(a+bε)", similar to complex128 values.
func (z *Real) String() string {
	a := make([]string, 5)
	a[0] = "("
	a[1] = fmt.Sprintf("%g", z[0])
	switch {
	case math.Signbit(z[1]):
		a[2] = fmt.Sprintf("%g", z[1])
	case math.IsInf(z[1], +1):
		a[2] = "+Inf"
	default:
		a[2] = fmt.Sprintf("+%g", z[1])
	}
	a[3] = "ε"
	a[4] = ")"
	return strings.Join(a, "")
}

// Equals returns true if z and y are equal.
func (z *Real) Equals(y *Real) bool {
	for i := range z {
		if notEquals(z[i], y[i]) {
			return false
		}
	}
	return true
}

// Copy copies y onto z, and returns z.
func (z *Real) Copy(y *Real) *Real {
	for i, v := range y {
		z[i] = v
	}
	return z
}

// NewReal returns a pointer to a Real value made from two given float64 values.
func NewReal(a, b float64) *Real {
	z := new(Real)
	z[0] = a
	z[1] = b
	return z
}

// IsRealInf returns true if any of the components of z are infinite.
func (z *Real) IsRealInf() bool {
	for _, v := range z {
		if math.IsInf(v, 0) {
			return true
		}
	}
	return false
}

// RealInf returns a pointer to a dual real infinity value.
func RealInf(a, b int) *Real {
	z := new(Real)
	z[0] = math.Inf(a)
	z[1] = math.Inf(b)
	return z
}

// IsRealNaN returns true if any component of z is NaN and neither is an
// infinity.
func (z *Real) IsRealNaN() bool {
	for _, v := range z {
		if math.IsInf(v, 0) {
			return false
		}
	}
	for _, v := range z {
		if math.IsNaN(v) {
			return true
		}
	}
	return false
}

// RealNaN returns a pointer to a dual real NaN value.
func RealNaN() *Real {
	nan := math.NaN()
	return &Real{nan, nan}
}

// Scal sets z equal to y scaled by a, and returns z.
func (z *Real) Scal(y *Real, a float64) *Real {
	for i, v := range y {
		z[i] = a * v
	}
	return z
}

// Neg sets z equal to the negative of y, and returns z.
func (z *Real) Neg(y *Real) *Real {
	return z.Scal(y, -1)
}

// DualConj sets z equal to the dual conjugate of y, and returns z.
func (z *Real) DualConj(y *Real) *Real {
	z[0] = +y[0]
	z[1] = -y[1]
	return z
}

// Add sets z equal to the sum of x and y, and returns z.
func (z *Real) Add(x, y *Real) *Real {
	for i, v := range x {
		z[i] = v + y[i]
	}
	return z
}

// Sub sets z equal to the difference of x and y, and returns z.
func (z *Real) Sub(x, y *Real) *Real {
	for i, v := range x {
		z[i] = v - y[i]
	}
	return z
}

// Mul sets z equal to the product of x and y, and returns z.
//
// The basic rule is:
// 		ε * ε = 0
// This multiplication operation is commutative and associative.
func (z *Real) Mul(x, y *Real) *Real {
	p := new(Real).Copy(x)
	q := new(Real).Copy(y)
	z[0] = p[0] * q[0]
	z[1] = (p[0] * q[1]) + (p[1] * q[0])
	return z
}

// DualQuad returns the non-negative dual quadrance of z, a float64 value.
func (z *Real) DualQuad() float64 {
	return z[0] * z[0]
}

// IsZeroDiv returns true if z is a zero divisor. This is equivalent to
// z being nilpotent (i.e. z² = 0).
func (z *Real) IsZeroDiv() bool {
	return !notEquals(z[0], 0)
}

// Inv sets z equal to the inverse of y, and returns z. If y is a zero divisor,
// then Inv panics.
func (z *Real) Inv(y *Real) *Real {
	if y.IsZeroDiv() {
		panic("zero divisor")
	}
	return z.Scal(new(Real).DualConj(y), 1/y.DualQuad())
}

// Quo sets z equal to the quotient of x and y, and returns z. If y is a zero
// divisor, then Quo panics.
func (z *Real) Quo(x, y *Real) *Real {
	if y.IsZeroDiv() {
		panic("zero divisor denominator")
	}
	return z.Scal(new(Real).Mul(x, new(Real).DualConj(y)), 1/y.DualQuad())
}

// Sin sets z equal to the dual sine of y, and returns z.
func (z *Real) Sin(y *Real) *Real {
	s, c := math.Sincos(y[0])
	z[0] = s
	z[1] = y[1] * c
	return z
}

// Cos sets z equal to the dual cosine of y, and returns z.
func (z *Real) Cos(y *Real) *Real {
	s, c := math.Sincos(y[0])
	z[0] = c
	z[1] = -y[1] * s
	return z
}

// Exp sets z equal to the dual exponential of y, and returns z.
func (z *Real) Exp(y *Real) *Real {
	e := math.Exp(y[0])
	z[0] = e
	z[1] = y[1] * e
	return z
}

// Sinh sets z equal to the dual hyperbolic sine of y, and returns z.
func (z *Real) Sinh(y *Real) *Real {
	z[0] = math.Sinh(y[0])
	z[1] = y[1] * math.Cosh(y[0])
	return z
}

// Cosh sets z equal to the dual hyperbolic cosine of y, and returns z.
func (z *Real) Cosh(y *Real) *Real {
	z[0] = math.Cosh(y[0])
	z[1] = y[1] * math.Sinh(y[0])
	return z
}
<commit_msg>Remove some for loops in real.go<commit_after>package dual

import (
	"fmt"
	"math"
	"strings"
)

// A Real represents a dual real number as an ordered array of two float64
// values.
type Real [2]float64

// String returns the string version of a Real value. If z = a + bε, then the
// string is "(a+bε)", similar to complex128 values.
func (z *Real) String() string {
	a := make([]string, 5)
	a[0] = "("
	a[1] = fmt.Sprintf("%g", z[0])
	switch {
	case math.Signbit(z[1]):
		a[2] = fmt.Sprintf("%g", z[1])
	case math.IsInf(z[1], +1):
		a[2] = "+Inf"
	default:
		a[2] = fmt.Sprintf("+%g", z[1])
	}
	a[3] = "ε"
	a[4] = ")"
	return strings.Join(a, "")
}

// Equals returns true if z and y are equal.
func (z *Real) Equals(y *Real) bool {
	if notEquals(z[0], y[0]) || notEquals(z[1], y[1]) {
		return false
	}
	return true
}

// Copy copies y onto z, and returns z.
func (z *Real) Copy(y *Real) *Real {
	z[0] = y[0]
	z[1] = y[1]
	return z
}

// NewReal returns a pointer to a Real value made from two given float64 values.
func NewReal(a, b float64) *Real {
	z := new(Real)
	z[0] = a
	z[1] = b
	return z
}

// IsRealInf returns true if any of the components of z are infinite.
func (z *Real) IsRealInf() bool {
	if math.IsInf(z[0], 0) || math.IsInf(z[1], 0) {
		return true
	}
	return false
}

// RealInf returns a pointer to a dual real infinity value.
func RealInf(a, b int) *Real {
	z := new(Real)
	z[0] = math.Inf(a)
	z[1] = math.Inf(b)
	return z
}

// IsRealNaN returns true if any component of z is NaN and neither is an
// infinity.
func (z *Real) IsRealNaN() bool {
	if math.IsInf(z[0], 0) || math.IsInf(z[1], 0) {
		return false
	}
	if math.IsNaN(z[0]) || math.IsNaN(z[1]) {
		return true
	}
	return false
}

// RealNaN returns a pointer to a dual real NaN value.
func RealNaN() *Real {
	nan := math.NaN()
	return &Real{nan, nan}
}

// Scal sets z equal to y scaled by a, and returns z.
func (z *Real) Scal(y *Real, a float64) *Real {
	z[0] = y[0] * a
	z[1] = y[1] * a
	return z
}

// Neg sets z equal to the negative of y, and returns z.
func (z *Real) Neg(y *Real) *Real {
	return z.Scal(y, -1)
}

// DualConj sets z equal to the dual conjugate of y, and returns z.
func (z *Real) DualConj(y *Real) *Real {
	z[0] = +y[0]
	z[1] = -y[1]
	return z
}

// Add sets z equal to the sum of x and y, and returns z.
func (z *Real) Add(x, y *Real) *Real {
	z[0] = x[0] + y[0]
	z[1] = x[1] + y[1]
	return z
}

// Sub sets z equal to the difference of x and y, and returns z.
func (z *Real) Sub(x, y *Real) *Real {
	z[0] = x[0] - y[0]
	z[1] = x[1] - y[1]
	return z
}

// Mul sets z equal to the product of x and y, and returns z.
//
// The basic rule is:
// 		ε * ε = 0
// This multiplication operation is commutative and associative.
func (z *Real) Mul(x, y *Real) *Real {
	p := new(Real).Copy(x)
	q := new(Real).Copy(y)
	z[0] = p[0] * q[0]
	z[1] = (p[0] * q[1]) + (p[1] * q[0])
	return z
}

// DualQuad returns the non-negative dual quadrance of z, a float64 value.
func (z *Real) DualQuad() float64 {
	return z[0] * z[0]
}

// IsZeroDiv returns true if z is a zero divisor. This is equivalent to
// z being nilpotent (i.e. z² = 0).
func (z *Real) IsZeroDiv() bool {
	return !notEquals(z[0], 0)
}

// Inv sets z equal to the inverse of y, and returns z. If y is a zero divisor,
// then Inv panics.
func (z *Real) Inv(y *Real) *Real {
	if y.IsZeroDiv() {
		panic("zero divisor")
	}
	return z.Scal(new(Real).DualConj(y), 1/y.DualQuad())
}

// Quo sets z equal to the quotient of x and y, and returns z. If y is a zero
// divisor, then Quo panics.
func (z *Real) Quo(x, y *Real) *Real {
	if y.IsZeroDiv() {
		panic("zero divisor denominator")
	}
	return z.Scal(new(Real).Mul(x, new(Real).DualConj(y)), 1/y.DualQuad())
}

// Sin sets z equal to the dual sine of y, and returns z.
func (z *Real) Sin(y *Real) *Real {
	s, c := math.Sincos(y[0])
	z[0] = s
	z[1] = y[1] * c
	return z
}

// Cos sets z equal to the dual cosine of y, and returns z.
func (z *Real) Cos(y *Real) *Real {
	s, c := math.Sincos(y[0])
	z[0] = c
	z[1] = -y[1] * s
	return z
}

// Exp sets z equal to the dual exponential of y, and returns z.
func (z *Real) Exp(y *Real) *Real {
	e := math.Exp(y[0])
	z[0] = e
	z[1] = y[1] * e
	return z
}

// Sinh sets z equal to the dual hyperbolic sine of y, and returns z.
func (z *Real) Sinh(y *Real) *Real {
	z[0] = math.Sinh(y[0])
	z[1] = y[1] * math.Cosh(y[0])
	return z
}

// Cosh sets z equal to the dual hyperbolic cosine of y, and returns z.
func (z *Real) Cosh(y *Real) *Real {
	z[0] = math.Cosh(y[0])
	z[1] = y[1] * math.Sinh(y[0])
	return z
}
<|endoftext|>


<commit_before>// Copyright 2020 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package spannerdriver

import (
	"database/sql/driver"
	"encoding/base64"
	"io"
	"log"
	"sync"
	"time"

	"cloud.google.com/go/spanner"
	"google.golang.org/api/iterator"
	sppb "google.golang.org/genproto/googleapis/spanner/v1"
)

type rows struct {
	it *spanner.RowIterator

	colsOnce sync.Once
	cols     []string

	dirtyRow *spanner.Row
}

// Columns returns the names of the columns. The number of
// columns of the result is inferred from the length of the
// slice. If a particular column name isn't known, an empty
// string should be returned for that entry.
func (r *rows) Columns() []string {
	r.getColumns()
	return r.cols
}

// Close closes the rows iterator.
func (r *rows) Close() error {
	r.it.Stop()
	return nil
}

func (r *rows) getColumns() {
	r.colsOnce.Do(func() {
		row, err := r.it.Next()
		if err != nil {
			log.Println(err)
			return
		}
		r.dirtyRow = row
		r.cols = row.ColumnNames()
	})
}

// Next is called to populate the next row of data into
// the provided slice. The provided slice will be the same
// size as the Columns() are wide.
//
// Next should return io.EOF when there are no more rows.
//
// The dest should not be written to outside of Next. Care
// should be taken when closing Rows not to modify
// a buffer held in dest.
func (r *rows) Next(dest []driver.Value) error {
	r.getColumns()
	var row *spanner.Row
	if r.dirtyRow != nil {
		row = r.dirtyRow
		r.dirtyRow = nil
	} else {
		var err error
		row, err = r.it.Next() // returns io.EOF when there is no next
		if err == iterator.Done {
			return io.EOF
		}
		if err != nil {
			return err
		}
	}

	for i := 0; i < row.Size(); i++ {
		var col spanner.GenericColumnValue
		if err := row.Column(i, &col); err != nil {
			return err
		}
		switch col.Type.Code {
		case sppb.TypeCode_INT64:
			var v spanner.NullInt64
			if err := col.Decode(&v); err != nil {
				return err
			}
			dest[i] = v.Int64
		case sppb.TypeCode_FLOAT64:
			var v spanner.NullFloat64
			if err := col.Decode(&v); err != nil {
				return err
			}
			dest[i] = v.Float64
		case sppb.TypeCode_STRING:
			var v spanner.NullString
			if err := col.Decode(&v); err != nil {
				return err
			}
			dest[i] = v.StringVal
		case sppb.TypeCode_BYTES:
			// The column value is a base64 encoded string.
			var v spanner.NullString
			if err := col.Decode(&v); err != nil {
				return err
			}
			if v.IsNull() {
				dest[i] = []byte(nil)
			} else {
				b, err := base64.StdEncoding.DecodeString(v.StringVal)
				if err != nil {
					return err
				}
				dest[i] = b
			}
		case sppb.TypeCode_BOOL:
			var v spanner.NullBool
			if err := col.Decode(&v); err != nil {
				return err
			}
			dest[i] = v.Bool
		case sppb.TypeCode_DATE:
			var v spanner.NullDate
			if err := col.Decode(&v); err != nil {
				return err
			}
			if v.IsNull() {
				dest[i] = v.Date // typed nil
			} else {
				dest[i] = v.Date.In(time.Local) // TODO(jbd): Add note about this.
			}
		case sppb.TypeCode_TIMESTAMP:
			var v spanner.NullTime
			if err := col.Decode(&v); err != nil {
				return err
			}
			dest[i] = v.Time
		}
		// TODO(jbd): Implement other types.
		// How to handle array and struct?
	}
	return nil
}
<commit_msg>fix: use correct type for decoding bytes<commit_after>// Copyright 2020 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package spannerdriver

import (
	"database/sql/driver"
	"io"
	"log"
	"sync"
	"time"

	"cloud.google.com/go/spanner"
	"google.golang.org/api/iterator"
	sppb "google.golang.org/genproto/googleapis/spanner/v1"
)

type rows struct {
	it *spanner.RowIterator

	colsOnce sync.Once
	cols     []string

	dirtyRow *spanner.Row
}

// Columns returns the names of the columns. The number of
// columns of the result is inferred from the length of the
// slice. If a particular column name isn't known, an empty
// string should be returned for that entry.
func (r *rows) Columns() []string {
	r.getColumns()
	return r.cols
}

// Close closes the rows iterator.
func (r *rows) Close() error {
	r.it.Stop()
	return nil
}

func (r *rows) getColumns() {
	r.colsOnce.Do(func() {
		row, err := r.it.Next()
		if err != nil {
			log.Println(err)
			return
		}
		r.dirtyRow = row
		r.cols = row.ColumnNames()
	})
}

// Next is called to populate the next row of data into
// the provided slice. The provided slice will be the same
// size as the Columns() are wide.
//
// Next should return io.EOF when there are no more rows.
//
// The dest should not be written to outside of Next. Care
// should be taken when closing Rows not to modify
// a buffer held in dest.
func (r *rows) Next(dest []driver.Value) error {
	r.getColumns()
	var row *spanner.Row
	if r.dirtyRow != nil {
		row = r.dirtyRow
		r.dirtyRow = nil
	} else {
		var err error
		row, err = r.it.Next() // returns io.EOF when there is no next
		if err == iterator.Done {
			return io.EOF
		}
		if err != nil {
			return err
		}
	}

	for i := 0; i < row.Size(); i++ {
		var col spanner.GenericColumnValue
		if err := row.Column(i, &col); err != nil {
			return err
		}
		switch col.Type.Code {
		case sppb.TypeCode_INT64:
			var v spanner.NullInt64
			if err := col.Decode(&v); err != nil {
				return err
			}
			dest[i] = v.Int64
		case sppb.TypeCode_FLOAT64:
			var v spanner.NullFloat64
			if err := col.Decode(&v); err != nil {
				return err
			}
			dest[i] = v.Float64
		case sppb.TypeCode_STRING:
			var v spanner.NullString
			if err := col.Decode(&v); err != nil {
				return err
			}
			dest[i] = v.StringVal
		case sppb.TypeCode_BYTES:
			// The column value is a base64 encoded string.
			var v []byte
			if err := col.Decode(&v); err != nil {
				return err
			}
			dest[i] = v
		case sppb.TypeCode_BOOL:
			var v spanner.NullBool
			if err := col.Decode(&v); err != nil {
				return err
			}
			dest[i] = v.Bool
		case sppb.TypeCode_DATE:
			var v spanner.NullDate
			if err := col.Decode(&v); err != nil {
				return err
			}
			if v.IsNull() {
				dest[i] = v.Date // typed nil
			} else {
				dest[i] = v.Date.In(time.Local) // TODO(jbd): Add note about this.
			}
		case sppb.TypeCode_TIMESTAMP:
			var v spanner.NullTime
			if err := col.Decode(&v); err != nil {
				return err
			}
			dest[i] = v.Time
		}
		// TODO(jbd): Implement other types.
		// How to handle array and struct?
	}
	return nil
}
<|endoftext|>


<commit_before>/*

Package rpio provides GPIO access on the Raspberry PI without any need
for external c libraries (ex: WiringPI or BCM2835).

Supports simple operations such as:
- Pin mode/direction (input/output)
- Pin write (high/low)
- Pin read (high/low)
- Pull up/down/off

Example of use:

	rpio.Open()
	defer rpio.Close()

	pin := rpio.Pin(4)
	pin.Output()

	for {
		pin.Toggle()
		time.Sleep(time.Second)
	}

The library use the raw BCM2835 pinouts, not the ports as they are mapped
on the output pins for the raspberry pi

   Rev 1 Raspberry Pi
+------+------+--------+
| GPIO | Phys | Name   |
+------+------+--------+
|   0  |   3  | SDA    |
|   1  |   5  | SCL    |
|   4  |   7  | GPIO 7 |
|   7  |  26  | CE1    |
|   8  |  24  | CE0    |
|   9  |  21  | MISO   |
|  10  |  19  | MOSI   |
|  11  |  23  | SCLK   |
|  14  |   8  | TxD    |
|  15  |  10  | RxD    |
|  17  |  11  | GPIO 0 |
|  18  |  12  | GPIO 1 |
|  21  |  13  | GPIO 2 |
|  22  |  15  | GPIO 3 |
|  23  |  16  | GPIO 4 |
|  24  |  18  | GPIO 5 |
|  25  |  22  | GPIO 6 |
+------+------+--------+

See the spec for full details of the BCM2835 controller:
http://www.raspberrypi.org/wp-content/uploads/2012/02/BCM2835-ARM-Peripherals.pdf

*/

package rpio

import (
	"bytes"
	"encoding/binary"
	"os"
	"reflect"
	"sync"
	"syscall"
	"time"
	"unsafe"
)

type Direction uint8
type Pin uint8
type State uint8
type Pull uint8

// Memory offsets for gpio, see the spec for more details
const (
	bcm2835Base = 0x20000000
	pi1GPIOBase = bcm2835Base + 0x200000
	memLength   = 4096

	pinMask uint32 = 7 // 0b111 - pinmode is 3 bits
)

// Pin direction, a pin can be set in Input or Output mode
const (
	Input Direction = iota
	Output
)

// State of pin, High / Low
const (
	Low State = iota
	High
)

// Pull Up / Down / Off
const (
	PullOff Pull = iota
	PullDown
	PullUp
)

// Arrays for 8 / 32 bit access to memory and a semaphore for write locking
var (
	memlock sync.Mutex
	mem     []uint32
	mem8    []uint8
)

// Set pin as Input
func (pin Pin) Input() {
	PinMode(pin, Input)
}

// Set pin as Output
func (pin Pin) Output() {
	PinMode(pin, Output)
}

// Set pin High
func (pin Pin) High() {
	WritePin(pin, High)
}

// Set pin Low
func (pin Pin) Low() {
	WritePin(pin, Low)
}

// Toggle pin state
func (pin Pin) Toggle() {
	TogglePin(pin)
}

// Set pin Direction
func (pin Pin) Mode(dir Direction) {
	PinMode(pin, dir)
}

// Set pin state (high/low)
func (pin Pin) Write(state State) {
	WritePin(pin, state)
}

// Read pin state (high/low)
func (pin Pin) Read() State {
	return ReadPin(pin)
}

// Set a given pull up/down mode
func (pin Pin) Pull(pull Pull) {
	PullMode(pin, pull)
}

// Pull up pin
func (pin Pin) PullUp() {
	PullMode(pin, PullUp)
}

// Pull down pin
func (pin Pin) PullDown() {
	PullMode(pin, PullDown)
}

// Disable pullup/down on pin
func (pin Pin) PullOff() {
	PullMode(pin, PullOff)
}

// PinMode sets the direction of a given pin (Input or Output)
func PinMode(pin Pin, direction Direction) {

	// Pin fsel register, 0 or 1 depending on bank
	fsel := uint8(pin) / 10
	shift := (uint8(pin) % 10) * 3

	memlock.Lock()
	defer memlock.Unlock()

	if direction == Input {
		mem[fsel] = mem[fsel] &^ (pinMask << shift)
	} else {
		mem[fsel] = (mem[fsel] &^ (pinMask << shift)) | (1 << shift)
	}

}

// WritePin sets a given pin High or Low
// by setting the clear or set registers respectively
func WritePin(pin Pin, state State) {

	p := uint8(pin)

	// Clear register, 10 / 11 depending on bank
	// Set register, 7 / 8 depending on bank
	clearReg := p/32 + 10
	setReg := p/32 + 7

	memlock.Lock()
	defer memlock.Unlock()

	if state == Low {
		mem[clearReg] = 1 << (p & 31)
	} else {
		mem[setReg] = 1 << (p & 31)
	}

}

// Read the state of a pin
func ReadPin(pin Pin) State {
	// Input level register offset (13 / 14 depending on bank)
	levelReg := uint8(pin)/32 + 13

	if (mem[levelReg] & (1 << uint8(pin))) != 0 {
		return High
	}

	return Low
}

// Toggle a pin state (high -> low -> high)
// TODO: probably possible to do this much faster without read
func TogglePin(pin Pin) {
	switch ReadPin(pin) {
	case Low:
		pin.High()
	case High:
		pin.Low()
	}
}

func PullMode(pin Pin, pull Pull) {
	// Pull up/down/off register has offset 38 / 39, pull is 37
	pullClkReg := uint8(pin)/32 + 38
	pullReg := 37
	shift := (uint8(pin) % 32)

	memlock.Lock()
	defer memlock.Unlock()

	switch pull {
	case PullDown, PullUp:
		mem[pullReg] = mem[pullReg]&^3 | uint32(pull)
	case PullOff:
		mem[pullReg] = mem[pullReg] &^ 3
	}

	// Wait for value to clock in, this is ugly, sorry :(
	time.Sleep(time.Microsecond)

	mem[pullClkReg] = 1 << shift

	// Wait for value to clock in
	time.Sleep(time.Microsecond)

	mem[pullReg] = mem[pullReg] &^ 3
	mem[pullClkReg] = 0

}

// Open and memory map GPIO memory range from /dev/mem .
// Some reflection magic is used to convert it to a unsafe []uint32 pointer
func Open() (err error) {
	var file *os.File
	var base int64

	// Open fd for rw mem access; try gpiomem first
	file, err = os.OpenFile("/dev/gpiomem", os.O_RDWR|os.O_SYNC, 0)
	if !os.IsNotExist(err) {
		return
	}
	file, err = os.OpenFile("/dev/mem", os.O_RDWR|os.O_SYNC, 0)
	if err != nil {
		return
	}
	// FD can be closed after memory mapping
	defer file.Close()

	base = getGPIOBase()

	memlock.Lock()
	defer memlock.Unlock()

	// Memory map GPIO registers to byte array
	mem8, err = syscall.Mmap(
		int(file.Fd()),
		base,
		memLength,
		syscall.PROT_READ|syscall.PROT_WRITE,
		syscall.MAP_SHARED,
	)

	if err != nil {
		return
	}

	// Convert mapped byte memory to unsafe []uint32 pointer, adjust length as needed
	header := *(*reflect.SliceHeader)(unsafe.Pointer(&mem8))
	header.Len /= (32 / 8) // (32 bit = 4 bytes)
	header.Cap /= (32 / 8)

	mem = *(*[]uint32)(unsafe.Pointer(&header))

	return nil
}

// Close unmaps GPIO memory
func Close() error {
	memlock.Lock()
	defer memlock.Unlock()
	return syscall.Munmap(mem8)
}

// Read /proc/device-tree/soc/ranges and determine the base address.
// Use the default Raspberry Pi 1 base address if this fails.
func getGPIOBase() (base int64) {
	base = pi1GPIOBase
	ranges, err := os.Open("/proc/device-tree/soc/ranges")
	defer ranges.Close()
	if err != nil {
		return
	}
	b := make([]byte, 4)
	n, err := ranges.ReadAt(b, 4)
	if n != 4 || err != nil {
		return
	}
	buf := bytes.NewReader(b)
	var out uint32
	err = binary.Read(buf, binary.BigEndian, &out)
	if err != nil {
		return
	}
	return int64(out + 0x200000)
}
<commit_msg>Add pin mode Clock<commit_after>/*

Package rpio provides GPIO access on the Raspberry PI without any need
for external c libraries (ex: WiringPI or BCM2835).

Supports simple operations such as:
- Pin mode/direction (input/output/clock)
- Pin write (high/low)
- Pin read (high/low)
- Pull up/down/off

Example of use:

	rpio.Open()
	defer rpio.Close()

	pin := rpio.Pin(4)
	pin.Output()

	for {
		pin.Toggle()
		time.Sleep(time.Second)
	}

The library use the raw BCM2835 pinouts, not the ports as they are mapped
on the output pins for the raspberry pi

   Rev 1 Raspberry Pi
+------+------+--------+
| GPIO | Phys | Name   |
+------+------+--------+
|   0  |   3  | SDA    |
|   1  |   5  | SCL    |
|   4  |   7  | GPIO 7 |
|   7  |  26  | CE1    |
|   8  |  24  | CE0    |
|   9  |  21  | MISO   |
|  10  |  19  | MOSI   |
|  11  |  23  | SCLK   |
|  14  |   8  | TxD    |
|  15  |  10  | RxD    |
|  17  |  11  | GPIO 0 |
|  18  |  12  | GPIO 1 |
|  21  |  13  | GPIO 2 |
|  22  |  15  | GPIO 3 |
|  23  |  16  | GPIO 4 |
|  24  |  18  | GPIO 5 |
|  25  |  22  | GPIO 6 |
+------+------+--------+

See the spec for full details of the BCM2835 controller:
http://www.raspberrypi.org/wp-content/uploads/2012/02/BCM2835-ARM-Peripherals.pdf

*/

package rpio

import (
	"bytes"
	"encoding/binary"
	"os"
	"reflect"
	"sync"
	"syscall"
	"time"
	"unsafe"
)

type Mode uint8
type Pin uint8
type State uint8
type Pull uint8

// Memory offsets for gpio, see the spec for more details
const (
	bcm2835Base = 0x20000000
	pi1GPIOBase = bcm2835Base + 0x200000
	memLength   = 4096

	pinMask uint32 = 7 // 0b111 - pinmode is 3 bits
)

// Pin mode, a pin can be set in Input or Output mode, or clock
const (
	Input Mode = iota
	Output
	Clock
)

// State of pin, High / Low
const (
	Low State = iota
	High
)

// Pull Up / Down / Off
const (
	PullOff Pull = iota
	PullDown
	PullUp
)

// Arrays for 8 / 32 bit access to memory and a semaphore for write locking
var (
	memlock sync.Mutex
	mem     []uint32
	mem8    []uint8
)

// Set pin as Input
func (pin Pin) Input() {
	PinMode(pin, Input)
}

// Set pin as Output
func (pin Pin) Output() {
	PinMode(pin, Output)
}

// Set pin as Clock
func (pin Pin) Clock() {
	PinMode(pin, Clock)
}

// Set pin High
func (pin Pin) High() {
	WritePin(pin, High)
}

// Set pin Low
func (pin Pin) Low() {
	WritePin(pin, Low)
}

// Toggle pin state
func (pin Pin) Toggle() {
	TogglePin(pin)
}

// Set pin Mode
func (pin Pin) Mode(mode Mode) {
	PinMode(pin, mode)
}

// Set pin state (high/low)
func (pin Pin) Write(state State) {
	WritePin(pin, state)
}

// Read pin state (high/low)
func (pin Pin) Read() State {
	return ReadPin(pin)
}

// Set a given pull up/down mode
func (pin Pin) Pull(pull Pull) {
	PullMode(pin, pull)
}

// Pull up pin
func (pin Pin) PullUp() {
	PullMode(pin, PullUp)
}

// Pull down pin
func (pin Pin) PullDown() {
	PullMode(pin, PullDown)
}

// Disable pullup/down on pin
func (pin Pin) PullOff() {
	PullMode(pin, PullOff)
}

// PinMode sets the mode (direction) of a given pin (Input, Output or Clock)
// Clock is possible only for some pins (bcm 4, 5, 6)
func PinMode(pin Pin, mode Mode) {

	// Pin fsel register, 0 or 1 depending on bank
	fsel := uint8(pin) / 10
	shift := (uint8(pin) % 10) * 3
	f := uint32(0)

	memlock.Lock()
	defer memlock.Unlock()

	switch mode {
	case Input:
		f = 0 // 000
	case Output:
		f = 1 // 001
	case Clock:
		switch pin {
		case 4, 5, 6, 32, 34, 42, 43, 44:
			f = 4 // 100 - alt0
		case 20, 21:
			f = 2 // 010 - alt5
		default:
			f = 1 // 001 - fallback to output
		}
	}
	mem[fsel] = (mem[fsel] &^ (pinMask << shift)) | (f << shift)
}

// WritePin sets a given pin High or Low
// by setting the clear or set registers respectively
func WritePin(pin Pin, state State) {

	p := uint8(pin)

	// Clear register, 10 / 11 depending on bank
	// Set register, 7 / 8 depending on bank
	clearReg := p/32 + 10
	setReg := p/32 + 7

	memlock.Lock()
	defer memlock.Unlock()

	if state == Low {
		mem[clearReg] = 1 << (p & 31)
	} else {
		mem[setReg] = 1 << (p & 31)
	}

}

// Read the state of a pin
func ReadPin(pin Pin) State {
	// Input level register offset (13 / 14 depending on bank)
	levelReg := uint8(pin)/32 + 13

	if (mem[levelReg] & (1 << uint8(pin))) != 0 {
		return High
	}

	return Low
}

// Toggle a pin state (high -> low -> high)
// TODO: probably possible to do this much faster without read
func TogglePin(pin Pin) {
	switch ReadPin(pin) {
	case Low:
		pin.High()
	case High:
		pin.Low()
	}
}

func PullMode(pin Pin, pull Pull) {
	// Pull up/down/off register has offset 38 / 39, pull is 37
	pullClkReg := uint8(pin)/32 + 38
	pullReg := 37
	shift := (uint8(pin) % 32)

	memlock.Lock()
	defer memlock.Unlock()

	switch pull {
	case PullDown, PullUp:
		mem[pullReg] = mem[pullReg]&^3 | uint32(pull)
	case PullOff:
		mem[pullReg] = mem[pullReg] &^ 3
	}

	// Wait for value to clock in, this is ugly, sorry :(
	time.Sleep(time.Microsecond)

	mem[pullClkReg] = 1 << shift

	// Wait for value to clock in
	time.Sleep(time.Microsecond)

	mem[pullReg] = mem[pullReg] &^ 3
	mem[pullClkReg] = 0

}

// Open and memory map GPIO memory range from /dev/mem .
// Some reflection magic is used to convert it to a unsafe []uint32 pointer
func Open() (err error) {
	var file *os.File
	var base int64

	// Open fd for rw mem access; try gpiomem first
	file, err = os.OpenFile("/dev/gpiomem", os.O_RDWR|os.O_SYNC, 0)
	if !os.IsNotExist(err) {
		return
	}
	file, err = os.OpenFile("/dev/mem", os.O_RDWR|os.O_SYNC, 0)
	if err != nil {
		return
	}
	// FD can be closed after memory mapping
	defer file.Close()

	base = getGPIOBase()

	memlock.Lock()
	defer memlock.Unlock()

	// Memory map GPIO registers to byte array
	mem8, err = syscall.Mmap(
		int(file.Fd()),
		base,
		memLength,
		syscall.PROT_READ|syscall.PROT_WRITE,
		syscall.MAP_SHARED,
	)

	if err != nil {
		return
	}

	// Convert mapped byte memory to unsafe []uint32 pointer, adjust length as needed
	header := *(*reflect.SliceHeader)(unsafe.Pointer(&mem8))
	header.Len /= (32 / 8) // (32 bit = 4 bytes)
	header.Cap /= (32 / 8)

	mem = *(*[]uint32)(unsafe.Pointer(&header))

	return nil
}

// Close unmaps GPIO memory
func Close() error {
	memlock.Lock()
	defer memlock.Unlock()
	return syscall.Munmap(mem8)
}

// Read /proc/device-tree/soc/ranges and determine the base address.
// Use the default Raspberry Pi 1 base address if this fails.
func getGPIOBase() (base int64) {
	base = pi1GPIOBase
	ranges, err := os.Open("/proc/device-tree/soc/ranges")
	defer ranges.Close()
	if err != nil {
		return
	}
	b := make([]byte, 4)
	n, err := ranges.ReadAt(b, 4)
	if n != 4 || err != nil {
		return
	}
	buf := bytes.NewReader(b)
	var out uint32
	err = binary.Read(buf, binary.BigEndian, &out)
	if err != nil {
		return
	}
	return int64(out + 0x200000)
}
<|endoftext|>


<commit_before>package main

import (
	"debug/pe"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"os"
	"reflect"
)

type ImageResourceDirectory struct {
	Characteristics      uint32
	TimeDateStamp        uint32
	MajorVersion         uint16
	MinorVersion         uint16
	NumberOfNamedEntries uint16
	NumberOfIdEntries    uint16
}

type ImageResourceDirectoryEntry struct {
	NameOrId     uint32
	OffsetToData uint32
}

type ImageResourceDataEntry struct {
	OffsetToData uint32
	Size1        uint32
	CodePage     uint32
	Reserved     uint32
}

type RelocationEntry struct {
	RVA         uint32 // "offset within the Section's raw data where the address starts."
	SymbolIndex uint32 // "(zero based) index in the Symbol table to which the reference refers."
	Type        uint16
}

type Symbol struct {
	Name           [8]byte
	Value          uint32
	SectionNumber  uint16
	Type           uint16
	StorageClass   uint8
	AuxiliaryCount uint8
}

type StringsHeader struct {
	Length uint32
}

const (
	MASK_SUBDIRECTORY = 1 << 31
	TYPE_MANIFEST     = 24
)

var (
	STRING_RSRC = [8]byte{'.', 'r', 's', 'r', 'c', 0, 0, 0}
)

func MustGetFieldOffset(t reflect.Type, field string) uintptr {
	f, ok := t.FieldByName(field)
	if !ok {
		panic("field " + field + " not found")
	}
	return f.Offset
}

type Writer struct {
	W      io.Writer
	Offset uint32 //FIXME: int64?
	Err    error
}

func (w *Writer) WriteLE(v interface{}) {
	if w.Err != nil {
		return
	}
	w.Err = binary.Write(w.W, binary.LittleEndian, v)
	if w.Err != nil {
		return
	}
	w.Offset += uint32(reflect.TypeOf(v).Size())
}

func (w *Writer) WriteFromSized(r SizedReader) {
	if w.Err != nil {
		return
	}
	var n int64
	n, w.Err = io.CopyN(w.W, r, r.Size())
	w.Offset += uint32(n)
}

type SizedReader interface {
	io.Reader
	Size() int64
}

func main() {
	//TODO: allow in options advanced specification of multiple resources, as a tree (json?)
	var fnamein, fnameout string
	flags := flag.NewFlagSet("", flag.ContinueOnError)
	flags.StringVar(&fnamein, "manifest", "", "path to a Windows manifest file to embed")
	flags.StringVar(&fnameout, "o", "rsrc.syso", "name of output COFF (.res or .syso) file")
	_ = flags.Parse(os.Args[1:])
	if fnamein == "" {
		fmt.Fprintf(os.Stderr, "USAGE: %s -manifest FILE.exe.manifest [-o FILE.syso]\n"+
			"Generates a .syso file with specified resources embedded in .rsrc section,\n"+
			"aimed for consumption by Go linker when building Win32 excecutables.\n"+
			"OPTIONS:\n",
			os.Args[0])
		flags.PrintDefaults()
		os.Exit(1)
	}

	err := run(fnamein, fnameout)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func run(fnamein, fnameout string) error {
	var manifest SizedReader
	{
		f, err := os.Open(fnamein)
		if err != nil {
			return fmt.Errorf("Error opening manifest file '%s': %s", fnamein, err)
		}
		defer f.Close()
		info, err := f.Stat()
		if err != nil {
			return err
		}
		manifest = io.NewSectionReader(f, 0, info.Size())
	}

	out, err := os.Create(fnameout)
	if err != nil {
		return err
	}
	defer out.Close()
	w := Writer{W: out}

	// precalculate some important offsets in resulting file, that we must know earlier
	//TODO: try to simplify by adding fake section at beginning, containing strings table in data, and characteristics saying "drop me when linking"
	rawdataoff := uint32(binary.Size(pe.FileHeader{}) + binary.Size(pe.SectionHeader32{}))
	hierarchylen := uint32(3*binary.Size(ImageResourceDirectory{}) +
		3*binary.Size(ImageResourceDirectoryEntry{}))
	rawdatalen := hierarchylen +
		uint32(1*binary.Size(ImageResourceDataEntry{})) +
		uint32(manifest.Size())
	diroff := rawdataoff
	relocoff := rawdataoff + rawdatalen
	relocp := hierarchylen + uint32(MustGetFieldOffset(reflect.TypeOf(ImageResourceDataEntry{}), "OffsetToData"))
	reloclen := uint32(binary.Size(RelocationEntry{}))
	symoff := relocoff + reloclen

	w.WriteLE(pe.FileHeader{
		Machine:              0x014c, //FIXME: find out how to differentiate this value, or maybe not necessary for Go
		NumberOfSections:     1,      // .rsrc
		TimeDateStamp:        0,      // was also 0 in sample data from MinGW's windres.exe
		PointerToSymbolTable: uint32(symoff),
		NumberOfSymbols:      1,
		SizeOfOptionalHeader: 0,
		Characteristics:      0x0104, //FIXME: copied from windres.exe output, find out what should be here and why
	})
	w.WriteLE(pe.SectionHeader32{
		Name:                 STRING_RSRC,
		SizeOfRawData:        rawdatalen,
		PointerToRawData:     rawdataoff,
		PointerToRelocations: relocoff,
		NumberOfRelocations:  1,
		Characteristics:      0x40000040, // "INITIALIZED_DATA MEM_READ" ?
	})

	// now, build "directory hierarchy" of .rsrc section: first type, then id/name, then language
	w.WriteLE(ImageResourceDirectory{
		NumberOfIdEntries: 1,
	})
	w.WriteLE(ImageResourceDirectoryEntry{
		NameOrId:     TYPE_MANIFEST,
		OffsetToData: MASK_SUBDIRECTORY | (w.Offset + uint32(binary.Size(ImageResourceDirectoryEntry{})) - diroff),
	})
	w.WriteLE(ImageResourceDirectory{
		NumberOfIdEntries: 1,
	})
	w.WriteLE(ImageResourceDirectoryEntry{
		NameOrId:     1, // ID
		OffsetToData: MASK_SUBDIRECTORY | (w.Offset + uint32(binary.Size(ImageResourceDirectoryEntry{})) - diroff),
	})
	w.WriteLE(ImageResourceDirectory{
		NumberOfIdEntries: 1,
	})
	w.WriteLE(ImageResourceDirectoryEntry{
		NameOrId:     0x0409, //FIXME: language; what value should be here?
		OffsetToData: w.Offset + uint32(binary.Size(ImageResourceDirectoryEntry{})) - diroff,
	})

	w.WriteLE(ImageResourceDataEntry{
		OffsetToData: w.Offset + uint32(binary.Size(ImageResourceDataEntry{})) - diroff,
		Size1:        uint32(manifest.Size()),
		CodePage:     0, //FIXME: what value here? for now just tried 0
	})

	w.WriteFromSized(manifest)

	w.WriteLE(RelocationEntry{
		RVA:         relocp, // FIXME: IIUC, this resolves to value contained in ImageResourceDataEntry.OffsetToData
		SymbolIndex: 0,      // "(zero based) index in the Symbol table to which the reference refers. Once you have loaded the COFF file into memory and know where each symbol is, you find the new updated address for the given symbol and update the reference accordingly."
		Type:        7,      // according to ldpe.c, this decodes to: IMAGE_REL_I386_DIR32NB
	})

	w.WriteLE(Symbol{
		Name:           STRING_RSRC,
		Value:          0,
		SectionNumber:  1,
		Type:           0, // FIXME: wtf?
		StorageClass:   3, // FIXME: is it ok? and uint8? and what does the value mean?
		AuxiliaryCount: 0, // FIXME: wtf?
	})

	w.WriteLE(StringsHeader{
		Length: uint32(binary.Size(StringsHeader{})), // empty strings table -- but we must still show size of the table's header...
	})

	if w.Err != nil {
		return fmt.Errorf("Error writing .rsrc Symbol Table & Strings: %s", w.Err)
	}

	return nil
}
<commit_msg>factor out SizedFile class<commit_after>package main

import (
	"debug/pe"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"os"
	"reflect"
)

type ImageResourceDirectory struct {
	Characteristics      uint32
	TimeDateStamp        uint32
	MajorVersion         uint16
	MinorVersion         uint16
	NumberOfNamedEntries uint16
	NumberOfIdEntries    uint16
}

type ImageResourceDirectoryEntry struct {
	NameOrId     uint32
	OffsetToData uint32
}

type ImageResourceDataEntry struct {
	OffsetToData uint32
	Size1        uint32
	CodePage     uint32
	Reserved     uint32
}

type RelocationEntry struct {
	RVA         uint32 // "offset within the Section's raw data where the address starts."
	SymbolIndex uint32 // "(zero based) index in the Symbol table to which the reference refers."
	Type        uint16
}

type Symbol struct {
	Name           [8]byte
	Value          uint32
	SectionNumber  uint16
	Type           uint16
	StorageClass   uint8
	AuxiliaryCount uint8
}

type StringsHeader struct {
	Length uint32
}

const (
	MASK_SUBDIRECTORY = 1 << 31
	TYPE_MANIFEST     = 24
)

var (
	STRING_RSRC = [8]byte{'.', 'r', 's', 'r', 'c', 0, 0, 0}
)

func MustGetFieldOffset(t reflect.Type, field string) uintptr {
	f, ok := t.FieldByName(field)
	if !ok {
		panic("field " + field + " not found")
	}
	return f.Offset
}

type Writer struct {
	W      io.Writer
	Offset uint32 //FIXME: int64?
	Err    error
}

func (w *Writer) WriteLE(v interface{}) {
	if w.Err != nil {
		return
	}
	w.Err = binary.Write(w.W, binary.LittleEndian, v)
	if w.Err != nil {
		return
	}
	w.Offset += uint32(reflect.TypeOf(v).Size())
}

type SizedReader interface {
	io.Reader
	Size() int64
}

func (w *Writer) WriteFromSized(r SizedReader) {
	if w.Err != nil {
		return
	}
	var n int64
	n, w.Err = io.CopyN(w.W, r, r.Size())
	w.Offset += uint32(n)
}

type SizedFile struct {
	f *os.File
	s *io.SectionReader // helper, for Size()
}

func (r *SizedFile) Read(p []byte) (n int, err error) { return r.s.Read(p) }
func (r *SizedFile) Size() int64                      { return r.s.Size() }
func (r *SizedFile) Close() error                     { return r.f.Close() }

func SizedOpen(filename string) (*SizedFile, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	info, err := f.Stat()
	if err != nil {
		return nil, err
	}
	return &SizedFile{
		f: f,
		s: io.NewSectionReader(f, 0, info.Size()),
	}, nil
}

func main() {
	//TODO: allow in options advanced specification of multiple resources, as a tree (json?)
	var fnamein, fnameout string
	flags := flag.NewFlagSet("", flag.ContinueOnError)
	flags.StringVar(&fnamein, "manifest", "", "path to a Windows manifest file to embed")
	flags.StringVar(&fnameout, "o", "rsrc.syso", "name of output COFF (.res or .syso) file")
	_ = flags.Parse(os.Args[1:])
	if fnamein == "" {
		fmt.Fprintf(os.Stderr, "USAGE: %s -manifest FILE.exe.manifest [-o FILE.syso]\n"+
			"Generates a .syso file with specified resources embedded in .rsrc section,\n"+
			"aimed for consumption by Go linker when building Win32 excecutables.\n"+
			"OPTIONS:\n",
			os.Args[0])
		flags.PrintDefaults()
		os.Exit(1)
	}

	err := run(fnamein, fnameout)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func run(fnamein, fnameout string) error {
	manifest, err := SizedOpen(fnamein)
	if err != nil {
		return fmt.Errorf("Error opening manifest file '%s': %s", fnamein, err)
	}
	defer manifest.Close()

	out, err := os.Create(fnameout)
	if err != nil {
		return err
	}
	defer out.Close()
	w := Writer{W: out}

	// precalculate some important offsets in resulting file, that we must know earlier
	//TODO: try to simplify by adding fake section at beginning, containing strings table in data, and characteristics saying "drop me when linking"
	rawdataoff := uint32(binary.Size(pe.FileHeader{}) + binary.Size(pe.SectionHeader32{}))
	hierarchylen := uint32(3*binary.Size(ImageResourceDirectory{}) +
		3*binary.Size(ImageResourceDirectoryEntry{}))
	rawdatalen := hierarchylen +
		uint32(1*binary.Size(ImageResourceDataEntry{})) +
		uint32(manifest.Size())
	diroff := rawdataoff
	relocoff := rawdataoff + rawdatalen
	relocp := hierarchylen + uint32(MustGetFieldOffset(reflect.TypeOf(ImageResourceDataEntry{}), "OffsetToData"))
	reloclen := uint32(binary.Size(RelocationEntry{}))
	symoff := relocoff + reloclen

	w.WriteLE(pe.FileHeader{
		Machine:              0x014c, //FIXME: find out how to differentiate this value, or maybe not necessary for Go
		NumberOfSections:     1,      // .rsrc
		TimeDateStamp:        0,      // was also 0 in sample data from MinGW's windres.exe
		PointerToSymbolTable: uint32(symoff),
		NumberOfSymbols:      1,
		SizeOfOptionalHeader: 0,
		Characteristics:      0x0104, //FIXME: copied from windres.exe output, find out what should be here and why
	})
	w.WriteLE(pe.SectionHeader32{
		Name:                 STRING_RSRC,
		SizeOfRawData:        rawdatalen,
		PointerToRawData:     rawdataoff,
		PointerToRelocations: relocoff,
		NumberOfRelocations:  1,
		Characteristics:      0x40000040, // "INITIALIZED_DATA MEM_READ" ?
	})

	// now, build "directory hierarchy" of .rsrc section: first type, then id/name, then language
	w.WriteLE(ImageResourceDirectory{
		NumberOfIdEntries: 1,
	})
	w.WriteLE(ImageResourceDirectoryEntry{
		NameOrId:     TYPE_MANIFEST,
		OffsetToData: MASK_SUBDIRECTORY | (w.Offset + uint32(binary.Size(ImageResourceDirectoryEntry{})) - diroff),
	})
	w.WriteLE(ImageResourceDirectory{
		NumberOfIdEntries: 1,
	})
	w.WriteLE(ImageResourceDirectoryEntry{
		NameOrId:     1, // ID
		OffsetToData: MASK_SUBDIRECTORY | (w.Offset + uint32(binary.Size(ImageResourceDirectoryEntry{})) - diroff),
	})
	w.WriteLE(ImageResourceDirectory{
		NumberOfIdEntries: 1,
	})
	w.WriteLE(ImageResourceDirectoryEntry{
		NameOrId:     0x0409, //FIXME: language; what value should be here?
		OffsetToData: w.Offset + uint32(binary.Size(ImageResourceDirectoryEntry{})) - diroff,
	})

	w.WriteLE(ImageResourceDataEntry{
		OffsetToData: w.Offset + uint32(binary.Size(ImageResourceDataEntry{})) - diroff,
		Size1:        uint32(manifest.Size()),
		CodePage:     0, //FIXME: what value here? for now just tried 0
	})

	w.WriteFromSized(manifest)

	w.WriteLE(RelocationEntry{
		RVA:         relocp, // FIXME: IIUC, this resolves to value contained in ImageResourceDataEntry.OffsetToData
		SymbolIndex: 0,      // "(zero based) index in the Symbol table to which the reference refers. Once you have loaded the COFF file into memory and know where each symbol is, you find the new updated address for the given symbol and update the reference accordingly."
		Type:        7,      // according to ldpe.c, this decodes to: IMAGE_REL_I386_DIR32NB
	})

	w.WriteLE(Symbol{
		Name:           STRING_RSRC,
		Value:          0,
		SectionNumber:  1,
		Type:           0, // FIXME: wtf?
		StorageClass:   3, // FIXME: is it ok? and uint8? and what does the value mean?
		AuxiliaryCount: 0, // FIXME: wtf?
	})

	w.WriteLE(StringsHeader{
		Length: uint32(binary.Size(StringsHeader{})), // empty strings table -- but we must still show size of the table's header...
	})

	if w.Err != nil {
		return fmt.Errorf("Error writing .rsrc Symbol Table & Strings: %s", w.Err)
	}

	return nil
}
<|endoftext|>