<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/howeyc/fsnotify"
	"github.com/kr/pty"
)

var seqCommands = &sync.Mutex{}

type OutMsg struct {
	reflexID int
	message  string
}

const (
	// ANSI colors -- using 32 - 36
	colorStart = 32
	numColors  = 5
)

func infoPrintln(id int, args ...interface{}) {
	stdout <- OutMsg{id, strings.TrimSpace(fmt.Sprintln(args...))}
}
func infoPrintf(id int, format string, args ...interface{}) {
	stdout <- OutMsg{id, fmt.Sprintf(format, args...)}
}

func walker(watcher *fsnotify.Watcher) filepath.WalkFunc {
	return func(path string, f os.FileInfo, err error) error {
		if err != nil || !f.IsDir() {
			// TODO: Is there some other thing we should be doing to handle errors? When watching large
			// directories that have lots of programs modifying them (esp. if they're making tempfiles along the
			// way), we often get errors.
			return nil
		}
		if err := watcher.Watch(path); err != nil {
			// TODO: handle this somehow?
			infoPrintf(-1, "Error while watching new path %s: %s\n", path, err)
		}
		return nil
	}
}

func broadcast(in <-chan string, outs []chan<- string) {
	for e := range in {
		for _, out := range outs {
			out <- e
		}
	}
}

func watch(root string, watcher *fsnotify.Watcher, names chan<- string, done chan<- error) {
	if err := filepath.Walk(root, walker(watcher)); err != nil {
		// TODO: handle this somehow?
		infoPrintf(-1, "Error while walking path %s: %s\n", root, err)
	}

	for {
		select {
		case e := <-watcher.Event:
			path := strings.TrimPrefix(e.Name, "./")
			names <- path
			if e.IsCreate() {
				if err := filepath.Walk(path, walker(watcher)); err != nil {
					// TODO: handle this somehow?
					infoPrintf(-1, "Error while walking path %s: %s\n", path, err)
				}
			}
			if e.IsDelete() {
				watcher.RemoveWatch(path)
			}
		case err := <-watcher.Error:
			done <- err
			return
		}
	}
}

// filterMatching passes on messages matching the regex/glob.
func filterMatching(in <-chan string, out chan<- string, reflex *Reflex) {
	for name := range in {
		if reflex.useRegex {
			if !reflex.regex.MatchString(name) {
				continue
			}
		} else {
			matches, err := filepath.Match(reflex.glob, name)
			// TODO: It would be good to notify the user on an error here.
			if err != nil {
				infoPrintln(reflex.id, "Error matching glob:", err)
				continue
			}
			if !matches {
				continue
			}
		}
		// TODO: These only match if the file/dir still exists...not sure if there's a better way.
		if reflex.onlyFiles || reflex.onlyDirs {
			stat, err := os.Stat(name)
			if err != nil {
				continue
			}
			if (reflex.onlyFiles && stat.IsDir()) || (reflex.onlyDirs && !stat.IsDir()) {
				continue
			}
		}
		out <- name
	}
}

// batch receives realtime file notification events and batches them up. It's a bit tricky, but here's what
// it accomplishes:
// * When we initially get a message, wait a bit and batch messages before trying to send anything. This is
//	 because the file events come in quick bursts.
// * Once it's time to send, don't do it until the out channel is unblocked. In the meantime, keep batching.
//   When we've sent off all the batched messages, go back to the beginning.
func batch(in <-chan string, out chan<- string, reflex *Reflex) {
	for name := range in {
		reflex.backlog.Add(name)
		timer := time.NewTimer(300 * time.Millisecond)
	outer:
		for {
			select {
			case name := <-in:
				reflex.backlog.Add(name)
			case <-timer.C:
				for {
					select {
					case name := <-in:
						reflex.backlog.Add(name)
					case out <- reflex.backlog.Next():
						if reflex.backlog.RemoveOne() {
							break outer
						}
					}
				}
			}
		}
	}
}

// runEach runs the command on each name that comes through the names channel. Each {} is replaced by the name
// of the file. The output of the command is passed line-by-line to the stdout chan.
func runEach(names <-chan string, reflex *Reflex) {
	for name := range names {
		if reflex.startService {
			if reflex.done != nil {
				infoPrintln(reflex.id, "Killing service")
				terminate(reflex)
			}
			infoPrintln(reflex.id, "Starting service")
			runCommand(reflex, name, stdout)
		} else {
			runCommand(reflex, name, stdout)
			<-reflex.done
			reflex.done = nil
		}
	}
}

func terminate(reflex *Reflex) {
	reflex.mut.Lock()
	reflex.killed = true
	reflex.mut.Unlock()
	// Write ascii 3 (what you get from ^C) to the controlling pty.
	// (should be harmless if the process already died as the write will simply fail)
	reflex.tty.Write([]byte{3})

	timer := time.NewTimer(500 * time.Millisecond)
	sig := syscall.SIGINT
	for {
		select {
		case <-reflex.done:
			return
		case <-timer.C:
			if sig == syscall.SIGINT {
				infoPrintln(reflex.id, "Sending SIGINT signal...")
			} else {
				infoPrintln(reflex.id, "Sending SIGKILL signal...")
			}

			// We do not use reflex.cmd.Process.Signal(..) here because we want
			// to kill the whole process group
			if err := syscall.Kill(-1*reflex.cmd.Process.Pid, sig); err != nil {
				infoPrintln(reflex.id, "Error killing:", err)
				// TODO: is there a better way to detect this?
				if err.Error() == "no such process" {
					return
				}
			}
			// next try will use the SIGKILL signal
			sig = syscall.SIGKILL
		}
	}
}

func replaceSubSymbol(command []string, subSymbol string, name string) []string {
	replacer := strings.NewReplacer(subSymbol, name)
	newCommand := make([]string, len(command))
	for i, c := range command {
		newCommand[i] = replacer.Replace(c)
	}
	return newCommand
}

// runCommand runs the given Command. All output is passed line-by-line to the stdout channel.
func runCommand(reflex *Reflex, name string, stdout chan<- OutMsg) {
	command := replaceSubSymbol(reflex.command, reflex.subSymbol, name)
	cmd := exec.Command(command[0], command[1:]...)
	reflex.cmd = cmd

	if flagSequential {
		seqCommands.Lock()
	}

	tty, err := pty.Start(cmd)
	if err != nil {
		infoPrintln(reflex.id, err)
		return
	}
	reflex.tty = tty

	go func() {
		scanner := bufio.NewScanner(tty)
		for scanner.Scan() {
			stdout <- OutMsg{reflex.id, scanner.Text()}
		}
		// Intentionally ignoring scanner.Err() for now.
		// Unfortunately, the pty returns a read error when the child dies naturally, so I'm just going to ignore
		// errors here unless I can find a better way to handle it.
	}()

	done := make(chan struct{})
	reflex.done = done
	go func() {
		err := cmd.Wait()
		reflex.mut.Lock()
		killed := reflex.killed
		reflex.mut.Unlock()
		if !killed && err != nil {
			stdout <- OutMsg{reflex.id, fmt.Sprintf("(error exit: %s)", err)}
		}
		done <- struct{}{}
		if flagSequential {
			seqCommands.Unlock()
		}
	}()
}

func printMsg(msg OutMsg, writer io.Writer) {
	tag := ""
	if decoration == DecorationFancy || decoration == DecorationPlain {
		if msg.reflexID < 0 {
			tag = "[info]"
		} else {
			tag = fmt.Sprintf("[%02d]", msg.reflexID)
		}
	}

	if decoration == DecorationFancy {
		color := (msg.reflexID % numColors) + colorStart
		if reflexID < 0 {
			color = 31 // red
		}
		fmt.Fprintf(writer, "\x1b[01;%dm%s ", color, tag)
	} else if decoration == DecorationPlain {
		fmt.Fprintf(writer, tag+" ")
	}
	fmt.Fprint(writer, msg.message)
	if decoration == DecorationFancy {
		fmt.Fprintf(writer, "\x1b[m")
	}
	fmt.Fprintln(writer)
}

func printOutput(out <-chan OutMsg, outWriter io.Writer) {
	for msg := range out {
		printMsg(msg, outWriter)
	}
}
<commit_msg>Change some comments.<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/howeyc/fsnotify"
	"github.com/kr/pty"
)

var seqCommands = &sync.Mutex{}

type OutMsg struct {
	reflexID int
	message  string
}

const (
	// ANSI colors -- using 32 - 36
	colorStart = 32
	numColors  = 5
)

func infoPrintln(id int, args ...interface{}) {
	stdout <- OutMsg{id, strings.TrimSpace(fmt.Sprintln(args...))}
}
func infoPrintf(id int, format string, args ...interface{}) {
	stdout <- OutMsg{id, fmt.Sprintf(format, args...)}
}

func walker(watcher *fsnotify.Watcher) filepath.WalkFunc {
	return func(path string, f os.FileInfo, err error) error {
		if err != nil || !f.IsDir() {
			// TODO: Is there some other thing we should be doing to handle errors? When watching large
			// directories that have lots of programs modifying them (esp. if they're making tempfiles along the
			// way), we often get errors.
			return nil
		}
		if err := watcher.Watch(path); err != nil {
			// TODO: handle this somehow?
			infoPrintf(-1, "Error while watching new path %s: %s\n", path, err)
		}
		return nil
	}
}

func broadcast(in <-chan string, outs []chan<- string) {
	for e := range in {
		for _, out := range outs {
			out <- e
		}
	}
}

func watch(root string, watcher *fsnotify.Watcher, names chan<- string, done chan<- error) {
	if err := filepath.Walk(root, walker(watcher)); err != nil {
		// TODO: handle this somehow?
		infoPrintf(-1, "Error while walking path %s: %s\n", root, err)
	}

	for {
		select {
		case e := <-watcher.Event:
			path := strings.TrimPrefix(e.Name, "./")
			names <- path
			if e.IsCreate() {
				if err := filepath.Walk(path, walker(watcher)); err != nil {
					// TODO: handle this somehow?
					infoPrintf(-1, "Error while walking path %s: %s\n", path, err)
				}
			}
			if e.IsDelete() {
				watcher.RemoveWatch(path)
			}
		case err := <-watcher.Error:
			done <- err
			return
		}
	}
}

// filterMatching passes on messages matching the regex/glob.
func filterMatching(in <-chan string, out chan<- string, reflex *Reflex) {
	for name := range in {
		if reflex.useRegex {
			if !reflex.regex.MatchString(name) {
				continue
			}
		} else {
			matches, err := filepath.Match(reflex.glob, name)
			// TODO: It would be good to notify the user on an error here.
			if err != nil {
				infoPrintln(reflex.id, "Error matching glob:", err)
				continue
			}
			if !matches {
				continue
			}
		}
		// TODO: These only match if the file/dir still exists...not sure if there's a better way.
		if reflex.onlyFiles || reflex.onlyDirs {
			stat, err := os.Stat(name)
			if err != nil {
				continue
			}
			if (reflex.onlyFiles && stat.IsDir()) || (reflex.onlyDirs && !stat.IsDir()) {
				continue
			}
		}
		out <- name
	}
}

// batch receives realtime file notification events and batches them up. It's a bit tricky, but here's what
// it accomplishes:
// * When we initially get a message, wait a bit and batch messages before trying to send anything. This is
//	 because the file events come in quick bursts.
// * Once it's time to send, don't do it until the out channel is unblocked. In the meantime, keep batching.
//   When we've sent off all the batched messages, go back to the beginning.
func batch(in <-chan string, out chan<- string, reflex *Reflex) {
	for name := range in {
		reflex.backlog.Add(name)
		timer := time.NewTimer(300 * time.Millisecond)
	outer:
		for {
			select {
			case name := <-in:
				reflex.backlog.Add(name)
			case <-timer.C:
				for {
					select {
					case name := <-in:
						reflex.backlog.Add(name)
					case out <- reflex.backlog.Next():
						if reflex.backlog.RemoveOne() {
							break outer
						}
					}
				}
			}
		}
	}
}

// runEach runs the command on each name that comes through the names channel. Each {} is replaced by the name
// of the file. The output of the command is passed line-by-line to the stdout chan.
func runEach(names <-chan string, reflex *Reflex) {
	for name := range names {
		if reflex.startService {
			if reflex.done != nil {
				infoPrintln(reflex.id, "Killing service")
				terminate(reflex)
			}
			infoPrintln(reflex.id, "Starting service")
			runCommand(reflex, name, stdout)
		} else {
			runCommand(reflex, name, stdout)
			<-reflex.done
			reflex.done = nil
		}
	}
}

func terminate(reflex *Reflex) {
	reflex.mut.Lock()
	reflex.killed = true
	reflex.mut.Unlock()
	// Write ascii 3 (what you get from ^C) to the controlling pty.
	// (This won't do anything if the process already died as the write will simply fail.)
	reflex.tty.Write([]byte{3})

	timer := time.NewTimer(500 * time.Millisecond)
	sig := syscall.SIGINT
	for {
		select {
		case <-reflex.done:
			return
		case <-timer.C:
			if sig == syscall.SIGINT {
				infoPrintln(reflex.id, "Sending SIGINT signal...")
			} else {
				infoPrintln(reflex.id, "Sending SIGKILL signal...")
			}

			// Instead of killing the process, we want to kill its whole pgroup in order to clean up any children
			// the process may have created.
			if err := syscall.Kill(-1*reflex.cmd.Process.Pid, sig); err != nil {
				infoPrintln(reflex.id, "Error killing:", err)
				// TODO: is there a better way to detect this?
				if err.Error() == "no such process" {
					return
				}
			}
			// After SIGINT doesn't do anything, try SIGKILL.
			sig = syscall.SIGKILL
		}
	}
}

func replaceSubSymbol(command []string, subSymbol string, name string) []string {
	replacer := strings.NewReplacer(subSymbol, name)
	newCommand := make([]string, len(command))
	for i, c := range command {
		newCommand[i] = replacer.Replace(c)
	}
	return newCommand
}

// runCommand runs the given Command. All output is passed line-by-line to the stdout channel.
func runCommand(reflex *Reflex, name string, stdout chan<- OutMsg) {
	command := replaceSubSymbol(reflex.command, reflex.subSymbol, name)
	cmd := exec.Command(command[0], command[1:]...)
	reflex.cmd = cmd

	if flagSequential {
		seqCommands.Lock()
	}

	tty, err := pty.Start(cmd)
	if err != nil {
		infoPrintln(reflex.id, err)
		return
	}
	reflex.tty = tty

	go func() {
		scanner := bufio.NewScanner(tty)
		for scanner.Scan() {
			stdout <- OutMsg{reflex.id, scanner.Text()}
		}
		// Intentionally ignoring scanner.Err() for now.
		// Unfortunately, the pty returns a read error when the child dies naturally, so I'm just going to ignore
		// errors here unless I can find a better way to handle it.
	}()

	done := make(chan struct{})
	reflex.done = done
	go func() {
		err := cmd.Wait()
		reflex.mut.Lock()
		killed := reflex.killed
		reflex.mut.Unlock()
		if !killed && err != nil {
			stdout <- OutMsg{reflex.id, fmt.Sprintf("(error exit: %s)", err)}
		}
		done <- struct{}{}
		if flagSequential {
			seqCommands.Unlock()
		}
	}()
}

func printMsg(msg OutMsg, writer io.Writer) {
	tag := ""
	if decoration == DecorationFancy || decoration == DecorationPlain {
		if msg.reflexID < 0 {
			tag = "[info]"
		} else {
			tag = fmt.Sprintf("[%02d]", msg.reflexID)
		}
	}

	if decoration == DecorationFancy {
		color := (msg.reflexID % numColors) + colorStart
		if reflexID < 0 {
			color = 31 // red
		}
		fmt.Fprintf(writer, "\x1b[01;%dm%s ", color, tag)
	} else if decoration == DecorationPlain {
		fmt.Fprintf(writer, tag+" ")
	}
	fmt.Fprint(writer, msg.message)
	if decoration == DecorationFancy {
		fmt.Fprintf(writer, "\x1b[m")
	}
	fmt.Fprintln(writer)
}

func printOutput(out <-chan OutMsg, outWriter io.Writer) {
	for msg := range out {
		printMsg(msg, outWriter)
	}
}
<|endoftext|>


<commit_before>package yum

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"sort"

	gocfg "github.com/gonuts/config"
	"github.com/gonuts/logger"
)

type Client struct {
	msg         *logger.Logger
	siteroot    string
	etcdir      string
	lbyumcache  string
	yumconf     string
	yumreposdir string
	configured  bool
	repos       map[string]*Repository
	repourls    map[string]string
}

// newClient returns a Client from siteroot and backends.
// manualConfig is just for internal tests
func newClient(siteroot string, backends []string, checkForUpdates, manualConfig bool) (*Client, error) {
	client := &Client{
		msg:         logger.NewLogger("yum", logger.WARNING, os.Stdout),
		siteroot:    siteroot,
		etcdir:      filepath.Join(siteroot, "etc"),
		lbyumcache:  filepath.Join(siteroot, "var", "cache", "lbyum"),
		yumconf:     filepath.Join(siteroot, "etc", "yum.conf"),
		yumreposdir: filepath.Join(siteroot, "etc", "yum.repos.d"),
		configured:  false,
		repos:       make(map[string]*Repository),
		repourls:    make(map[string]string),
	}

	if manualConfig {
		return client, nil
	}

	// load the config and set the URLs accordingly
	urls, err := client.loadConfig()
	if err != nil {
		return nil, err
	}

	// At this point we have the repo names and URLs in self.repourls
	// we know connect to them to get the best method to get the appropriate files
	err = client.initRepositories(urls, checkForUpdates, backends)
	if err != nil {
		return nil, err
	}

	return client, err
}

// New returns a new YUM Client, rooted at siteroot.
func New(siteroot string) (*Client, error) {
	checkForUpdates := true
	manualConfig := false
	backends := []string{
		"RepositorySQLiteBackend",
		"RepositoryXMLBackend",
	}
	return newClient(siteroot, backends, checkForUpdates, manualConfig)
}

// Close cleans up after use
func (yum *Client) Close() error {
	var err error
	for name, repo := range yum.repos {
		e := repo.Close()
		if e != nil {
			yum.msg.Errorf("error closing repo [%s]: %v\n", name, e)
			e = err
		} else {
			yum.msg.Debugf("closed repo [%s]\n", name)
		}
	}
	return err
}

// SetLevel sets the verbosity level of Client
func (yum *Client) SetLevel(lvl logger.Level) {
	yum.msg.SetLevel(lvl)
	for _, repo := range yum.repos {
		repo.msg.SetLevel(lvl)
	}
}

// FindLatestMatchingName locates a package by name and returns the latest available version
func (yum *Client) FindLatestMatchingName(name, version, release string) (*Package, error) {
	var err error
	var pkg *Package
	found := make(Packages, 0)
	errors := make([]error, 0, len(yum.repos))

	for _, repo := range yum.repos {
		p, err := repo.FindLatestMatchingName(name, version, release)
		if err != nil {
			errors = append(errors, err)
			continue
		}
		found = append(found, p)
	}

	if len(found) > 0 {
		sort.Sort(found)
		pkg = found[len(found)-1]
		return pkg, err
	}

	if len(errors) == len(yum.repos) && len(errors) > 0 {
		return nil, errors[0]
	}

	return pkg, err
}

// FindLatestMatchingRequire locates a package providing a given functionality.
func (yum *Client) FindLatestMatchingRequire(requirement *Requires) (*Package, error) {
	var err error
	var pkg *Package
	found := make(Packages, 0)
	errors := make([]error, 0, len(yum.repos))

	for _, repo := range yum.repos {
		p, err := repo.FindLatestMatchingRequire(requirement)
		if err != nil {
			errors = append(errors, err)
			continue
		}
		found = append(found, p)
	}

	if len(found) > 0 {
		sort.Sort(found)
		pkg = found[len(found)-1]
		return pkg, err
	}

	if len(errors) == len(yum.repos) && len(errors) > 0 {
		return nil, errors[0]
	}

	return pkg, err
}

// ListPackages lists all packages satisfying pattern (a regexp)
func (yum *Client) ListPackages(name, version, release string) ([]*Package, error) {
	var err error
	re_name := regexp.MustCompile(name)
	re_vers := regexp.MustCompile(version)
	re_rel := regexp.MustCompile(release)
	pkgs := make([]*Package, 0)
	for _, repo := range yum.repos {
		for _, pkg := range repo.GetPackages() {
			if re_name.MatchString(pkg.Name()) &&
				re_vers.MatchString(pkg.Version()) &&
				// FIXME: sprintf is ugly
				re_rel.MatchString(fmt.Sprintf("%d", pkg.Release())) {
				pkgs = append(pkgs, pkg)
			}
		}
	}
	return pkgs, err
}

// RequiredPackages returns the list of all required packages for pkg (including pkg itself)
func (yum *Client) RequiredPackages(pkg *Package) ([]*Package, error) {
	pkgs, err := yum.PackageDeps(pkg)
	if err != nil {
		return nil, err
	}
	pkgs = append(pkgs, pkg)
	return pkgs, err
}

// PackageDeps returns all dependencies for the package (excluding the package itself)
func (yum *Client) PackageDeps(pkg *Package) ([]*Package, error) {
	var err error
	processed := make(map[*Package]struct{})
	deps, err := yum.pkgDeps(pkg, processed)
	if err != nil {
		return nil, err
	}

	pkgs := make([]*Package, 0, len(deps))
	for p := range deps {
		pkgs = append(pkgs, p)
	}
	return pkgs, err
}

// pkgDeps returns all dependencies for the package (excluding the package itself)
func (yum *Client) pkgDeps(pkg *Package, processed map[*Package]struct{}) (map[*Package]struct{}, error) {
	var err error
	var lasterr error
	msg := yum.msg

	processed[pkg] = struct{}{}
	required := make(map[*Package]struct{})

	msg.Verbosef(">>> pkg %s.%s-%s (req=%d)\n", pkg.Name(), pkg.Version(), pkg.Release(), len(pkg.Requires()))
	for _, req := range pkg.Requires() {
		msg.Verbosef("processing deps for %s.%s-%s\n", req.Name(), req.Version(), req.Release())
		if str_in_slice(req.Name(), g_IGNORED_PACKAGES) {
			msg.Verbosef("processing deps for %s.%s-%s [IGNORE]\n", req.Name(), req.Version(), req.Release())
			continue
		}
		p, err := yum.FindLatestMatchingRequire(req)
		if err != nil {
			lasterr = err
			continue
		}
		if _, dup := processed[p]; dup {
			msg.Warnf("cyclic dependency in repository with package: %s.%s-%s\n", p.Name(), p.Version(), p.Release())
			continue
		}
		if p == nil {
			msg.Errorf("package %s.%s-%d not found!\n", req.Name(), req.Version(), req.Release())
			lasterr = fmt.Errorf("package %s.%s-%s not found", req.Name(), req.Version(), req.Release())
			continue
			//return nil, fmt.Errorf("package %s.%s-%s not found", req.Name(), req.Version(), req.Release())
		}
		msg.Verbosef("--> adding dep %s.%s-%s\n", p.Name(), p.Version(), p.Release())
		required[p] = struct{}{}
		sdeps, err := yum.pkgDeps(p, processed)
		if err != nil {
			lasterr = err
			continue
			//return nil, err
		}
		for sdep := range sdeps {
			required[sdep] = struct{}{}
		}
	}

	if lasterr != nil {
		err = lasterr
	}
	return required, err
}

// loadConfig looks up the location of the yum repository
func (yum *Client) loadConfig() (map[string]string, error) {
	fis, err := ioutil.ReadDir(yum.yumreposdir)
	if err != nil {
		return nil, err
	}
	pattern := regexp.MustCompile(`(.*)\.repo$`)
	for _, fi := range fis {
		if fi.IsDir() {
			continue
		}
		if !pattern.MatchString(fi.Name()) {
			continue
		}
		fname := filepath.Join(yum.yumreposdir, fi.Name())
		repos, err := yum.parseRepoConfigFile(fname)
		if err != nil {
			return nil, err
		}
		for k, v := range repos {
			yum.repourls[k] = v
		}
	}

	yum.configured = true
	if len(yum.repourls) <= 0 {
		return nil, fmt.Errorf("could not find repository config file in [%s]", yum.yumreposdir)
	}
	return yum.repourls, err
}

// parseRepoConfigFile parses the xyz.repo file and returns a map of reponame/repourl
func (yum *Client) parseRepoConfigFile(fname string) (map[string]string, error) {
	var err error
	repos := make(map[string]string)

	cfg, err := gocfg.ReadDefault(fname)
	if err != nil {
		return nil, err
	}

	for _, section := range cfg.Sections() {
		if !cfg.HasOption(section, "baseurl") {
			continue
		}
		repourl, err := cfg.String(section, "baseurl")
		if err != nil {
			return nil, err
		}
		yum.msg.Debugf("adding repo=%q url=%q from file [%s]\n", section, repourl, fname)
		repos[section] = repourl
	}
	return repos, err
}

func (yum *Client) initRepositories(urls map[string]string, checkForUpdates bool, backends []string) error {
	var err error

	const setupBackend = true

	// setup the repositories
	for repo, repourl := range urls {
		cachedir := filepath.Join(yum.lbyumcache, repo)
		err = os.MkdirAll(cachedir, 0644)
		if err != nil {
			return err
		}
		r, err := NewRepository(
			repo, repourl, cachedir,
			backends, setupBackend, checkForUpdates,
		)
		if err != nil {
			return err
		}
		r.msg = yum.msg
		yum.repos[repo] = r
	}

	yum.repourls = urls
	return err
}

// EOF
<commit_msg>yum: cosmetics<commit_after>package yum

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"sort"

	gocfg "github.com/gonuts/config"
	"github.com/gonuts/logger"
)

type Client struct {
	msg         *logger.Logger
	siteroot    string
	etcdir      string
	lbyumcache  string
	yumconf     string
	yumreposdir string
	configured  bool
	repos       map[string]*Repository
	repourls    map[string]string
}

// newClient returns a Client from siteroot and backends.
// manualConfig is just for internal tests
func newClient(siteroot string, backends []string, checkForUpdates, manualConfig bool) (*Client, error) {
	client := &Client{
		msg:         logger.NewLogger("yum", logger.WARNING, os.Stdout),
		siteroot:    siteroot,
		etcdir:      filepath.Join(siteroot, "etc"),
		lbyumcache:  filepath.Join(siteroot, "var", "cache", "lbyum"),
		yumconf:     filepath.Join(siteroot, "etc", "yum.conf"),
		yumreposdir: filepath.Join(siteroot, "etc", "yum.repos.d"),
		configured:  false,
		repos:       make(map[string]*Repository),
		repourls:    make(map[string]string),
	}

	if manualConfig {
		return client, nil
	}

	// load the config and set the URLs accordingly
	urls, err := client.loadConfig()
	if err != nil {
		return nil, err
	}

	// At this point we have the repo names and URLs in self.repourls
	// we know connect to them to get the best method to get the appropriate files
	err = client.initRepositories(urls, checkForUpdates, backends)
	if err != nil {
		return nil, err
	}

	return client, err
}

// New returns a new YUM Client, rooted at siteroot.
func New(siteroot string) (*Client, error) {
	checkForUpdates := true
	manualConfig := false
	backends := []string{
		"RepositorySQLiteBackend",
		"RepositoryXMLBackend",
	}
	return newClient(siteroot, backends, checkForUpdates, manualConfig)
}

// Close cleans up after use
func (yum *Client) Close() error {
	var err error
	for name, repo := range yum.repos {
		e := repo.Close()
		if e != nil {
			yum.msg.Errorf("error closing repo [%s]: %v\n", name, e)
			e = err
		} else {
			yum.msg.Debugf("closed repo [%s]\n", name)
		}
	}
	return err
}

// SetLevel sets the verbosity level of Client
func (yum *Client) SetLevel(lvl logger.Level) {
	yum.msg.SetLevel(lvl)
	for _, repo := range yum.repos {
		repo.msg.SetLevel(lvl)
	}
}

// FindLatestMatchingName locates a package by name and returns the latest available version
func (yum *Client) FindLatestMatchingName(name, version, release string) (*Package, error) {
	var err error
	var pkg *Package
	found := make(Packages, 0)
	errors := make([]error, 0, len(yum.repos))

	for _, repo := range yum.repos {
		p, err := repo.FindLatestMatchingName(name, version, release)
		if err != nil {
			errors = append(errors, err)
			continue
		}
		found = append(found, p)
	}

	if len(found) > 0 {
		sort.Sort(found)
		pkg = found[len(found)-1]
		return pkg, err
	}

	if len(errors) == len(yum.repos) && len(errors) > 0 {
		return nil, errors[0]
	}

	return pkg, err
}

// FindLatestMatchingRequire locates a package providing a given functionality.
func (yum *Client) FindLatestMatchingRequire(requirement *Requires) (*Package, error) {
	var err error
	var pkg *Package
	found := make(Packages, 0)
	errors := make([]error, 0, len(yum.repos))

	for _, repo := range yum.repos {
		p, err := repo.FindLatestMatchingRequire(requirement)
		if err != nil {
			errors = append(errors, err)
			continue
		}
		found = append(found, p)
	}

	if len(found) > 0 {
		sort.Sort(found)
		pkg = found[len(found)-1]
		return pkg, err
	}

	if len(errors) == len(yum.repos) && len(errors) > 0 {
		return nil, errors[0]
	}

	return pkg, err
}

// ListPackages lists all packages satisfying pattern (a regexp)
func (yum *Client) ListPackages(name, version, release string) ([]*Package, error) {
	var err error
	re_name := regexp.MustCompile(name)
	re_vers := regexp.MustCompile(version)
	re_rel := regexp.MustCompile(release)
	pkgs := make([]*Package, 0)
	for _, repo := range yum.repos {
		for _, pkg := range repo.GetPackages() {
			if re_name.MatchString(pkg.Name()) &&
				re_vers.MatchString(pkg.Version()) &&
				// FIXME: sprintf is ugly
				re_rel.MatchString(fmt.Sprintf("%d", pkg.Release())) {
				pkgs = append(pkgs, pkg)
			}
		}
	}
	return pkgs, err
}

// RequiredPackages returns the list of all required packages for pkg (including pkg itself)
func (yum *Client) RequiredPackages(pkg *Package) ([]*Package, error) {
	pkgs, err := yum.PackageDeps(pkg)
	if err != nil {
		return nil, err
	}
	pkgs = append(pkgs, pkg)
	return pkgs, err
}

// PackageDeps returns all dependencies for the package (excluding the package itself)
func (yum *Client) PackageDeps(pkg *Package) ([]*Package, error) {
	var err error
	processed := make(map[*Package]struct{})
	deps, err := yum.pkgDeps(pkg, processed)
	if err != nil {
		return nil, err
	}

	pkgs := make([]*Package, 0, len(deps))
	for p := range deps {
		pkgs = append(pkgs, p)
	}
	return pkgs, err
}

// pkgDeps returns all dependencies for the package (excluding the package itself)
func (yum *Client) pkgDeps(pkg *Package, processed map[*Package]struct{}) (map[*Package]struct{}, error) {
	var err error
	var lasterr error
	msg := yum.msg

	processed[pkg] = struct{}{}
	required := make(map[*Package]struct{})

	nreqs := len(pkg.Requires())
	msg.Verbosef(">>> pkg %s.%s-%s (req=%d)\n", pkg.Name(), pkg.Version(), pkg.Release(), nreqs)
	for ireq, req := range pkg.Requires() {
		msg.Verbosef("[%03d/%03d] processing deps for %s.%s-%s\n", ireq, nreqs, req.Name(), req.Version(), req.Release())
		if str_in_slice(req.Name(), g_IGNORED_PACKAGES) {
			msg.Verbosef("[%03d/%03d] processing deps for %s.%s-%s [IGNORE]\n", ireq, nreqs, req.Name(), req.Version(), req.Release())
			continue
		}
		p, err := yum.FindLatestMatchingRequire(req)
		if err != nil {
			lasterr = err
			continue
		}
		if _, dup := processed[p]; dup {
			msg.Warnf("cyclic dependency in repository with package: %s.%s-%s\n", p.Name(), p.Version(), p.Release())
			continue
		}
		if p == nil {
			msg.Errorf("package %s.%s-%s not found!\n", req.Name(), req.Version(), req.Release())
			lasterr = fmt.Errorf("package %s.%s-%s not found", req.Name(), req.Version(), req.Release())
			continue
			//return nil, fmt.Errorf("package %s.%s-%s not found", req.Name(), req.Version(), req.Release())
		}
		msg.Verbosef("--> adding dep %s.%s-%s\n", p.Name(), p.Version(), p.Release())
		required[p] = struct{}{}
		sdeps, err := yum.pkgDeps(p, processed)
		if err != nil {
			lasterr = err
			continue
			//return nil, err
		}
		for sdep := range sdeps {
			required[sdep] = struct{}{}
		}
	}

	if lasterr != nil {
		err = lasterr
	}
	return required, err
}

// loadConfig looks up the location of the yum repository
func (yum *Client) loadConfig() (map[string]string, error) {
	fis, err := ioutil.ReadDir(yum.yumreposdir)
	if err != nil {
		return nil, err
	}
	pattern := regexp.MustCompile(`(.*)\.repo$`)
	for _, fi := range fis {
		if fi.IsDir() {
			continue
		}
		if !pattern.MatchString(fi.Name()) {
			continue
		}
		fname := filepath.Join(yum.yumreposdir, fi.Name())
		repos, err := yum.parseRepoConfigFile(fname)
		if err != nil {
			return nil, err
		}
		for k, v := range repos {
			yum.repourls[k] = v
		}
	}

	yum.configured = true
	if len(yum.repourls) <= 0 {
		return nil, fmt.Errorf("could not find repository config file in [%s]", yum.yumreposdir)
	}
	return yum.repourls, err
}

// parseRepoConfigFile parses the xyz.repo file and returns a map of reponame/repourl
func (yum *Client) parseRepoConfigFile(fname string) (map[string]string, error) {
	var err error
	repos := make(map[string]string)

	cfg, err := gocfg.ReadDefault(fname)
	if err != nil {
		return nil, err
	}

	for _, section := range cfg.Sections() {
		if !cfg.HasOption(section, "baseurl") {
			continue
		}
		repourl, err := cfg.String(section, "baseurl")
		if err != nil {
			return nil, err
		}
		yum.msg.Debugf("adding repo=%q url=%q from file [%s]\n", section, repourl, fname)
		repos[section] = repourl
	}
	return repos, err
}

func (yum *Client) initRepositories(urls map[string]string, checkForUpdates bool, backends []string) error {
	var err error

	const setupBackend = true

	// setup the repositories
	for repo, repourl := range urls {
		cachedir := filepath.Join(yum.lbyumcache, repo)
		err = os.MkdirAll(cachedir, 0644)
		if err != nil {
			return err
		}
		r, err := NewRepository(
			repo, repourl, cachedir,
			backends, setupBackend, checkForUpdates,
		)
		if err != nil {
			return err
		}
		r.msg = yum.msg
		yum.repos[repo] = r
	}

	yum.repourls = urls
	return err
}

// EOF
<|endoftext|>


<commit_before>package zip

func Archive(file string, archive string) error {
	//TODO
	return nil
}

func Unarchive(archive string, file string) error {
	//TODO
	return nil
}
<commit_msg>Add Archive() for zip<commit_after>package zip

import (
	"archive/zip"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
)

func Archive(inPath string, outPath string, includeRootDir bool) error {
	inFileInfo, err := os.Stat(inPath)
	if err != nil {
		return err
	}

	outFile, err := os.Create(outPath)
	if err != nil {
		return err
	}
	defer outFile.Close()

	zipWriter := zip.NewWriter(outFile)

	inIsDir := inFileInfo.IsDir()
	archivePath := ""
	if includeRootDir {
		archivePath = inFileInfo.Name()
	}

	err = archive(zipWriter, inPath, inIsDir, archivePath)
	if err != nil {
		return err
	}

	err = zipWriter.Close()
	if err != nil {
		return err
	}

	return nil
}

func archive(zipWriter *zip.Writer, inPath string, inIsDir bool, archivePath string) error {
	if inIsDir {
		return archiveDir(zipWriter, inPath, archivePath)
	} else {
		return archiveFile(zipWriter, inPath, archivePath)
	}
}

func archiveDir(zipWriter *zip.Writer, inPath string, archivePath string) error {
	childFileInfos, err := ioutil.ReadDir(inPath)
	if err != nil {
		return err
	}

	for _, childFileInfo := range childFileInfos {
		childFileName := childFileInfo.Name()
		childInPath := filepath.Join(inPath, childFileName)
		childArchivePath := path.Join(archivePath, childFileName)
		childIsDir := childFileInfo.IsDir()
		err = archive(zipWriter, childInPath, childIsDir, childArchivePath)
		if err != nil {
			return err
		}
	}

	return nil
}

func archiveFile(zipWriter *zip.Writer, inPath string, archivePath string) error {
	inFile, err := os.Open(inPath)
	if err != nil {
		return err
	}
	defer inFile.Close()
	writer, err := zipWriter.Create(archivePath)
	if err != nil {
		return err
	}
	_, err = io.Copy(writer, inFile)
	if err != nil {
		return err
	}
	return nil
}

func Unarchive(archivePath string, filePath string) error {
	//TODO
	return nil
}
<|endoftext|>


<commit_before>// Paquete encargado de comprimir un archivo zip en 32 bits
package zip

import (
	"archive/zip"
	"io"
	"os"
	"time"
)

// Estructura enacargada de archivar la información del directorio/arhivo comprimido/descomprimio
type ZipContent struct {
	FileName    string    `json:"file_name"`
	IsDirectory bool      `json:"is_directory"`
	Size        int64     `json:"size"`
	Error       error     `json:"error"`
	CreatedAt   time.Time `json:"created_at"`
}

var (
	zipFile   *os.File
	zipWriter *zip.Writer
)

// Función encargada de elimina un directorio/archivo
func removeFileDirectory(path string) error {
	return os.Remove(path)
}

// Función encargada de crear un archivo
func createFile(filePath string) (*os.File, error) {
	// Eliminamos el archivo si existe
	removeFileDirectory(filePath)

	// Creamos archivo
	file, err := os.Create(filePath)
	if err != nil {
		return nil, err
	}
	// Asignamos permisos al archivo
	err = file.Chmod(0777)
	return file, err
}

// Función encargada de crear un directorio
func createDirectory(path string, mode os.FileMode) error {
	if err := os.MkdirAll(path, mode); err != nil {
		return err
	}
	return nil
}

// Función encargada de leer el contenido de un archivo y escibirlo en el writer
func copyContent(file *os.File, writer io.Writer) (int64, error) {
	// Obtenemos la información del archivo
	fileInfo, err := file.Stat()
	if err != nil {
		return 0, err
	}

	// Leemos archivo
	bytes := make([]byte, fileInfo.Size())
	_, err = file.Read(bytes)
	if err != nil {
		return 0, err
	}

	// Guardamos contenido de archivo en writer (archivo contenido dentro del ZIP)
	size, err := writer.Write(bytes)
	return int64(size), err
}

// Función encargada de abrir una ruta (archivo/directorio)
func openPath(path string) (*os.File, error) {
	return os.Open(path)
}

// Función encargada de recorrer un directorio y agregar los archivos al ZIP
func walkDirectory(directory *os.File) error {
	// Leemos contenido del directorio
	content, err := directory.Readdir(-1)
	if err != nil {
		return err
	}

	// Recorremos contenido del directorio
	for _, val := range content {
		if !val.IsDir() {
			// Es un archivo: anrimos el archivo y lo agregamos al Writer
			file, err := openPath(directory.Name() + "/" + val.Name())
			if err != nil {
				return err
			}

			// Agregamos un archivo al ZIP
			fileWriter, err := zipWriter.Create(file.Name())
			if err != nil {
				file.Close()
				return err
			}
			// Escrivimos sobre el nuevo archivo ZIP
			_, err = copyContent(file, fileWriter)
		} else {
			// Es un directorio: abrimos el directorio y recorremos su contenido
			dir, err := openPath(directory.Name() + "/" + val.Name())
			if err != nil {
				return err
			}
			err = walkDirectory(dir)
			if err != nil {
				return err
			}
		}
	}

	// Cerramos directorio
	directory.Close()
	return nil
}

// Función encargada de crear un archivo ZIP y escribir sobre él el contenido
// de un directorio/archivo
func Zip(startPath, finalFileName, finalFilePath string) error {
	// Abrimos la ruta inicial
	startDirectory, err := openPath(startPath)
	if err != nil {
		return err
	}
	// Cerramos la ruta inicial
	defer startDirectory.Close()

	// Creamos directorio destino
	if err := createDirectory(finalFilePath, 0777); err != nil {
		return err
	}

	// Creamos archivo ZIP
	zipFile, err = createFile(finalFilePath + finalFileName)
	if err != nil {
		return err
	}
	// Cerramos archivo ZIP
	defer zipFile.Close()

	// Creamos writer para el archivo ZIP
	zipWriter = zip.NewWriter(zipFile)
	// Cerramos Writer
	defer zipWriter.Close()

	// Iniciamos recorrido del directorio
	if err := walkDirectory(startDirectory); err != nil {
		return err
	}
	return nil
}
<commit_msg>testing<commit_after>// Paquete encargado de comprimir un archivo zip en 32 bits
package zip

import (
	"archive/zip"
	"io"
	"os"
)

// Función encargada de elimina un directorio/archivo
func removeFileDirectory(path string) error {
	return os.Remove(path)
}

// Función encargada de crear un archivo
func createFile(filePath string) (*os.File, error) {
	// Eliminamos el archivo si existe
	removeFileDirectory(filePath)

	// Creamos archivo
	file, err := os.Create(filePath)
	if err != nil {
		return nil, err
	}
	// Asignamos permisos al archivo
	err = file.Chmod(0777)
	return file, err
}

// Función encargada de crear un directorio
func createDirectory(path string, mode os.FileMode) error {
	if err := os.MkdirAll(path, mode); err != nil {
		return err
	}
	return nil
}

// Función encargada de leer el contenido de un archivo y escibirlo en el writer
func copyContent(file *os.File, writer io.Writer) (int64, error) {
	// Obtenemos la información del archivo
	fileInfo, err := file.Stat()
	if err != nil {
		return 0, err
	}

	// Leemos archivo
	bytes := make([]byte, fileInfo.Size())
	_, err = file.Read(bytes)
	if err != nil {
		return 0, err
	}

	// Guardamos contenido de archivo en writer (archivo contenido dentro del ZIP)
	size, err := writer.Write(bytes)
	return int64(size), err
}

// Función encargada de abrir una ruta (archivo/directorio)
func openPath(path string) (*os.File, error) {
	return os.Open(path)
}

// Función encargada de recorrer un directorio y agregar los archivos al ZIP
func walkDirectory(directory *os.File, zipWriter *zip.Writer) error {
	// Leemos contenido del directorio
	content, err := directory.Readdir(-1)
	if err != nil {
		return err
	}

	// Recorremos contenido del directorio
	for _, val := range content {
		if !val.IsDir() {
			// Es un archivo: anrimos el archivo y lo agregamos al Writer
			file, err := openPath(directory.Name() + "/" + val.Name())
			if err != nil {
				return err
			}

			// Agregamos un archivo al ZIP
			fileWriter, err := zipWriter.Create(file.Name())
			if err != nil {
				file.Close()
				return err
			}
			// Escrivimos sobre el nuevo archivo ZIP
			_, err = copyContent(file, fileWriter)
		} else {
			// Es un directorio: abrimos el directorio y recorremos su contenido
			dir, err := openPath(directory.Name() + "/" + val.Name())
			if err != nil {
				return err
			}
			err = walkDirectory(dir, zipWriter)
			if err != nil {
				return err
			}
		}
	}

	// Cerramos directorio
	directory.Close()
	return nil
}

// Función encargada de crear un archivo ZIP y escribir sobre él el contenido
// de un directorio/archivo
func Zip(startPath, finalFileName, finalFilePath string) error {
	// Abrimos la ruta inicial
	startDirectory, err := openPath(startPath)
	if err != nil {
		return err
	}
	// Cerramos la ruta inicial
	defer startDirectory.Close()

	// Creamos directorio destino
	if err := createDirectory(finalFilePath, 0777); err != nil {
		return err
	}

	// Creamos archivo ZIP
	zipFile, err := createFile(finalFilePath + finalFileName)
	if err != nil {
		return err
	}
	// Cerramos archivo ZIP
	defer zipFile.Close()

	// Creamos writer para el archivo ZIP
	zipWriter := zip.NewWriter(zipFile)
	// Cerramos Writer
	defer zipWriter.Close()

	// Iniciamos recorrido del directorio
	if err := walkDirectory(startDirectory, zipWriter); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package main_test

import (
	"os"
	"os/exec"
	"path"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"testing"
)

func TestMain(t *testing.T) {
	RegisterFailHandler(Fail)
	dir, err := os.Getwd()
	Expect(err).NotTo(HaveOccurred())

	cmd := exec.Command("go", "build", "-o", path.Join(dir, "..", "fixtures", "plugins", "test"), path.Join(dir, "..", "fixtures", "plugins", "test.go"))
	err = cmd.Run()
	Expect(err).NotTo(HaveOccurred())

	cmd = exec.Command("go", "build", "-o", path.Join(dir, "..", "fixtures", "plugins", "plugin2"), path.Join(dir, "..", "fixtures", "plugins", "plugin2.go"))
	err = cmd.Run()
	Expect(err).NotTo(HaveOccurred())

	RunSpecs(t, "Main Suite")
}
<commit_msg>Add ginkgo defer to allow us to see error message<commit_after>package main_test

import (
	"os"
	"os/exec"
	"path"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"testing"
)

func TestMain(t *testing.T) {
	RegisterFailHandler(Fail)
	dir, err := os.Getwd()
	Expect(err).NotTo(HaveOccurred())

	cmd := exec.Command("go", "build", "-o", path.Join(dir, "..", "fixtures", "plugins", "test"), path.Join(dir, "..", "fixtures", "plugins", "test.go"))
	err = cmd.Run()
	defer GinkgoRecover()
	Expect(err).NotTo(HaveOccurred())

	cmd = exec.Command("go", "build", "-o", path.Join(dir, "..", "fixtures", "plugins", "plugin2"), path.Join(dir, "..", "fixtures", "plugins", "plugin2.go"))
	err = cmd.Run()
	Expect(err).NotTo(HaveOccurred())

	RunSpecs(t, "Main Suite")
}
<|endoftext|>


<commit_before>package aliases

import "time"

//Alias map user defined project names to a Common Platform Enumeration (CPE).
type Alias struct {
	ID        string    `json:"id"`
	Name      string    `json:"name"`
	CreatedAt time.Time `json:"created_at"`
	UpdatedAt time.Time `json:"updated_at"`
	Version   string    `json:"version"`
}
<commit_msg>adding org field to the alias struct<commit_after>package aliases

import "time"

//Alias map user defined project names to a Common Platform Enumeration (CPE).
type Alias struct {
	ID        string    `json:"id"`
	Name      string    `json:"name"`
	Org       string    `json:"org"`
	CreatedAt time.Time `json:"created_at"`
	UpdatedAt time.Time `json:"updated_at"`
	Version   string    `json:"version"`
}
<|endoftext|>


<commit_before>// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package project

import (
	"context"
	"fmt"

	"code.gitea.io/gitea/models/db"
)

// ProjectIssue saves relation from issue to a project
type ProjectIssue struct { //revive:disable-line:exported
	ID        int64 `xorm:"pk autoincr"`
	IssueID   int64 `xorm:"INDEX"`
	ProjectID int64 `xorm:"INDEX"`

	// If 0, then it has not been added to a specific board in the project
	ProjectBoardID int64 `xorm:"INDEX"`
}

func init() {
	db.RegisterModel(new(ProjectIssue))
}

func deleteProjectIssuesByProjectID(e db.Engine, projectID int64) error {
	_, err := e.Where("project_id=?", projectID).Delete(&ProjectIssue{})
	return err
}

// NumIssues return counter of all issues assigned to a project
func (p *Project) NumIssues() int {
	c, err := db.GetEngine(db.DefaultContext).Table("project_issue").
		Where("project_id=?", p.ID).
		GroupBy("issue_id").
		Cols("issue_id").
		Count()
	if err != nil {
		return 0
	}
	return int(c)
}

// NumClosedIssues return counter of closed issues assigned to a project
func (p *Project) NumClosedIssues() int {
	c, err := db.GetEngine(db.DefaultContext).Table("project_issue").
		Join("INNER", "issue", "project_issue.issue_id=issue.id").
		Where("project_issue.project_id=? AND issue.is_closed=?", p.ID, true).
		Cols("issue_id").
		Count()
	if err != nil {
		return 0
	}
	return int(c)
}

// NumOpenIssues return counter of open issues assigned to a project
func (p *Project) NumOpenIssues() int {
	c, err := db.GetEngine(db.DefaultContext).Table("project_issue").
		Join("INNER", "issue", "project_issue.issue_id=issue.id").
		Where("project_issue.project_id=? AND issue.is_closed=?", p.ID, false).Count("issue.id")
	if err != nil {
		return 0
	}
	return int(c)
}

// MoveIssuesOnProjectBoard moves or keeps issues in a column and sorts them inside that column
func MoveIssuesOnProjectBoard(board *Board, sortedIssueIDs map[int64]int64) error {
	return db.WithTx(func(ctx context.Context) error {
		sess := db.GetEngine(ctx)

		issueIDs := make([]int64, 0, len(sortedIssueIDs))
		for _, issueID := range sortedIssueIDs {
			issueIDs = append(issueIDs, issueID)
		}
		count, err := sess.Table(new(ProjectIssue)).Where("project_id=?", board.ProjectID).In("issue_id", issueIDs).Count()
		if err != nil {
			return err
		}
		if int(count) != len(sortedIssueIDs) {
			return fmt.Errorf("all issues have to be added to a project first")
		}

		for sorting, issueID := range sortedIssueIDs {
			_, err = sess.Exec("UPDATE `project_issue` SET project_board_id=?, sorting=? WHERE issue_id=?", board.ID, sorting, issueID)
			if err != nil {
				return err
			}
		}
		return nil
	})
}

func (pb *Board) removeIssues(e db.Engine) error {
	_, err := e.Exec("UPDATE `project_issue` SET project_board_id = 0 WHERE project_board_id = ? ", pb.ID)
	return err
}
<commit_msg>Add missing `sorting` column in `project_issue` table (#19635)<commit_after>// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package project

import (
	"context"
	"fmt"

	"code.gitea.io/gitea/models/db"
)

// ProjectIssue saves relation from issue to a project
type ProjectIssue struct { //revive:disable-line:exported
	ID        int64 `xorm:"pk autoincr"`
	IssueID   int64 `xorm:"INDEX"`
	ProjectID int64 `xorm:"INDEX"`

	// If 0, then it has not been added to a specific board in the project
	ProjectBoardID int64 `xorm:"INDEX"`

	// the sorting order on the board
	Sorting int64 `xorm:"NOT NULL DEFAULT 0"`
}

func init() {
	db.RegisterModel(new(ProjectIssue))
}

func deleteProjectIssuesByProjectID(e db.Engine, projectID int64) error {
	_, err := e.Where("project_id=?", projectID).Delete(&ProjectIssue{})
	return err
}

// NumIssues return counter of all issues assigned to a project
func (p *Project) NumIssues() int {
	c, err := db.GetEngine(db.DefaultContext).Table("project_issue").
		Where("project_id=?", p.ID).
		GroupBy("issue_id").
		Cols("issue_id").
		Count()
	if err != nil {
		return 0
	}
	return int(c)
}

// NumClosedIssues return counter of closed issues assigned to a project
func (p *Project) NumClosedIssues() int {
	c, err := db.GetEngine(db.DefaultContext).Table("project_issue").
		Join("INNER", "issue", "project_issue.issue_id=issue.id").
		Where("project_issue.project_id=? AND issue.is_closed=?", p.ID, true).
		Cols("issue_id").
		Count()
	if err != nil {
		return 0
	}
	return int(c)
}

// NumOpenIssues return counter of open issues assigned to a project
func (p *Project) NumOpenIssues() int {
	c, err := db.GetEngine(db.DefaultContext).Table("project_issue").
		Join("INNER", "issue", "project_issue.issue_id=issue.id").
		Where("project_issue.project_id=? AND issue.is_closed=?", p.ID, false).Count("issue.id")
	if err != nil {
		return 0
	}
	return int(c)
}

// MoveIssuesOnProjectBoard moves or keeps issues in a column and sorts them inside that column
func MoveIssuesOnProjectBoard(board *Board, sortedIssueIDs map[int64]int64) error {
	return db.WithTx(func(ctx context.Context) error {
		sess := db.GetEngine(ctx)

		issueIDs := make([]int64, 0, len(sortedIssueIDs))
		for _, issueID := range sortedIssueIDs {
			issueIDs = append(issueIDs, issueID)
		}
		count, err := sess.Table(new(ProjectIssue)).Where("project_id=?", board.ProjectID).In("issue_id", issueIDs).Count()
		if err != nil {
			return err
		}
		if int(count) != len(sortedIssueIDs) {
			return fmt.Errorf("all issues have to be added to a project first")
		}

		for sorting, issueID := range sortedIssueIDs {
			_, err = sess.Exec("UPDATE `project_issue` SET project_board_id=?, sorting=? WHERE issue_id=?", board.ID, sorting, issueID)
			if err != nil {
				return err
			}
		}
		return nil
	})
}

func (pb *Board) removeIssues(e db.Engine) error {
	_, err := e.Exec("UPDATE `project_issue` SET project_board_id = 0 WHERE project_board_id = ? ", pb.ID)
	return err
}
<|endoftext|>


<commit_before>package messages

import "database/sql"

func HandleOIP041Edit(o Oip041, txid string, block int, dbtx *sql.Tx) error {

	return nil
}
<commit_msg>Early stages of oip edits<commit_after>package messages

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"log"
	"strings"
)

func HandleOIP041Edit(o Oip041, txid string, block int, dbtx *sql.Tx) error {

	// ToDo: Check the signature... but first decide what to sign

	stmtstr := `SELECT ('json','txid','publisher')
		FROM oip_artifact WHERE txid=? LIMIT 1`

	stmt, err := dbtx.Prepare(stmtstr)
	if err != nil {
		fmt.Println("RIP Handle Edit 1")
		log.Fatal(err)
	}
	row := stmt.QueryRow(txid)

	var old Oip041
	var sJSON string
	var publisher string
	var txID string

	err = row.Scan(&sJSON, &txID, &publisher)
	if err != nil {
		fmt.Println("RIP Handle Edit 2")
		log.Fatal(err)
	}

	err = json.Unmarshal([]byte(sJSON), &old)
	if err != nil {
		fmt.Println("RIP Handle Edit 3")
		return err
	}

	if len(o.Edit.Add) > 0 {
		// we got stuff to add
	}
	if len(o.Edit.Edit) > 0 {
		// we got stuff to edit
		for k, v := range o.Edit.Edit {
			p := strings.Split(k, ".")
			if len(p) == 1 {
				updateField(k, v, txid, dbtx)
				// something along these lines...
			}
		}
	}
	if len(o.Edit.Remove) > 0 {
		// we got stuff to remove
	}

	b, err := json.Marshal(old)
	if err != nil {
		fmt.Println("RIP Handle Edit 4")
		return err
	}

	fmt.Println(string(b))

	return nil
}

/*
var oip041_edit_example_obj Oip041 = Oip041{
	Edit: Oip041Edit{
		Add: map[string]string{
			"payment.tokens": "FREEBIEOFTHEWEEK:\"1\"",
		},
		Edit: map[string]string{
			"files[0].dname": "Throwing Stones",
			"files[0].fname": "1 - Throwing Stones.mp3",
		},
		Remove: []string{
			"tokens.LTBCOIN",
		},
		Timestamp: 1234,
		TxID:      "96bad8e17f908da4c695c58b0f843a03928e338b361b3035ed16a864eafc31a2",
	},
	Signature: "<SignatureOfSomething>",
}
*/

func updateField(key string, value string, txid string, dbtx *sql.Tx) {
	stmtstr := `UPDATE oip_artifact SET ?=? WHERE txid=?`

	stmt, err := dbtx.Prepare(stmtstr)
	if err != nil {
		fmt.Println("RIP Update Field 1")
		log.Fatal(err)
	}

	_, stmterr := stmt.Exec(key, value, txid)
	if err != nil {
		fmt.Println("RIP Update Field 1")
		log.Fatal(stmterr)
	}

	stmt.Close()
}
<|endoftext|>


<commit_before>package host

import (
	"path/filepath"

	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
)

type savedHost struct {
	FileCounter  int
	Obligations  []contractObligation
	HostSettings modules.HostSettings
}

func (h *Host) save() (err error) {
	sHost := savedHost{
		FileCounter:  h.fileCounter,
		Obligations:  make([]contractObligation, 0, len(h.obligationsByID)),
		HostSettings: h.HostSettings,
	}
	for _, obligation := range h.obligationsByID {
		sHost.Obligations = append(sHost.Obligations, obligation)
	}

	return encoding.WriteFile(filepath.Join(h.saveDir, "settings.dat"), sHost)
}

func (h *Host) load() error {
	var sHost savedHost
	err := encoding.ReadFile(filepath.Join(h.saveDir, "settings.dat"), &sHost)
	if err != nil {
		return err
	}

	h.spaceRemaining = sHost.HostSettings.TotalStorage
	h.fileCounter = sHost.FileCounter
	h.HostSettings = sHost.HostSettings
	// recreate maps
	for _, obligation := range sHost.Obligations {
		height := obligation.FileContract.WindowStart + StorageProofReorgDepth
		h.obligationsByHeight[height] = append(h.obligationsByHeight[height], obligation)
		h.obligationsByID[obligation.ID] = obligation
		// update spaceRemaining
		h.spaceRemaining -= int64(obligation.FileContract.FileSize)
	}

	return nil
}
<commit_msg>use persist in host<commit_after>package host

import (
	"path/filepath"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/persist"
)

var persistMetadata = persist.Metadata{
	Header:  "Sia Host",
	Version: "0.3.3",
}

type savedHost struct {
	FileCounter  int
	Obligations  []contractObligation
	HostSettings modules.HostSettings
}

func (h *Host) save() error {
	sHost := savedHost{
		FileCounter:  h.fileCounter,
		Obligations:  make([]contractObligation, 0, len(h.obligationsByID)),
		HostSettings: h.HostSettings,
	}
	for _, obligation := range h.obligationsByID {
		sHost.Obligations = append(sHost.Obligations, obligation)
	}

	return persist.SaveFile(persistMetadata, sHost, filepath.Join(h.saveDir, "settings.json"))
}

func (h *Host) load() error {
	var sHost savedHost
	err := persist.LoadFile(persistMetadata, &sHost, filepath.Join(h.saveDir, "settings.json"))
	if err != nil {
		return err
	}

	h.spaceRemaining = sHost.HostSettings.TotalStorage
	h.fileCounter = sHost.FileCounter
	h.HostSettings = sHost.HostSettings
	// recreate maps
	for _, obligation := range sHost.Obligations {
		height := obligation.FileContract.WindowStart + StorageProofReorgDepth
		h.obligationsByHeight[height] = append(h.obligationsByHeight[height], obligation)
		h.obligationsByID[obligation.ID] = obligation
		// update spaceRemaining
		h.spaceRemaining -= int64(obligation.FileContract.FileSize)
	}

	return nil
}
<|endoftext|>


<commit_before>package changelog

import (
	"reflect"
	"testing"
)

var raw1 = `# Big Changelog

introduction stuff

##         unRelEASed - don't release me!

not ready

## 1.2.3 2016-10-10

Last Release

## 1.2.0 2016-01-01

First Release
`

var raw2 = `# Big Changelog

introduction stuff

## Unreleased

not ready

## 1.2.3

Last Release, invalid, missing date

## 1.2.0 2016-01-01

First Release
`

func TestCLParse(t *testing.T) {
	cl, err := Parse(raw1)
	if err != nil {
		t.Fatalf("could not parse: %s", err)
	}
	if len(cl.Released) != 2 {
		t.Errorf("Expected two releases, got %d", len(cl.Released))
	}
	if cl.Unreleased != "not ready" {
		t.Errorf("Parse error on cl.Unreleased: %q", cl.Unreleased)
	}
	if cl.Intro != "# Big Changelog\n\nintroduction stuff" {
		t.Errorf("Parse error of cl.Intro: %q", cl.Intro)
	}
}

func TestCLParseError(t *testing.T) {
	_, err := Parse(raw2)
	if err == nil {
		t.Errorf("invalid changelog was parsed!")
	}
}

func TestMostRecent(t *testing.T) {
	// MostRecent on empty changelog should return empty Entry
	cl := ChangeLog{}
	want := Entry{}
	got := cl.Top()
	if !reflect.DeepEqual(got, want) {
		t.Errorf("cl.MostRecent = %v, want %v", got, want)
	}
	cl, err := Parse(raw1)
	if err != nil {
		t.Fatalf("could not parse: %s", err)
	}
	want = Entry{"1.2.3", "2016-10-10", "Last Release"}
	got = cl.Top()
	if !reflect.DeepEqual(got, want) {
		t.Errorf("cl.MostRecent = %v, want %v", got, want)
	}
}
<commit_msg>add MarshalText test<commit_after>package changelog

import (
	"bytes"
	"log"
	"reflect"
	"testing"
)

var raw1 = `# Big Changelog

introduction stuff

##         unRelEASed - don't release me!

not ready

## 1.2.3 2016-10-10

Last Release

## 1.2.0 2016-01-01

First Release
`

var raw2 = `# Big Changelog

introduction stuff

## Unreleased

not ready

## 1.2.3

Last Release, invalid, missing date

## 1.2.0 2016-01-01

First Release
`

func TestCLParse(t *testing.T) {
	cl, err := Parse(raw1)
	if err != nil {
		t.Fatalf("could not parse: %s", err)
	}
	if len(cl.Released) != 2 {
		t.Errorf("Expected two releases, got %d", len(cl.Released))
	}
	if cl.Unreleased != "not ready" {
		t.Errorf("Parse error on cl.Unreleased: %q", cl.Unreleased)
	}
	if cl.Intro != "# Big Changelog\n\nintroduction stuff" {
		t.Errorf("Parse error of cl.Intro: %q", cl.Intro)
	}
}

func TestCLParseError(t *testing.T) {
	_, err := Parse(raw2)
	if err == nil {
		t.Errorf("invalid changelog was parsed!")
	}
}

func TestMostRecent(t *testing.T) {
	// MostRecent on empty changelog should return empty Entry
	cl := ChangeLog{}
	want := Entry{}
	got := cl.Top()
	if !reflect.DeepEqual(got, want) {
		t.Errorf("cl.MostRecent = %v, want %v", got, want)
	}
	cl, err := Parse(raw1)
	if err != nil {
		t.Fatalf("could not parse: %s", err)
	}
	want = Entry{"1.2.3", "2016-10-10", "Last Release"}
	got = cl.Top()
	if !reflect.DeepEqual(got, want) {
		t.Errorf("cl.MostRecent = %v, want %v", got, want)
	}
}

func TestMarshalText(t *testing.T) {
	cl := ChangeLog{
		Unreleased: "unreleased test",
		Released: []Entry{
			{"1.2.3", "2017-01-01", "notes"},
		},
	}
	b, err := cl.MarshalText()
	if err != nil {
		log.Fatal(err)
	}

	want := []byte(`
## Unreleased

unreleased test

## 1.2.3 2017-01-01

notes
`)
	if !bytes.Equal(b, want) {
		t.Errorf("cl.MarshalText() = %q, want %q", string(b), string(want))
	}

}
<|endoftext|>


<commit_before>// Package msgboard provides a message board application for users to
// create and edit HTML pages shown on a screen.
package msgboard

import (
	"appengine"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/gorilla/mux"
)

func init() {
	r := mux.NewRouter()
	r.HandleFunc("/api/pages", listPagesHandler).Methods("GET")
	r.HandleFunc("/api/pages", createPageHandler).Methods("POST")
	r.HandleFunc("/api/pages/{id}", getPageHandler).Methods("GET")
	r.HandleFunc("/api/pages/{id}", updatePageHandler).Methods("PATCH")
	http.Handle("/api/", r)
}

func listPagesHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	view := r.FormValue("view")

	pages, err := ListPages(ctx, view)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	b, _ := json.Marshal(pages)
	fmt.Fprint(w, string(b))
	return
}

func createPageHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)

	page, err := CreatePage(ctx, r.Body)
	if err != nil {
		ec := http.StatusInternalServerError
		if err == ErrMissingTitle {
			ec = http.StatusBadRequest
		}
		http.Error(w, err.Error(), ec)
		return
	}

	b, _ := json.Marshal(page)
	fmt.Fprint(w, string(b))
	return
}

func getPageHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	vars := mux.Vars(r)
	ID := vars["id"]

	page, err := GetPage(ctx, ID)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	b, _ := json.Marshal(page)
	fmt.Fprint(w, string(b))
	return
}

func updatePageHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	vars := mux.Vars(r)
	ID := vars["id"]

	page, err := UpdatePage(ctx, ID, r.Body)
	if err != nil {
		ec := http.StatusInternalServerError
		if err == ErrMissingTitle {
			ec = http.StatusBadRequest
		}
		http.Error(w, err.Error(), ec)
		return
	}

	b, _ := json.Marshal(page)
	fmt.Fprint(w, string(b))
	return
}
<commit_msg>DRY up the JSON writing and error handling.<commit_after>// Package msgboard provides a message board application for users to
// create and edit HTML pages shown on a screen.
package msgboard

import (
	"appengine"
	"appengine/datastore"
	"encoding/json"
	"net/http"

	"github.com/gorilla/mux"
)

func init() {
	r := mux.NewRouter()
	r.HandleFunc("/api/pages", listPagesHandler).Methods("GET")
	r.HandleFunc("/api/pages", createPageHandler).Methods("POST")
	r.HandleFunc("/api/pages/{id}", getPageHandler).Methods("GET")
	r.HandleFunc("/api/pages/{id}", updatePageHandler).Methods("PATCH")
	http.Handle("/api/", r)
}

func writeJSON(w http.ResponseWriter, v interface{}) {
	b, err := json.Marshal(v)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.Write(b)
	return
}

func writeError(w http.ResponseWriter, err error) {
	ec := http.StatusInternalServerError
	switch err {
	case datastore.ErrNoSuchEntity:
		ec = http.StatusNotFound
	case ErrMissingTitle:
		ec = http.StatusBadRequest
	}
	http.Error(w, err.Error(), ec)
	return
}

func listPagesHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	view := r.FormValue("view")

	pages, err := ListPages(ctx, view)
	if err != nil {
		writeError(w, err)
		return
	}
	writeJSON(w, pages)
	return
}

func createPageHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)

	page, err := CreatePage(ctx, r.Body)
	if err != nil {
		writeError(w, err)
		return
	}

	writeJSON(w, page)
	return
}

func getPageHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	vars := mux.Vars(r)
	ID := vars["id"]

	page, err := GetPage(ctx, ID)
	if err != nil {
		writeError(w, err)
		return
	}

	writeJSON(w, page)
	return
}

func updatePageHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	vars := mux.Vars(r)
	ID := vars["id"]

	page, err := UpdatePage(ctx, ID, r.Body)
	if err != nil {
		writeError(w, err)
		return
	}

	writeJSON(w, page)
	return
}
<|endoftext|>


<commit_before>package coinbase

import (
	"fmt"
	"os"
	"testing"
)

func createAccountClient(t *testing.T) (c *Client) {
	c = &Client{
		APIKey: os.Getenv("COINBASE_API_KEY"),
	}

	if c.APIKey == "" {
		t.Error("Coinbase api key is missing (should be in the COINBASE_API_KEY environment variable)")
	}

	return c
}

func TestAccountBalance(t *testing.T) {
	c := createAccountClient(t)

	balance, err := c.AccountBalance()
	if err != nil {
		t.Fatal(err)
	}

	fmt.Println(balance)
}
<commit_msg>Skip tests that require auth<commit_after>package coinbase

import (
	"fmt"
	"os"
	"testing"
)

func createAccountClient(t *testing.T) (c *Client) {
	c = &Client{
		APIKey: os.Getenv("COINBASE_API_KEY"),
	}

	if c.APIKey == "" {
		t.Skip("Coinbase api key is missing (should be in the COINBASE_API_KEY environment variable)")
	}

	return c
}

func TestAccountBalance(t *testing.T) {
	c := createAccountClient(t)

	balance, err := c.AccountBalance()
	if err != nil {
		t.Fatal(err)
	}

	fmt.Println(balance)
}
<|endoftext|>


<commit_before>// Package genetics holds data holders and helper utilities used to implement genetic evolution algorithm
package genetics

import (
	"github.com/leesper/go_rng"
	"time"
)

// The innovation method to be applied
const (
	// The novelty will be introduced by new NN node
	NEWNODE = iota
	// The novelty will be introduced by new NN link
	NEWLINK
)

// Mutators are variables that specify a kind of mutation of connection weights between NN nodes
const (
	//This adds Gaussian noise to the weights
	GAUSSIAN = iota
	//This sets weights to numbers chosen from a Gaussian distribution
	COLDGAUSSIAN
)

// The Gaussian random number generator
const gaussian = rng.NewGaussianGenerator(time.Now().UnixNano())
<commit_msg>Changed RNG to be var instead of const<commit_after>// Package genetics holds data holders and helper utilities used to implement genetic evolution algorithm
package genetics

import (
	"github.com/leesper/go_rng"
	"time"
)

// The innovation method to be applied
const (
	// The novelty will be introduced by new NN node
	NEWNODE = iota
	// The novelty will be introduced by new NN link
	NEWLINK
)

// Mutators are variables that specify a kind of mutation of connection weights between NN nodes
const (
	//This adds Gaussian noise to the weights
	GAUSSIAN = iota
	//This sets weights to numbers chosen from a Gaussian distribution
	COLDGAUSSIAN
)

// The Gaussian random number generator
var gaussian = rng.NewGaussianGenerator(time.Now().UnixNano())
<|endoftext|>


<commit_before>package ma

import (
	"yap/alg/graph"
	"yap/nlp/format/lex"
	. "yap/nlp/types"
	"yap/util"

	"fmt"
	"log"
	"regexp"
)

const ESTIMATED_MORPHS_PER_TOKEN = 5

type BGULex struct {
	MaxPrefixLen int
	Prefixes     map[string][]BasicMorphemes

	Lex map[string][]BasicMorphemes

	Files []string
	Stats *AnalyzeStats
}

var (
	PUNCT = map[string]string{
		":":   "yyCLN",
		",":   "yyCM",
		"-":   "yyDASH",
		".":   "yyDOT",
		"...": "yyELPS",
		"!":   "yyEXCL",
		"(":   "yyLRB",
		"?":   "yyQM",
		")":   "yyRRB",
		";":   "yySCLN",
		"\"":  "yyQUOT",
	}
	REGEX = []struct {
		RE  *regexp.Regexp
		POS string
	}{
		{regexp.MustCompile("^[[:digit:]]+$"), "CD"},
	}
	_ MorphologicalAnalyzer = &BGULex{}
)

func (l *BGULex) loadTokens(file, format string) {
	tokens, err := lex.ReadFile(file, format)
	if err != nil {
		panic(fmt.Sprintf("Failed to load %v: %v", file, err))
	}
	var m map[string][]BasicMorphemes
	if format == "prefix" {
		l.Prefixes = make(map[string][]BasicMorphemes, len(tokens))
		m = l.Prefixes
	} else if format == "lexicon" {
		l.Lex = make(map[string][]BasicMorphemes, len(tokens))
		m = l.Lex
	}
	log.Println("Found", len(tokens), "tokens in lexicon file:", file)
	for _, token := range tokens {
		if cur, exists := m[token.Token]; exists {
			m[token.Token] = append(cur, token.Morphemes...)
		} else {
			m[token.Token] = token.Morphemes
		}
	}
}

func (l *BGULex) LoadPrefixes(file string) {
	l.loadTokens(file, "prefix")
	l.MaxPrefixLen = 0
	for _, morphs := range l.Prefixes {
		if l.MaxPrefixLen < len(morphs) {
			l.MaxPrefixLen = len(morphs)
		}
	}
	log.Println("Loaded", len(l.Prefixes), "prefixes from lexicon")
}

func (l *BGULex) LoadLex(file string) {
	l.loadTokens(file, "lexicon")
	log.Println("Loaded", len(l.Lex), "tokens from lexicon")
}

func makeMorphWithPOS(input, lemma, POS string) []BasicMorphemes {
	return []BasicMorphemes{BasicMorphemes([]*Morpheme{
		&Morpheme{
			BasicDirectedEdge: graph.BasicDirectedEdge{0, 0, 1},
			Form:              input,
			Lemma:             lemma,
			CPOS:              POS,
			POS:               POS,
			FeatureStr:        "",
		},
	})}
}

func (l *BGULex) OOVAnalysis(input string) []BasicMorphemes {
	return makeMorphWithPOS(input, input, "NNP")
}

func checkRegexes(input string) ([]BasicMorphemes, bool) {
	for _, curRegex := range REGEX {
		if curRegex.RE.MatchString(input) {
			return makeMorphWithPOS(input, "", curRegex.POS), true
		}
	}
	return nil, false
}

var logAnalyze bool = false

func (l *BGULex) analyzeTokenForLen(lat *Lattice, input string, startingNode, numToken, prefixLen int) bool {
	var (
		found, hostExists bool
		hostLat           []BasicMorphemes
	)
	if len(input) < prefixLen*2 {
		return found
	}
	prefixLat, prefixExists := l.Prefixes[input[0:prefixLen*2]]
	// log.Println("\tPrefixes", input[0:prefixLen*2], prefixExists)
	if prefixExists {

		hostLat, hostExists = l.Lex[input[2*prefixLen:]]
		if !hostExists {
			hostLat, hostExists = checkRegexes(input[2*prefixLen:])
		}
		// log.Println("\tHosts", input[2*prefixLen:], hostExists)
		if hostExists {
			for _, prefix := range prefixLat {
				// log.Println("\t\tAdding", prefix, hostLat)
				lat.AddAnalysis(prefix, hostLat, numToken)
			}
			found = true
		}
	}
	return found
}

func (l *BGULex) AnalyzeToken(input string, startingNode, numToken int) (*Lattice, interface{}) {
	if logAnalyze {
		log.Println("Analyzing token", numToken, "starting at", startingNode)
	}
	lat := &Lattice{
		Token:     Token(input),
		Morphemes: make(Morphemes, 0, ESTIMATED_MORPHS_PER_TOKEN),
		Next:      make(map[int][]int, ESTIMATED_MORPHS_PER_TOKEN),
		BottomId:  startingNode,
		TopId:     startingNode,
	}
	lat.Next[0] = make([]int, 0, 1)
	var (
		hostLat               []BasicMorphemes
		hostExists, anyExists bool
	)
	if punctVal, exists := PUNCT[input]; exists {
		m := &Morpheme{
			BasicDirectedEdge: graph.BasicDirectedEdge{0, 0, 0},
			Form:              input,
			CPOS:              punctVal,
			POS:               punctVal,
		}
		basics := []BasicMorphemes{BasicMorphemes{m}}
		lat.AddAnalysis(nil, basics, numToken)
		return lat, nil
	}
	hostLat, hostExists = l.Lex[input]
	if !hostExists {
		hostLat, hostExists = checkRegexes(input)
	}
	if hostExists {
		if logAnalyze {
			log.Println("\tPrefix 0")
		}
		lat.AddAnalysis(nil, hostLat, numToken)
		anyExists = true
	} else {
		hostLat = l.OOVAnalysis(input)
		lat.AddAnalysis(nil, hostLat, numToken)
	}
	for i := 1; i < util.Min(l.MaxPrefixLen, len(input)); i++ {
		if logAnalyze {
			log.Println("\ti is", i)
		}
		found := l.analyzeTokenForLen(lat, input, startingNode, numToken, i)
		anyExists = anyExists || found
	}
	if !anyExists {
		// if logAnalyze {
		log.Println("Token", numToken, "is OOV:", input)
		// }
		if l.Stats != nil {
			l.Stats.OOVTokens++
			l.Stats.AddOOVToken(input)
		}
	}
	lat.Optimize()
	return lat, nil
}

func (l *BGULex) Analyze(input []string) (LatticeSentence, interface{}) {
	retval := make(LatticeSentence, len(input))
	var (
		lat     *Lattice
		curNode int
	)
	for i, token := range input {
		if l.Stats != nil {
			l.Stats.TotalTokens++
			l.Stats.AddToken(token)
		}
		lat, _ = l.AnalyzeToken(token, curNode, i)
		curNode = lat.Top()
		// log.Println("New top is", curNode)
		retval[i] = *lat
	}
	return retval, nil
}
<commit_msg>Add fallback in hebrew lexicon MA for non-number token with digits<commit_after>package ma

import (
	"yap/alg/graph"
	"yap/nlp/format/lex"
	. "yap/nlp/types"
	"yap/util"

	"fmt"
	"log"
	"regexp"
)

const ESTIMATED_MORPHS_PER_TOKEN = 5

type BGULex struct {
	MaxPrefixLen int
	Prefixes     map[string][]BasicMorphemes

	Lex map[string][]BasicMorphemes

	Files []string
	Stats *AnalyzeStats
}

var (
	PUNCT = map[string]string{
		":":   "yyCLN",
		",":   "yyCM",
		"-":   "yyDASH",
		".":   "yyDOT",
		"...": "yyELPS",
		"!":   "yyEXCL",
		"(":   "yyLRB",
		"?":   "yyQM",
		")":   "yyRRB",
		";":   "yySCLN",
		"\"":  "yyQUOT",
	}
	REGEX = []struct {
		RE  *regexp.Regexp
		POS string
	}{
		{regexp.MustCompile("^[[:digit:]]+$"), "CD"},
		{regexp.MustCompile("[[:digit:]]"), "NCD"},
	}
	_ MorphologicalAnalyzer = &BGULex{}
)

func (l *BGULex) loadTokens(file, format string) {
	tokens, err := lex.ReadFile(file, format)
	if err != nil {
		panic(fmt.Sprintf("Failed to load %v: %v", file, err))
	}
	var m map[string][]BasicMorphemes
	if format == "prefix" {
		l.Prefixes = make(map[string][]BasicMorphemes, len(tokens))
		m = l.Prefixes
	} else if format == "lexicon" {
		l.Lex = make(map[string][]BasicMorphemes, len(tokens))
		m = l.Lex
	}
	log.Println("Found", len(tokens), "tokens in lexicon file:", file)
	for _, token := range tokens {
		if cur, exists := m[token.Token]; exists {
			m[token.Token] = append(cur, token.Morphemes...)
		} else {
			m[token.Token] = token.Morphemes
		}
	}
}

func (l *BGULex) LoadPrefixes(file string) {
	l.loadTokens(file, "prefix")
	l.MaxPrefixLen = 0
	for _, morphs := range l.Prefixes {
		if l.MaxPrefixLen < len(morphs) {
			l.MaxPrefixLen = len(morphs)
		}
	}
	log.Println("Loaded", len(l.Prefixes), "prefixes from lexicon")
}

func (l *BGULex) LoadLex(file string) {
	l.loadTokens(file, "lexicon")
	log.Println("Loaded", len(l.Lex), "tokens from lexicon")
}

func makeMorphWithPOS(input, lemma, POS string) []BasicMorphemes {
	return []BasicMorphemes{BasicMorphemes([]*Morpheme{
		&Morpheme{
			BasicDirectedEdge: graph.BasicDirectedEdge{0, 0, 1},
			Form:              input,
			Lemma:             lemma,
			CPOS:              POS,
			POS:               POS,
			FeatureStr:        "",
		},
	})}
}

func (l *BGULex) OOVAnalysis(input string) []BasicMorphemes {
	return makeMorphWithPOS(input, input, "NNP")
}

func checkRegexes(input string) ([]BasicMorphemes, bool) {
	for _, curRegex := range REGEX {
		if curRegex.RE.MatchString(input) {
			return makeMorphWithPOS(input, "", curRegex.POS), true
		}
	}
	return nil, false
}

var logAnalyze bool = false

func (l *BGULex) analyzeTokenForLen(lat *Lattice, input string, startingNode, numToken, prefixLen int) bool {
	var (
		found, hostExists bool
		hostLat           []BasicMorphemes
	)
	if len(input) < prefixLen*2 {
		return found
	}
	prefixLat, prefixExists := l.Prefixes[input[0:prefixLen*2]]
	// log.Println("\tPrefixes", input[0:prefixLen*2], prefixExists)
	if prefixExists {

		hostLat, hostExists = l.Lex[input[2*prefixLen:]]
		if !hostExists {
			hostLat, hostExists = checkRegexes(input[2*prefixLen:])
		}
		// log.Println("\tHosts", input[2*prefixLen:], hostExists)
		if hostExists {
			for _, prefix := range prefixLat {
				// log.Println("\t\tAdding", prefix, hostLat)
				lat.AddAnalysis(prefix, hostLat, numToken)
			}
			found = true
		}
	}
	return found
}

func (l *BGULex) AnalyzeToken(input string, startingNode, numToken int) (*Lattice, interface{}) {
	if logAnalyze {
		log.Println("Analyzing token", numToken, "starting at", startingNode)
	}
	lat := &Lattice{
		Token:     Token(input),
		Morphemes: make(Morphemes, 0, ESTIMATED_MORPHS_PER_TOKEN),
		Next:      make(map[int][]int, ESTIMATED_MORPHS_PER_TOKEN),
		BottomId:  startingNode,
		TopId:     startingNode,
	}
	lat.Next[0] = make([]int, 0, 1)
	var (
		hostLat               []BasicMorphemes
		hostExists, anyExists bool
	)
	if punctVal, exists := PUNCT[input]; exists {
		m := &Morpheme{
			BasicDirectedEdge: graph.BasicDirectedEdge{0, 0, 0},
			Form:              input,
			CPOS:              punctVal,
			POS:               punctVal,
		}
		basics := []BasicMorphemes{BasicMorphemes{m}}
		lat.AddAnalysis(nil, basics, numToken)
		return lat, nil
	}
	hostLat, hostExists = l.Lex[input]
	if !hostExists {
		hostLat, hostExists = checkRegexes(input)
	}
	if hostExists {
		if logAnalyze {
			log.Println("\tPrefix 0")
		}
		lat.AddAnalysis(nil, hostLat, numToken)
		anyExists = true
	} else {
		hostLat = l.OOVAnalysis(input)
		lat.AddAnalysis(nil, hostLat, numToken)
	}
	for i := 1; i < util.Min(l.MaxPrefixLen, len(input)); i++ {
		if logAnalyze {
			log.Println("\ti is", i)
		}
		found := l.analyzeTokenForLen(lat, input, startingNode, numToken, i)
		anyExists = anyExists || found
	}
	if !anyExists {
		// if logAnalyze {
		log.Println("Token", numToken, "is OOV:", input)
		// }
		if l.Stats != nil {
			l.Stats.OOVTokens++
			l.Stats.AddOOVToken(input)
		}
	}
	lat.Optimize()
	return lat, nil
}

func (l *BGULex) Analyze(input []string) (LatticeSentence, interface{}) {
	retval := make(LatticeSentence, len(input))
	var (
		lat     *Lattice
		curNode int
	)
	for i, token := range input {
		if l.Stats != nil {
			l.Stats.TotalTokens++
			l.Stats.AddToken(token)
		}
		lat, _ = l.AnalyzeToken(token, curNode, i)
		curNode = lat.Top()
		// log.Println("New top is", curNode)
		retval[i] = *lat
	}
	return retval, nil
}
<|endoftext|>


<commit_before>// Implementation of several algorithms for doing pattern matching with
// strings.  Preprocessing data can be maintained so that the cost of doing
// the preprocessing is not incurred multiple times for the same string or set
// of strings.
//
// This file is a high level interface to these algorithms.  This should be
// sufficient for most applications, but if necessary the underlying
// algorithms are exposed in github.com/runningwild/stringz/core.
package stringz

import (
  "github.com/runningwild/stringz/core"
  "io"
)

type SerialStringFinder struct {
  bmd core.BmData
  buf []byte
  res []int
}

func SerialFind(p []byte) *SerialStringFinder {
  return &SerialStringFinder{
    bmd: core.BoyerMoorePreprocess(p),
  }
}
func (ssf *SerialStringFinder) In(t []byte) []int {
  ssf.res = ssf.res[0:0]
  core.BoyerMoore(ssf.bmd, t, &ssf.res)
  return ssf.res
}

type StringFinder struct {
  bmd core.BmData
}

// Preprocesses p and returns a *StringFinder that can be used to quickly
// search for occurrences of p in other strings.  Uses Boyer-Moore, which
// requires O(n) time to preprocess p, and O(n) space to store the result.
// Methods on StringFinder can be called concurrently from multiple
// go-routines.
func Find(p []byte) *StringFinder {
  return &StringFinder{bmd: core.BoyerMoorePreprocess(p)}
}

// Searches t for the pattern, p, that was used to create the StringFinder.
// Returns a list of all indices at which p occurs in t, including overlaps,
// and in ascending order.  The search takes O(m) time in the worst case, and
// O(m/n) in the best case, where m and n are the lengths of t and p,
// respectively.  The search requires O(k) space, where k is the number of
// times p occurs in t.
func (sf *StringFinder) In(t []byte) []int {
  var res []int
  core.BoyerMoore(sf.bmd, t, &res)
  return res
}

// Like In(), but searches the data from a Reader instead of a []byte.
func (sf *StringFinder) InReader(r io.Reader) []int {
  var res []int
  core.BoyerMooreFromReader(sf.bmd, r, make([]byte, 100000), &res)
  return res
}

type StringSetFinder struct {
  acd core.AcData
}

// Preprocesses ps and returns a *StringSetFinder that can be used to quickly
// search for all occurrences of all elements of ps in other strings.  Uses
// Aho-Corasick, which requires O(n) time to preprocess ps, and O(n) to store
// the result, where n is the sum of the lengths of all of the elements in ps.
// Methods on StringSetFinder can be called concurrently from multiple
// go-routines.
func FindSet(ps [][]byte) *StringSetFinder {
  return &StringSetFinder{acd: core.AhoCorasickPreprocess(ps)}
}

// Searches t for all patterns in the set of patterns, ps, that was used to
// create the StringSetFinder.  Returns a list, H, such that H[i] is a list of
// every index in t at which ps[i] occurs.  Each H[i] includes overlaps and is
// in ascending order.  The search takes O(m) time and O(k) space, where m is
// the length of t, and k is the total number of occurrences of all elements
// of ps in t.
func (ssf *StringSetFinder) In(t []byte) map[int][]int {
  return core.AhoCorasick(ssf.acd, t)
}

// Like In(), but searches the data from a Reader instead of a []byte.
func (sf *StringSetFinder) InReader(input io.Reader) map[int][]int {
  return core.AhoCorasickFromReader(sf.acd, input, 100000)
}
<commit_msg>Removed serial string finder<commit_after>// Implementation of several algorithms for doing pattern matching with
// strings.  Preprocessing data can be maintained so that the cost of doing
// the preprocessing is not incurred multiple times for the same string or set
// of strings.
//
// This file is a high level interface to these algorithms.  This should be
// sufficient for most applications, but if necessary the underlying
// algorithms are exposed in github.com/runningwild/stringz/core.
package stringz

import (
  "github.com/runningwild/stringz/core"
  "io"
)

// This was just an experiment, probably shouldn't publish it yet
// type SerialStringFinder struct {
//   bmd core.BmData
//   buf []byte
//   res []int
// }

// func SerialFind(p []byte) *SerialStringFinder {
//   return &SerialStringFinder{
//     bmd: core.BoyerMoorePreprocess(p),
//   }
// }
// func (ssf *SerialStringFinder) In(t []byte) []int {
//   ssf.res = ssf.res[0:0]
//   core.BoyerMoore(ssf.bmd, t, &ssf.res)
//   return ssf.res
// }

type StringFinder struct {
  bmd core.BmData
}

// Preprocesses p and returns a *StringFinder that can be used to quickly
// search for occurrences of p in other strings.  Uses Boyer-Moore, which
// requires O(n) time to preprocess p, and O(n) space to store the result.
// Methods on StringFinder can be called concurrently from multiple
// go-routines.
func Find(p []byte) *StringFinder {
  return &StringFinder{bmd: core.BoyerMoorePreprocess(p)}
}

// Searches t for the pattern, p, that was used to create the StringFinder.
// Returns a list of all indices at which p occurs in t, including overlaps,
// and in ascending order.  The search takes O(m) time in the worst case, and
// O(m/n) in the best case, where m and n are the lengths of t and p,
// respectively.  The search requires O(k) space, where k is the number of
// times p occurs in t.
func (sf *StringFinder) In(t []byte) []int {
  var res []int
  core.BoyerMoore(sf.bmd, t, &res)
  return res
}

// Like In(), but searches the data from a Reader instead of a []byte.
func (sf *StringFinder) InReader(r io.Reader) []int {
  var res []int
  core.BoyerMooreFromReader(sf.bmd, r, make([]byte, 100000), &res)
  return res
}

type StringSetFinder struct {
  acd core.AcData
}

// Preprocesses ps and returns a *StringSetFinder that can be used to quickly
// search for all occurrences of all elements of ps in other strings.  Uses
// Aho-Corasick, which requires O(n) time to preprocess ps, and O(n) to store
// the result, where n is the sum of the lengths of all of the elements in ps.
// Methods on StringSetFinder can be called concurrently from multiple
// go-routines.
func FindSet(ps [][]byte) *StringSetFinder {
  return &StringSetFinder{acd: core.AhoCorasickPreprocess(ps)}
}

// Searches t for all patterns in the set of patterns, ps, that was used to
// create the StringSetFinder.  Returns a list, H, such that H[i] is a list of
// every index in t at which ps[i] occurs.  Each H[i] includes overlaps and is
// in ascending order.  The search takes O(m) time and O(k) space, where m is
// the length of t, and k is the total number of occurrences of all elements
// of ps in t.
func (ssf *StringSetFinder) In(t []byte) map[int][]int {
  return core.AhoCorasick(ssf.acd, t)
}

// Like In(), but searches the data from a Reader instead of a []byte.
func (sf *StringSetFinder) InReader(input io.Reader) map[int][]int {
  return core.AhoCorasickFromReader(sf.acd, input, 100000)
}
<|endoftext|>


<commit_before>package buffer

import(
	"github.com/Virepri/Shoraldele/GlobalVars"
	"fmt"
)

func Init(ini string) {
	fmt.Println("buffer.Init: STUB")
}

func Entry() {
	fmt.Println("buffer.Entry: STUB")
	GlobalVars.WaitGroup.Done()
}

func PushCommand(cmd, args string) {
	fmt.Println("buffer.PushCommand: STUB")
}
<commit_msg>Provide Buffer API v1<commit_after>package buffer

var work_buffer []byte

func Overwrite(offset int, data string) {
	if len(data) + len(work_buffer) > cap(work_buffer) {
		work_buffer = append(work_buffer, make([]byte, 1024)...)
	}

	copy(work_buffer[offset:], []byte(data))
}

func Insert(offset int, data string) {
	if len(data) + len(work_buffer) > cap(work_buffer) {
		work_buffer = append(work_buffer, make([]byte, 1024)...)
	}

	copy(work_buffer[offset+len(data):], work_buffer[offset:])
	overwrite(offset, data)
}

func Delete(offset int, length int) {
	work_buffer = append(work_buffer[:offset], work_buffer[offset+len(data):])
}

func GetBufferContents(offset int, length int) {
	if length < 1 {
		return work_buffer[offset:]
	}

	return work_buffer[offset:offset + length]
}
<|endoftext|>


<commit_before>// Copyright (C) 2017, 2018 Damon Revoe. All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
)

type fileProcessor func(sourcePathname, relativePathname string,
	info os.FileInfo) error

// processAllFiles calls the processFile() function for every file in
// sourceDir. All hidden files and all files in hidden subdirectories
// as well as package definition files are skipped.
func processAllFiles(sourceDir, targetDir string,
	processFile fileProcessor) error {

	sourceDir = filepath.Clean(sourceDir)
	sourceDirWithSlash := sourceDir + "/"

	return filepath.Walk(sourceDir, func(sourcePathname string,
		info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Ignore the top-level directory (sourceDir itself).
		if len(sourcePathname) <= len(sourceDirWithSlash) {
			return nil
		}

		// Panic if filepath.Walk() does not behave as expected.
		if !strings.HasPrefix(sourcePathname, sourceDirWithSlash) {
			panic(sourcePathname + " does not start with " +
				sourceDirWithSlash)
		}

		// Relative pathname of the source file in the source
		// directory (and the target file in the target directory).
		relativePathname := sourcePathname[len(sourceDirWithSlash):]

		// Ignore hidden files and the package definition file.
		if filepath.Base(relativePathname)[0] == '.' {
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		} else if info.IsDir() {
			return nil
		} else if relativePathname == packageDefinitionFilename {
			return nil
		}

		return processFile(sourcePathname, relativePathname, info)
	})
}

type directoryTree struct {
	entries map[string]*directoryTree
}

func newDirectoryTree() *directoryTree {
	return &directoryTree{make(map[string]*directoryTree)}
}

func (dirTree *directoryTree) addFile(filePath string) {
	pathComponents := strings.Split(filePath, "/")

	nComp := len(pathComponents)
	if nComp == 0 {
		return
	}

	node := dirTree

	for _, pathComponent := range pathComponents[:nComp-1] {
		child := node.entries[pathComponent]
		if child == nil {
			child = newDirectoryTree()
			node.entries[pathComponent] = child
		}
		node = child
	}

	node.entries[pathComponents[nComp-1]] = nil
}

func (dirTree *directoryTree) hasFile(filePath string) bool {
	pathComponents := strings.Split(filePath, "/")

	nComp := len(pathComponents)
	if nComp == 0 {
		return false
	}

	node := dirTree

	for _, pathComponent := range pathComponents[:nComp-1] {
		child := node.entries[pathComponent]
		if child == nil {
			return false
		}
		node = child
	}

	entry, entryExists := node.entries[pathComponents[nComp-1]]
	return entryExists && entry == nil
}

func (dirTree *directoryTree) subtree(filePath string) *directoryTree {
	node := dirTree

	for _, pathComponent := range strings.Split(filePath, "/") {
		if pathComponent == "." {
			continue
		}
		child := node.entries[pathComponent]
		if child == nil {
			return nil
		}
		node = child
	}

	return node
}

func listFiles(basePath string, dirTree *directoryTree) []string {
	var list []string

	basePath += "/"

	for entry, child := range dirTree.entries {
		if child == nil {
			list = append(list, basePath+entry)
		} else {
			list = append(list, listFiles(basePath+entry, child)...)
		}
	}

	return list
}

func (dirTree *directoryTree) list() []string {
	var list []string

	for entry, child := range dirTree.entries {
		if child == nil {
			list = append(list, entry)
		} else {
			list = append(list, listFiles(entry, child)...)
		}
	}

	sort.Strings(list)

	return list
}

func linkFilesFromSourceDir(pd *packageDefinition,
	projectDir string) (*directoryTree, bool, error) {
	dirTree := newDirectoryTree()
	sourceDir := filepath.Dir(pd.pathname)
	changesMade := false

	linkFile := func(sourcePathname, relativePathname string,
		sourceFileInfo os.FileInfo) error {
		dirTree.addFile(relativePathname)
		targetPathname := path.Join(projectDir, relativePathname)
		targetFileInfo, err := os.Lstat(targetPathname)
		if err == nil {
			if (targetFileInfo.Mode() & os.ModeSymlink) != 0 {
				originalLink, err := os.Readlink(targetPathname)

				if err != nil {
					return err
				}

				if originalLink == sourcePathname {
					return nil
				}
			}

			if err = os.Remove(targetPathname); err != nil {
				return err
			}
		}

		fmt.Println("L", targetPathname)

		if err = os.MkdirAll(filepath.Dir(targetPathname),
			os.ModePerm); err != nil {
			return err
		}

		changesMade = true

		return os.Symlink(sourcePathname, targetPathname)
	}

	err := processAllFiles(sourceDir, projectDir, linkFile)

	return dirTree, changesMade, err
}

func pathnamesNotInDir(pathnameTemplate string, params templateParams,
	dirTree *directoryTree) []outputFileParams {
	var fileParams []outputFileParams
	for _, fp := range expandPathnameTemplate(pathnameTemplate, params) {
		if !dirTree.hasFile(fp.filename) {
			fileParams = append(fileParams, fp)
		}
	}
	return fileParams
}

// generateBuildFilesFromProjectTemplate generates an output file inside
// 'projectDir' with the same relative pathname as the respective source
// file in 'templateDir'.
func generateBuildFilesFromProjectTemplate(templateDir,
	projectDir string, pd *packageDefinition) (bool, error) {

	dirTree, changesMade, err := linkFilesFromSourceDir(pd, projectDir)
	if err != nil {
		return false, err
	}

	generateFile := func(sourcePathname, relativePathname string,
		sourceFileInfo os.FileInfo) error {
		fileParams := pathnamesNotInDir(relativePathname,
			pd.params, dirTree)

		if len(fileParams) == 0 {
			return nil
		}

		// Read the contents of the template file. Cannot use
		// template.ParseFiles() because a Funcs() call must be
		// made between New() and Parse().
		templateContents, err := ioutil.ReadFile(sourcePathname)
		if err != nil {
			return err
		}

		filesUpdated, err := generateFilesFromProjectFileTemplate(
			projectDir, relativePathname, templateContents,
			sourceFileInfo.Mode(), pd, dirTree, fileParams)
		if err != nil {
			return err
		}
		if filesUpdated {
			changesMade = true
		}
		return nil
	}

	err = processAllFiles(templateDir, projectDir, generateFile)

	return changesMade, err
}

// embeddedTemplateFile defines the file mode and the contents
// of a single file that is a part of an embedded project template.
type embeddedTemplateFile struct {
	pathname string
	mode     os.FileMode
	contents []byte
}

// generateBuildFilesFromEmbeddedTemplate generates project build
// files from a built-in template pointed to by the 't' parameter.
func generateBuildFilesFromEmbeddedTemplate(t []embeddedTemplateFile,
	projectDir string, pd *packageDefinition) (bool, error) {

	dirTree, changesMade, err := linkFilesFromSourceDir(pd, projectDir)
	if err != nil {
		return false, err
	}

	for _, fileInfo := range append(t, commonTemplateFiles...) {
		fileParams := pathnamesNotInDir(fileInfo.pathname,
			pd.params, dirTree)

		if len(fileParams) == 0 {
			continue
		}

		filesUpdated, err := generateFilesFromProjectFileTemplate(
			projectDir, fileInfo.pathname, fileInfo.contents,
			fileInfo.mode, pd, dirTree, fileParams)
		if err != nil {
			return false, err
		}
		if filesUpdated {
			changesMade = true
		}
	}

	return changesMade, nil
}

func (pd *packageDefinition) getPackageGeneratorFunc(
	packageDir string) (func() (bool, error), error) {
	switch pd.packageType {
	case "app", "application":
		return func() (bool, error) {
			return generateBuildFilesFromEmbeddedTemplate(
				appTemplate, packageDir, pd)
		}, nil

	case "lib", "library":
		return func() (bool, error) {
			return generateBuildFilesFromEmbeddedTemplate(
				libTemplate, packageDir, pd)
		}, nil

	default:
		return nil, errors.New(pd.PackageName +
			": unknown package type '" + pd.packageType + "'")
	}
}
<commit_msg>Document directoryTree and some of its methods<commit_after>// Copyright (C) 2017, 2018 Damon Revoe. All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
)

type fileProcessor func(sourcePathname, relativePathname string,
	info os.FileInfo) error

// processAllFiles calls the processFile() function for every file in
// sourceDir. All hidden files and all files in hidden subdirectories
// as well as package definition files are skipped.
func processAllFiles(sourceDir, targetDir string,
	processFile fileProcessor) error {

	sourceDir = filepath.Clean(sourceDir)
	sourceDirWithSlash := sourceDir + "/"

	return filepath.Walk(sourceDir, func(sourcePathname string,
		info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Ignore the top-level directory (sourceDir itself).
		if len(sourcePathname) <= len(sourceDirWithSlash) {
			return nil
		}

		// Panic if filepath.Walk() does not behave as expected.
		if !strings.HasPrefix(sourcePathname, sourceDirWithSlash) {
			panic(sourcePathname + " does not start with " +
				sourceDirWithSlash)
		}

		// Relative pathname of the source file in the source
		// directory (and the target file in the target directory).
		relativePathname := sourcePathname[len(sourceDirWithSlash):]

		// Ignore hidden files and the package definition file.
		if filepath.Base(relativePathname)[0] == '.' {
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		} else if info.IsDir() {
			return nil
		} else if relativePathname == packageDefinitionFilename {
			return nil
		}

		return processFile(sourcePathname, relativePathname, info)
	})
}

// directoryTree represents a directory structure.
// The 'entries' map contains directory entries.
// If an entry name resolves into nil, it's a file,
// otherwise, it's a subtree.
type directoryTree struct {
	entries map[string]*directoryTree
}

// newDirectoryTree creates a new directory tree
// consisting of an empty root directory.
func newDirectoryTree() *directoryTree {
	return &directoryTree{make(map[string]*directoryTree)}
}

// addFile adds the specified pathname to the directory tree.
func (dirTree *directoryTree) addFile(filePath string) {
	pathComponents := strings.Split(filePath, "/")

	nComp := len(pathComponents)
	if nComp == 0 {
		return
	}

	node := dirTree

	for _, pathComponent := range pathComponents[:nComp-1] {
		child := node.entries[pathComponent]
		if child == nil {
			child = newDirectoryTree()
			node.entries[pathComponent] = child
		}
		node = child
	}

	node.entries[pathComponents[nComp-1]] = nil
}

// hasFile() can be used to check for whether the specified
// file is in the directory tree.
func (dirTree *directoryTree) hasFile(filePath string) bool {
	pathComponents := strings.Split(filePath, "/")

	nComp := len(pathComponents)
	if nComp == 0 {
		return false
	}

	node := dirTree

	for _, pathComponent := range pathComponents[:nComp-1] {
		child := node.entries[pathComponent]
		if child == nil {
			return false
		}
		node = child
	}

	entry, entryExists := node.entries[pathComponents[nComp-1]]
	return entryExists && entry == nil
}

// subtree returns a pointer to the branch of the directory tree
// rooted at the specified pathname.
func (dirTree *directoryTree) subtree(pathname string) *directoryTree {
	node := dirTree

	for _, pathComponent := range strings.Split(pathname, "/") {
		if pathComponent == "." {
			continue
		}
		child := node.entries[pathComponent]
		if child == nil {
			return nil
		}
		node = child
	}

	return node
}

func listFiles(basePath string, dirTree *directoryTree) []string {
	var list []string

	basePath += "/"

	for entry, child := range dirTree.entries {
		if child == nil {
			list = append(list, basePath+entry)
		} else {
			list = append(list, listFiles(basePath+entry, child)...)
		}
	}

	return list
}

func (dirTree *directoryTree) list() []string {
	var list []string

	for entry, child := range dirTree.entries {
		if child == nil {
			list = append(list, entry)
		} else {
			list = append(list, listFiles(entry, child)...)
		}
	}

	sort.Strings(list)

	return list
}

func linkFilesFromSourceDir(pd *packageDefinition,
	projectDir string) (*directoryTree, bool, error) {
	dirTree := newDirectoryTree()
	sourceDir := filepath.Dir(pd.pathname)
	changesMade := false

	linkFile := func(sourcePathname, relativePathname string,
		sourceFileInfo os.FileInfo) error {
		dirTree.addFile(relativePathname)
		targetPathname := path.Join(projectDir, relativePathname)
		targetFileInfo, err := os.Lstat(targetPathname)
		if err == nil {
			if (targetFileInfo.Mode() & os.ModeSymlink) != 0 {
				originalLink, err := os.Readlink(targetPathname)

				if err != nil {
					return err
				}

				if originalLink == sourcePathname {
					return nil
				}
			}

			if err = os.Remove(targetPathname); err != nil {
				return err
			}
		}

		fmt.Println("L", targetPathname)

		if err = os.MkdirAll(filepath.Dir(targetPathname),
			os.ModePerm); err != nil {
			return err
		}

		changesMade = true

		return os.Symlink(sourcePathname, targetPathname)
	}

	err := processAllFiles(sourceDir, projectDir, linkFile)

	return dirTree, changesMade, err
}

func pathnamesNotInDir(pathnameTemplate string, params templateParams,
	dirTree *directoryTree) []outputFileParams {
	var fileParams []outputFileParams
	for _, fp := range expandPathnameTemplate(pathnameTemplate, params) {
		if !dirTree.hasFile(fp.filename) {
			fileParams = append(fileParams, fp)
		}
	}
	return fileParams
}

// generateBuildFilesFromProjectTemplate generates an output file inside
// 'projectDir' with the same relative pathname as the respective source
// file in 'templateDir'.
func generateBuildFilesFromProjectTemplate(templateDir,
	projectDir string, pd *packageDefinition) (bool, error) {

	dirTree, changesMade, err := linkFilesFromSourceDir(pd, projectDir)
	if err != nil {
		return false, err
	}

	generateFile := func(sourcePathname, relativePathname string,
		sourceFileInfo os.FileInfo) error {
		fileParams := pathnamesNotInDir(relativePathname,
			pd.params, dirTree)

		if len(fileParams) == 0 {
			return nil
		}

		// Read the contents of the template file. Cannot use
		// template.ParseFiles() because a Funcs() call must be
		// made between New() and Parse().
		templateContents, err := ioutil.ReadFile(sourcePathname)
		if err != nil {
			return err
		}

		filesUpdated, err := generateFilesFromProjectFileTemplate(
			projectDir, relativePathname, templateContents,
			sourceFileInfo.Mode(), pd, dirTree, fileParams)
		if err != nil {
			return err
		}
		if filesUpdated {
			changesMade = true
		}
		return nil
	}

	err = processAllFiles(templateDir, projectDir, generateFile)

	return changesMade, err
}

// embeddedTemplateFile defines the file mode and the contents
// of a single file that is a part of an embedded project template.
type embeddedTemplateFile struct {
	pathname string
	mode     os.FileMode
	contents []byte
}

// generateBuildFilesFromEmbeddedTemplate generates project build
// files from a built-in template pointed to by the 't' parameter.
func generateBuildFilesFromEmbeddedTemplate(t []embeddedTemplateFile,
	projectDir string, pd *packageDefinition) (bool, error) {

	dirTree, changesMade, err := linkFilesFromSourceDir(pd, projectDir)
	if err != nil {
		return false, err
	}

	for _, fileInfo := range append(t, commonTemplateFiles...) {
		fileParams := pathnamesNotInDir(fileInfo.pathname,
			pd.params, dirTree)

		if len(fileParams) == 0 {
			continue
		}

		filesUpdated, err := generateFilesFromProjectFileTemplate(
			projectDir, fileInfo.pathname, fileInfo.contents,
			fileInfo.mode, pd, dirTree, fileParams)
		if err != nil {
			return false, err
		}
		if filesUpdated {
			changesMade = true
		}
	}

	return changesMade, nil
}

func (pd *packageDefinition) getPackageGeneratorFunc(
	packageDir string) (func() (bool, error), error) {
	switch pd.packageType {
	case "app", "application":
		return func() (bool, error) {
			return generateBuildFilesFromEmbeddedTemplate(
				appTemplate, packageDir, pd)
		}, nil

	case "lib", "library":
		return func() (bool, error) {
			return generateBuildFilesFromEmbeddedTemplate(
				libTemplate, packageDir, pd)
		}, nil

	default:
		return nil, errors.New(pd.PackageName +
			": unknown package type '" + pd.packageType + "'")
	}
}
<|endoftext|>