<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
)

func main() {
	if len(os.Args) < 7 {
		fmt.Println("The snelSLiM prepraser requires 6 arguments:")
		fmt.Println("1. the zip or tar file containing the corpus")
		fmt.Println("2. the folder the corpus should be extracted to")
		fmt.Println("3. the format of the corpus (autodetect, conll, folia, dcoi, plain, alpino, bnc, eindhoven, gysseling, graf, textgrid, xpath)")
		fmt.Println("4. option for the relevant format (usually lemma or text), enter - if not using")
		fmt.Println("5. extra option for very specific formats (e.g. fast or xpath for folia), enter - if not using")
		fmt.Println("6. the directory to write the preparsed results to")
		fmt.Println("7. whether to write plaintext wordlists, 1 for yes, 0 for no, optional (0 is then presumed)")
		os.Exit(1)
	}
	filename := os.Args[1]
	outdir := os.Args[2] + "/"
	format := os.Args[3]
	option := os.Args[4]
	extra := os.Args[5]
	savedir := os.Args[6] + "/"
	plainwords := false
	if len(os.Args) > 7 {
		plainwordsarg, err := strconv.Atoi(os.Args[7])
		if err != nil {
			err = ioutil.WriteFile(savedir+"error", []byte("error: Could not cast plainwordsarg to integer"), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		}
		if plainwordsarg == 1 {
			plainwords = true
		}
	}

	if strings.HasSuffix(filename, "zip") {
		err := exec.Command("/usr/bin/unzip", filename, "-d", outdir).Run()
		if err != nil {
			err = ioutil.WriteFile(savedir+"error", []byte("error: "+err.Error()), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		}
	} else {
		err := exec.Command("/bin/tar", "-xf", filename, "-C", outdir).Run()
		if err != nil {
			err = ioutil.WriteFile(savedir+"error", []byte("error: "+err.Error()), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		}
	}

	files := []string{}
	filepath.Walk(outdir, func(path string, info os.FileInfo, err error) error {
		// skip any hidden folder starting with a . (such as .git, .DS_Store and .Trashes)
		if info.IsDir() && strings.HasPrefix(info.Name(), ".") {
			return filepath.SkipDir
		}
		// skip hidden windows trash folder
		if info.IsDir() && info.Name() == "$RECYCLE.BIN" {
			return filepath.SkipDir
		}
		// ignore file: windows thumbnails, windows ini settings, linux/mac dotfiles and windows link files
		if info.Name() == "Thumbs.db" || info.Name() == "Thumbs.db:encryptable" || info.Name() == "ehthumbs.db" || info.Name() == "ehthumbs_vista.db" ||
			info.Name() == "desktop.ini" || info.Name() == "Desktop.ini" || strings.HasPrefix(info.Name(), ".") || strings.HasSuffix(info.Name(), ".lnk") {
			return nil
		}

		// any file (not a folder) left over is added to the file list with absolute filepaths
		if !info.IsDir() {
			abs, err := filepath.Abs(path)
			if err != nil {
				fmt.Println("Could not convert relative to absolute path")
				panic(err)
			}
			files = append(files, abs)
		}
		return nil
	})

	execlocation, err := os.Executable()
	if err != nil {
		err = ioutil.WriteFile(savedir+"error", []byte("Error: Could lookup preparser executable location to resolve format parser locations"), 0644)
		if err != nil {
			fmt.Println("Could not write error")
			panic(err)
		}
		panic(err)
	}
	binfolder := filepath.Dir(execlocation)

	if format == "autodetect" {
		autodetect, err := exec.Command(binfolder+"/formats/autodetect", outdir).Output()
		if err != nil {
			err = ioutil.WriteFile(savedir+"error", []byte("error: "+err.Error()), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		}
		format = string(autodetect)
		err = ioutil.WriteFile(savedir+"autodetect", autodetect, 0644)
		if err != nil {
			fmt.Println("Could not write done signal")
			panic(err)
		}
		if format == "unknown" {
			err := ioutil.WriteFile(savedir+"error", []byte("error: corpus format autodetection was unable to detect the format of your corpus. Please refer to help page for more information."), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		} else if format == "partknown" {
			err := ioutil.WriteFile(savedir+"error", []byte("error: corpus format autodetection detected some known and some unknown formats. You may have to clean up your corpus files."), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		} else if format == "mixed" {
			err := ioutil.WriteFile(savedir+"error", []byte("error: corpus format autodetection detected files in several corpus formats. You may have to clean up your corpus files."), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		} else if format == "xml-opus" {
			err := ioutil.WriteFile(savedir+"error", []byte("error: corpus format autodetection detected files in an XML format that may or may not be NLPL OPUS. Please refer to help page for more information."), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		} else if format == "xml" {
			err := ioutil.WriteFile(savedir+"error", []byte("error: corpus format autodetection detected files in an unknown XML format. Please refer to help page for more information."), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		} else if format == "tabs" {
			err := ioutil.WriteFile(savedir+"error", []byte("error: corpus format autodetection detected files that might be tab seperated (CoNLL) but does not know which column to use."), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		} else if format != "graf" && format != "alpino" && format != "bnc" && format != "dcoi" &&
			format != "folia" && format != "textgrid" && format != "gysseling" && format != "eindhoven" {

			err := ioutil.WriteFile(savedir+"error", []byte("error: corpus format autodetection did not return a valid response"), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		} else {
			option = "lemma"
			extra = "fast"
		}
	}

	corpussize := 0
	for _, file := range files {
		var output []byte
		var err error
		base := filepath.Base(file)
		plainwordsfile := "-"
		if plainwords {
			plainwordsfile = savedir + "/" + base + ".plainwords"
		}
		if format == "conll" {
			output, err = exec.Command(binfolder+"/formats/CoNLL/parser", file, extra, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "folia" {
			output, err = exec.Command(binfolder+"/formats/FoLiA/parser", file, option, extra, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "dcoi" {
			output, err = exec.Command(binfolder+"/formats/DCOI/parser", file, option, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "plain" {
			output, err = exec.Command(binfolder+"/formats/plain/parser", file, option, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "alpino" {
			output, err = exec.Command(binfolder+"/formats/alpino/parser", file, option, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "bnc" {
			output, err = exec.Command(binfolder+"/formats/BNC/parser", file, option, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "eindhoven" {
			output, err = exec.Command(binfolder+"/formats/eindhoven/parser", file, savedir+"/", plainwordsfile).Output()
		} else if format == "gysseling" {
			output, err = exec.Command(binfolder+"/formats/gysseling/parser", file, option, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "graf" {
			output, err = exec.Command(binfolder+"/formats/XCES-GrAF/parser", file, option, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "textgrid" {
			output, err = exec.Command(binfolder+"/formats/TextGrid/parser", file, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "opus" {
			output, err = exec.Command(binfolder+"/formats/OPUS/parser", file, option, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "xpath" {
			output, err = exec.Command(binfolder+"/formats/xpath/parser", file, extra, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else {
			err := ioutil.WriteFile(savedir+"error", []byte("error: unknown format"), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		}
		if err != nil {
			err = ioutil.WriteFile(savedir+"error", []byte("error: execution of format parser "+format+" failed: "+err.Error()), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		}
		outputsplit := strings.Split(string(output), "\n")
		filesize, err := strconv.Atoi(outputsplit[0])
		if err != nil {
			err = ioutil.WriteFile(savedir+"error", []byte("error: Could not cast corpussize to integer"), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		}
		corpussize += filesize
		status := outputsplit[1]
		if err != nil || status != "OK" {
			err := ioutil.WriteFile(savedir+"error", []byte("error executing parser: "+string(output)), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		}
	}

	err = ioutil.WriteFile(savedir+"corpussize", []byte(strconv.Itoa(corpussize)), 0644)
	if err != nil {
		fmt.Println("Could not write the corpus size")
		panic(err)
	}

	if plainwords {
		err := ioutil.WriteFile(savedir+"plainwords", []byte("active"), 0644)
		if err != nil {
			fmt.Println("Could not write plainwords signal")
			panic(err)
		}
	}

	err = ioutil.WriteFile(savedir+"done", []byte("done"), 0644)
	if err != nil {
		fmt.Println("Could not write done signal")
		panic(err)
	}
	err = os.RemoveAll(outdir)
	err = os.RemoveAll(filename)
}
<commit_msg>Improve autodetection errors pass to the UI in the preparser<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
)

func main() {
	if len(os.Args) < 7 {
		fmt.Println("The snelSLiM prepraser requires 6 arguments:")
		fmt.Println("1. the zip or tar file containing the corpus")
		fmt.Println("2. the folder the corpus should be extracted to")
		fmt.Println("3. the format of the corpus (autodetect, conll, folia, dcoi, plain, alpino, bnc, eindhoven, gysseling, graf, textgrid, xpath)")
		fmt.Println("4. option for the relevant format (usually lemma or text), enter - if not using")
		fmt.Println("5. extra option for very specific formats (e.g. fast or xpath for folia), enter - if not using")
		fmt.Println("6. the directory to write the preparsed results to")
		fmt.Println("7. whether to write plaintext wordlists, 1 for yes, 0 for no, optional (0 is then presumed)")
		os.Exit(1)
	}
	filename := os.Args[1]
	outdir := os.Args[2] + "/"
	format := os.Args[3]
	option := os.Args[4]
	extra := os.Args[5]
	savedir := os.Args[6] + "/"
	plainwords := false
	if len(os.Args) > 7 {
		plainwordsarg, err := strconv.Atoi(os.Args[7])
		if err != nil {
			err = ioutil.WriteFile(savedir+"error", []byte("error: Could not cast plainwordsarg to integer"), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		}
		if plainwordsarg == 1 {
			plainwords = true
		}
	}

	if strings.HasSuffix(filename, "zip") {
		err := exec.Command("/usr/bin/unzip", filename, "-d", outdir).Run()
		if err != nil {
			err = ioutil.WriteFile(savedir+"error", []byte("error: "+err.Error()), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		}
	} else {
		err := exec.Command("/bin/tar", "-xf", filename, "-C", outdir).Run()
		if err != nil {
			err = ioutil.WriteFile(savedir+"error", []byte("error: "+err.Error()), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		}
	}

	files := []string{}
	filepath.Walk(outdir, func(path string, info os.FileInfo, err error) error {
		// skip any hidden folder starting with a . (such as .git, .DS_Store and .Trashes)
		if info.IsDir() && strings.HasPrefix(info.Name(), ".") {
			return filepath.SkipDir
		}
		// skip hidden windows trash folder
		if info.IsDir() && info.Name() == "$RECYCLE.BIN" {
			return filepath.SkipDir
		}
		// ignore file: windows thumbnails, windows ini settings, linux/mac dotfiles and windows link files
		if info.Name() == "Thumbs.db" || info.Name() == "Thumbs.db:encryptable" || info.Name() == "ehthumbs.db" || info.Name() == "ehthumbs_vista.db" ||
			info.Name() == "desktop.ini" || info.Name() == "Desktop.ini" || strings.HasPrefix(info.Name(), ".") || strings.HasSuffix(info.Name(), ".lnk") {
			return nil
		}

		// any file (not a folder) left over is added to the file list with absolute filepaths
		if !info.IsDir() {
			abs, err := filepath.Abs(path)
			if err != nil {
				fmt.Println("Could not convert relative to absolute path")
				panic(err)
			}
			files = append(files, abs)
		}
		return nil
	})

	execlocation, err := os.Executable()
	if err != nil {
		err = ioutil.WriteFile(savedir+"error", []byte("Error: Could lookup preparser executable location to resolve format parser locations"), 0644)
		if err != nil {
			fmt.Println("Could not write error")
			panic(err)
		}
		panic(err)
	}
	binfolder := filepath.Dir(execlocation)

	if format == "autodetect" {
		autodetect, err := exec.Command(binfolder+"/formats/autodetect", outdir).Output()
		if err != nil {
			err = ioutil.WriteFile(savedir+"error", []byte("error: "+err.Error()), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		}
		format = string(autodetect)
		err = ioutil.WriteFile(savedir+"autodetect", autodetect, 0644)
		if err != nil {
			fmt.Println("Could not write done signal")
			panic(err)
		}
		if format == "unknown" {
			err := ioutil.WriteFile(savedir+"error", []byte("error: corpus format autodetection was unable to detect the format of your corpus. Please refer to the user manual and the corpus formats help page for more information."), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		} else if format == "partknown" {
			err := ioutil.WriteFile(savedir+"error", []byte("error: corpus format autodetection detected some known and some unknown formats. You may have to clean up your corpus files."), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		} else if format == "mixed" {
			err := ioutil.WriteFile(savedir+"error", []byte("error: corpus format autodetection detected files in several corpus formats. You may have to clean up your corpus files."), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		} else if format == "xml-opus" {
			err := ioutil.WriteFile(savedir+"error", []byte("error: corpus format autodetection detected files in an XML format that may or may not be NLPL OPUS. Please refer to the user manual and the corpus formats help page for more information."), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		} else if format == "xml" {
			err := ioutil.WriteFile(savedir+"error", []byte("error: corpus format autodetection detected files in an unknown XML format. Please refer to the user manual and the corpus formats help page for more information."), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		} else if format == "tabs" {
			err := ioutil.WriteFile(savedir+"error", []byte("error: corpus format autodetection detected files that might be tab seperated (CoNLL) but does not know which column to use."), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		} else if format != "graf" && format != "alpino" && format != "bnc" && format != "dcoi" &&
			format != "folia" && format != "textgrid" && format != "gysseling" && format != "eindhoven" {

			err := ioutil.WriteFile(savedir+"error", []byte("error: corpus format autodetection did not return a valid response"), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		} else {
			option = "lemma"
			extra = "fast"
		}
	}

	corpussize := 0
	for _, file := range files {
		var output []byte
		var err error
		base := filepath.Base(file)
		plainwordsfile := "-"
		if plainwords {
			plainwordsfile = savedir + "/" + base + ".plainwords"
		}
		if format == "conll" {
			output, err = exec.Command(binfolder+"/formats/CoNLL/parser", file, extra, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "folia" {
			output, err = exec.Command(binfolder+"/formats/FoLiA/parser", file, option, extra, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "dcoi" {
			output, err = exec.Command(binfolder+"/formats/DCOI/parser", file, option, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "plain" {
			output, err = exec.Command(binfolder+"/formats/plain/parser", file, option, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "alpino" {
			output, err = exec.Command(binfolder+"/formats/alpino/parser", file, option, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "bnc" {
			output, err = exec.Command(binfolder+"/formats/BNC/parser", file, option, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "eindhoven" {
			output, err = exec.Command(binfolder+"/formats/eindhoven/parser", file, savedir+"/", plainwordsfile).Output()
		} else if format == "gysseling" {
			output, err = exec.Command(binfolder+"/formats/gysseling/parser", file, option, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "graf" {
			output, err = exec.Command(binfolder+"/formats/XCES-GrAF/parser", file, option, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "textgrid" {
			output, err = exec.Command(binfolder+"/formats/TextGrid/parser", file, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "opus" {
			output, err = exec.Command(binfolder+"/formats/OPUS/parser", file, option, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else if format == "xpath" {
			output, err = exec.Command(binfolder+"/formats/xpath/parser", file, extra, savedir+"/"+base+".snelslim", plainwordsfile).Output()
		} else {
			err := ioutil.WriteFile(savedir+"error", []byte("error: unknown format"), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		}
		if err != nil {
			err = ioutil.WriteFile(savedir+"error", []byte("error: execution of format parser "+format+" failed: "+err.Error()), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		}
		outputsplit := strings.Split(string(output), "\n")
		filesize, err := strconv.Atoi(outputsplit[0])
		if err != nil {
			err = ioutil.WriteFile(savedir+"error", []byte("error: Could not cast corpussize to integer"), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		}
		corpussize += filesize
		status := outputsplit[1]
		if err != nil || status != "OK" {
			err := ioutil.WriteFile(savedir+"error", []byte("error executing parser: "+string(output)), 0644)
			if err != nil {
				fmt.Println("Could not write error")
				panic(err)
			}
			panic(err)
		}
	}

	err = ioutil.WriteFile(savedir+"corpussize", []byte(strconv.Itoa(corpussize)), 0644)
	if err != nil {
		fmt.Println("Could not write the corpus size")
		panic(err)
	}

	if plainwords {
		err := ioutil.WriteFile(savedir+"plainwords", []byte("active"), 0644)
		if err != nil {
			fmt.Println("Could not write plainwords signal")
			panic(err)
		}
	}

	err = ioutil.WriteFile(savedir+"done", []byte("done"), 0644)
	if err != nil {
		fmt.Println("Could not write done signal")
		panic(err)
	}
	err = os.RemoveAll(outdir)
	err = os.RemoveAll(filename)
}
<|endoftext|>


<commit_before>package main

import (
	"go/ast"
	"go/parser"
	"go/token"
	"bytes"
)

const fixlen = len("package p;")

func parseDeclList(fset *token.FileSet, data []byte) ([]ast.Decl, error) {
	var buf bytes.Buffer
	buf.WriteString("package p;")
	buf.Write(data)
	file, err := parser.ParseFile(fset, "", buf.Bytes(), 0)
	if err != nil {
		return file.Decls, err
	}
	return file.Decls, nil
}

//-------------------------------------------------------------------------
// AutoCompleteFile
//-------------------------------------------------------------------------

type AutoCompleteFile struct {
	name        string
	packageName string

	decls     map[string]*Decl
	packages  PackageImports
	filescope *Scope
	scope     *Scope

	cursor int // for current file buffer only
	fset   *token.FileSet
}

func NewAutoCompleteFile(name string) *AutoCompleteFile {
	p := new(AutoCompleteFile)
	p.name = name
	p.cursor = -1
	p.fset = token.NewFileSet()
	return p
}

// this one is used for current file buffer exclusively
func (f *AutoCompleteFile) processData(data []byte) {
	cur, filedata, block := RipOffDecl(data, f.cursor)
	file, _ := parser.ParseFile(f.fset, "", filedata, 0)
	f.packageName = packageName(file)

	f.decls = make(map[string]*Decl)
	f.packages = NewPackageImports(f.name, file.Decls)
	f.filescope = NewScope(nil)
	f.scope = f.filescope

	for _, d := range file.Decls {
		anonymifyAst(d, 0, f.filescope)
	}

	// process all top-level declarations
	for _, decl := range file.Decls {
		appendToTopDecls(f.decls, decl, f.scope)
	}
	if block != nil {
		// process local function as top-level declaration
		decls, _ := parseDeclList(f.fset, block)

		for _, d := range decls {
			anonymifyAst(d, 0, f.filescope)
		}

		for _, decl := range decls {
			appendToTopDecls(f.decls, decl, f.scope)
		}

		// process function internals
		f.cursor = cur
		for _, decl := range decls {
			f.processDeclLocals(decl)
		}
	}

}

func (f *AutoCompleteFile) processDeclLocals(decl ast.Decl) {
	switch t := decl.(type) {
	case *ast.FuncDecl:
		if f.cursorIn(t.Body) {
			s := f.scope
			f.scope = NewScope(f.scope)

			f.processFieldList(t.Recv, s)
			f.processFieldList(t.Type.Params, s)
			f.processFieldList(t.Type.Results, s)
			f.processBlockStmt(t.Body)

		}
	}
}

func (f *AutoCompleteFile) processDecl(decl ast.Decl) {
	if t, ok := decl.(*ast.GenDecl); ok && f.fset.Position(t.TokPos).Offset - fixlen > f.cursor {
		return
	}
	prevscope := f.scope
	foreachDecl(decl, func(data *foreachDeclStruct) {
		class := astDeclClass(data.decl)
		if class != DECL_TYPE {
			f.scope, prevscope = AdvanceScope(f.scope)
		}
		for i, name := range data.names {
			typ, v, vi := data.typeValueIndex(i, 0)

			d := NewDecl2(name.Name, class, 0, typ, v, vi, prevscope)
			if d == nil {
				return
			}

			f.scope.addNamedDecl(d)
		}
	})
}

func (f *AutoCompleteFile) processBlockStmt(block *ast.BlockStmt) {
	if block != nil && f.cursorIn(block) {
		f.scope, _ = AdvanceScope(f.scope)

		for _, stmt := range block.List {
			f.processStmt(stmt)
		}

		// hack to process all func literals
		v := new(funcLitVisitor)
		v.ctx = f
		ast.Walk(v, block)
	}
}

type funcLitVisitor struct {
	ctx *AutoCompleteFile
}

func (v *funcLitVisitor) Visit(node ast.Node) ast.Visitor {
	if t, ok := node.(*ast.FuncLit); ok && v.ctx.cursorIn(t.Body) {
		s := v.ctx.scope
		v.ctx.scope, _ = AdvanceScope(v.ctx.scope)

		v.ctx.processFieldList(t.Type.Params, s)
		v.ctx.processFieldList(t.Type.Results, s)
		v.ctx.processBlockStmt(t.Body)

		return nil
	}
	return v
}

func (f *AutoCompleteFile) processStmt(stmt ast.Stmt) {
	switch t := stmt.(type) {
	case *ast.DeclStmt:
		f.processDecl(t.Decl)
	case *ast.AssignStmt:
		f.processAssignStmt(t)
	case *ast.IfStmt:
		if f.cursorIn(t.Body) {
			f.scope, _ = AdvanceScope(f.scope)

			f.processStmt(t.Init)
			f.processBlockStmt(t.Body)
		}
		f.processStmt(t.Else)
	case *ast.BlockStmt:
		f.processBlockStmt(t)
	case *ast.RangeStmt:
		f.processRangeStmt(t)
	case *ast.ForStmt:
		if f.cursorIn(t.Body) {
			f.scope, _ = AdvanceScope(f.scope)

			f.processStmt(t.Init)
			f.processBlockStmt(t.Body)
		}
	case *ast.SwitchStmt:
		f.processSwitchStmt(t)
	case *ast.TypeSwitchStmt:
		f.processTypeSwitchStmt(t)
	case *ast.SelectStmt:
		f.processSelectStmt(t)
	case *ast.LabeledStmt:
		f.processStmt(t.Stmt)
	}
}

func (f *AutoCompleteFile) processSelectStmt(a *ast.SelectStmt) {
	if !f.cursorIn(a.Body) {
		return
	}
	var prevscope *Scope
	f.scope, prevscope = AdvanceScope(f.scope)

	var lastCursorAfter *ast.CommClause
	for _, s := range a.Body.List {
		if cc := s.(*ast.CommClause); f.cursor > f.fset.Position(cc.Colon).Offset - fixlen {
			lastCursorAfter = cc
		}
	}

	if lastCursorAfter != nil {
		if lastCursorAfter.Comm != nil {
			//if lastCursorAfter.Lhs != nil && lastCursorAfter.Tok == token.DEFINE {
			if astmt, ok := lastCursorAfter.Comm.(*ast.AssignStmt); ok && astmt.Tok == token.DEFINE {
				vname := astmt.Lhs[0].(*ast.Ident).Name
				v := NewDeclVar(vname, nil, astmt.Rhs[0], -1, prevscope)
				f.scope.addNamedDecl(v)
			}
		}
		for _, s := range lastCursorAfter.Body {
			f.processStmt(s)
		}
	}
}

func (f *AutoCompleteFile) processTypeSwitchStmt(a *ast.TypeSwitchStmt) {
	if !f.cursorIn(a.Body) {
		return
	}
	var prevscope *Scope
	f.scope, prevscope = AdvanceScope(f.scope)

	f.processStmt(a.Init)
	// type var
	var tv *Decl
	if a, ok := a.Assign.(*ast.AssignStmt); ok {
		lhs := a.Lhs
		rhs := a.Rhs
		if lhs != nil && len(lhs) == 1 {
			tvname := lhs[0].(*ast.Ident).Name
			tv = NewDeclVar(tvname, nil, rhs[0], -1, prevscope)
		}
	}

	var lastCursorAfter *ast.CaseClause
	for _, s := range a.Body.List {
		if cc := s.(*ast.CaseClause); f.cursor > f.fset.Position(cc.Colon).Offset - fixlen {
			lastCursorAfter = cc
		}
	}

	if lastCursorAfter != nil {
		if tv != nil {
			if lastCursorAfter.List != nil && len(lastCursorAfter.List) == 1 {
				tv.Type = lastCursorAfter.List[0]
				tv.Value = nil
			}
			f.scope.addNamedDecl(tv)
		}
		for _, s := range lastCursorAfter.Body {
			f.processStmt(s)
		}
	}
}

func (f *AutoCompleteFile) processSwitchStmt(a *ast.SwitchStmt) {
	if !f.cursorIn(a.Body) {
		return
	}
	f.scope, _ = AdvanceScope(f.scope)

	f.processStmt(a.Init)
	var lastCursorAfter *ast.CaseClause
	for _, s := range a.Body.List {
		if cc := s.(*ast.CaseClause); f.cursor > f.fset.Position(cc.Colon).Offset - fixlen {
			lastCursorAfter = cc
		}
	}
	if lastCursorAfter != nil {
		for _, s := range lastCursorAfter.Body {
			f.processStmt(s)
		}
	}
}

func (f *AutoCompleteFile) processRangeStmt(a *ast.RangeStmt) {
	if !f.cursorIn(a.Body) {
		return
	}
	var prevscope *Scope
	f.scope, prevscope = AdvanceScope(f.scope)

	if a.Tok == token.DEFINE {
		if t, ok := a.Key.(*ast.Ident); ok {
			d := NewDeclVar(t.Name, nil, a.X, 0, prevscope)
			if d != nil {
				d.Flags |= DECL_RANGEVAR
				f.scope.addNamedDecl(d)
			}
		}

		if a.Value != nil {
			if t, ok := a.Value.(*ast.Ident); ok {
				d := NewDeclVar(t.Name, nil, a.X, 1, prevscope)
				if d != nil {
					d.Flags |= DECL_RANGEVAR
					f.scope.addNamedDecl(d)
				}
			}
		}
	}

	f.processBlockStmt(a.Body)
}

func (f *AutoCompleteFile) processAssignStmt(a *ast.AssignStmt) {
	if a.Tok != token.DEFINE || f.fset.Position(a.TokPos).Offset - fixlen > f.cursor {
		return
	}

	names := make([]*ast.Ident, len(a.Lhs))
	for i, name := range a.Lhs {
		id, ok := name.(*ast.Ident)
		if !ok {
			// something is wrong, just ignore the whole stmt
			return
		}
		names[i] = id
	}

	var prevscope *Scope
	f.scope, prevscope = AdvanceScope(f.scope)

	pack := declPack{names, nil, a.Rhs}
	for i, name := range pack.names {
		typ, v, vi := pack.typeValueIndex(i, 0)
		d := NewDeclVar(name.Name, typ, v, vi, prevscope)
		if d == nil {
			continue
		}

		f.scope.addNamedDecl(d)
	}
}

func (f *AutoCompleteFile) processFieldList(fieldList *ast.FieldList, s *Scope) {
	if fieldList != nil {
		decls := astFieldListToDecls(fieldList, DECL_VAR, 0, s)
		for _, d := range decls {
			f.scope.addNamedDecl(d)
		}
	}
}

func (f *AutoCompleteFile) cursorIn(block *ast.BlockStmt) bool {
	if f.cursor == -1 || block == nil {
		return false
	}

	loff := f.fset.Position(block.Lbrace).Offset
	roff := f.fset.Position(block.Rbrace).Offset

	if f.cursor >= loff - fixlen && f.cursor <= roff - fixlen {
		return true
	}
	return false
}
<commit_msg>Use a helper function for correcting offsets.<commit_after>package main

import (
	"go/ast"
	"go/parser"
	"go/token"
	"bytes"
)

func parseDeclList(fset *token.FileSet, data []byte) ([]ast.Decl, error) {
	var buf bytes.Buffer
	buf.WriteString("package p;")
	buf.Write(data)
	file, err := parser.ParseFile(fset, "", buf.Bytes(), 0)
	if err != nil {
		return file.Decls, err
	}
	return file.Decls, nil
}

//-------------------------------------------------------------------------
// AutoCompleteFile
//-------------------------------------------------------------------------

type AutoCompleteFile struct {
	name        string
	packageName string

	decls     map[string]*Decl
	packages  PackageImports
	filescope *Scope
	scope     *Scope

	cursor int // for current file buffer only
	fset   *token.FileSet
}

func NewAutoCompleteFile(name string) *AutoCompleteFile {
	p := new(AutoCompleteFile)
	p.name = name
	p.cursor = -1
	p.fset = token.NewFileSet()
	return p
}

func (f *AutoCompleteFile) offset(p token.Pos) int {
	const fixlen = len("package p;")
	return f.fset.Position(p).Offset - fixlen
}

// this one is used for current file buffer exclusively
func (f *AutoCompleteFile) processData(data []byte) {
	cur, filedata, block := RipOffDecl(data, f.cursor)
	file, _ := parser.ParseFile(f.fset, "", filedata, 0)
	f.packageName = packageName(file)

	f.decls = make(map[string]*Decl)
	f.packages = NewPackageImports(f.name, file.Decls)
	f.filescope = NewScope(nil)
	f.scope = f.filescope

	for _, d := range file.Decls {
		anonymifyAst(d, 0, f.filescope)
	}

	// process all top-level declarations
	for _, decl := range file.Decls {
		appendToTopDecls(f.decls, decl, f.scope)
	}
	if block != nil {
		// process local function as top-level declaration
		decls, _ := parseDeclList(f.fset, block)

		for _, d := range decls {
			anonymifyAst(d, 0, f.filescope)
		}

		for _, decl := range decls {
			appendToTopDecls(f.decls, decl, f.scope)
		}

		// process function internals
		f.cursor = cur
		for _, decl := range decls {
			f.processDeclLocals(decl)
		}
	}

}

func (f *AutoCompleteFile) processDeclLocals(decl ast.Decl) {
	switch t := decl.(type) {
	case *ast.FuncDecl:
		if f.cursorIn(t.Body) {
			s := f.scope
			f.scope = NewScope(f.scope)

			f.processFieldList(t.Recv, s)
			f.processFieldList(t.Type.Params, s)
			f.processFieldList(t.Type.Results, s)
			f.processBlockStmt(t.Body)

		}
	}
}

func (f *AutoCompleteFile) processDecl(decl ast.Decl) {
	if t, ok := decl.(*ast.GenDecl); ok && f.offset(t.TokPos) > f.cursor {
		return
	}
	prevscope := f.scope
	foreachDecl(decl, func(data *foreachDeclStruct) {
		class := astDeclClass(data.decl)
		if class != DECL_TYPE {
			f.scope, prevscope = AdvanceScope(f.scope)
		}
		for i, name := range data.names {
			typ, v, vi := data.typeValueIndex(i, 0)

			d := NewDecl2(name.Name, class, 0, typ, v, vi, prevscope)
			if d == nil {
				return
			}

			f.scope.addNamedDecl(d)
		}
	})
}

func (f *AutoCompleteFile) processBlockStmt(block *ast.BlockStmt) {
	if block != nil && f.cursorIn(block) {
		f.scope, _ = AdvanceScope(f.scope)

		for _, stmt := range block.List {
			f.processStmt(stmt)
		}

		// hack to process all func literals
		v := new(funcLitVisitor)
		v.ctx = f
		ast.Walk(v, block)
	}
}

type funcLitVisitor struct {
	ctx *AutoCompleteFile
}

func (v *funcLitVisitor) Visit(node ast.Node) ast.Visitor {
	if t, ok := node.(*ast.FuncLit); ok && v.ctx.cursorIn(t.Body) {
		s := v.ctx.scope
		v.ctx.scope, _ = AdvanceScope(v.ctx.scope)

		v.ctx.processFieldList(t.Type.Params, s)
		v.ctx.processFieldList(t.Type.Results, s)
		v.ctx.processBlockStmt(t.Body)

		return nil
	}
	return v
}

func (f *AutoCompleteFile) processStmt(stmt ast.Stmt) {
	switch t := stmt.(type) {
	case *ast.DeclStmt:
		f.processDecl(t.Decl)
	case *ast.AssignStmt:
		f.processAssignStmt(t)
	case *ast.IfStmt:
		if f.cursorIn(t.Body) {
			f.scope, _ = AdvanceScope(f.scope)

			f.processStmt(t.Init)
			f.processBlockStmt(t.Body)
		}
		f.processStmt(t.Else)
	case *ast.BlockStmt:
		f.processBlockStmt(t)
	case *ast.RangeStmt:
		f.processRangeStmt(t)
	case *ast.ForStmt:
		if f.cursorIn(t.Body) {
			f.scope, _ = AdvanceScope(f.scope)

			f.processStmt(t.Init)
			f.processBlockStmt(t.Body)
		}
	case *ast.SwitchStmt:
		f.processSwitchStmt(t)
	case *ast.TypeSwitchStmt:
		f.processTypeSwitchStmt(t)
	case *ast.SelectStmt:
		f.processSelectStmt(t)
	case *ast.LabeledStmt:
		f.processStmt(t.Stmt)
	}
}

func (f *AutoCompleteFile) processSelectStmt(a *ast.SelectStmt) {
	if !f.cursorIn(a.Body) {
		return
	}
	var prevscope *Scope
	f.scope, prevscope = AdvanceScope(f.scope)

	var lastCursorAfter *ast.CommClause
	for _, s := range a.Body.List {
		if cc := s.(*ast.CommClause); f.cursor > f.offset(cc.Colon) {
			lastCursorAfter = cc
		}
	}

	if lastCursorAfter != nil {
		if lastCursorAfter.Comm != nil {
			//if lastCursorAfter.Lhs != nil && lastCursorAfter.Tok == token.DEFINE {
			if astmt, ok := lastCursorAfter.Comm.(*ast.AssignStmt); ok && astmt.Tok == token.DEFINE {
				vname := astmt.Lhs[0].(*ast.Ident).Name
				v := NewDeclVar(vname, nil, astmt.Rhs[0], -1, prevscope)
				f.scope.addNamedDecl(v)
			}
		}
		for _, s := range lastCursorAfter.Body {
			f.processStmt(s)
		}
	}
}

func (f *AutoCompleteFile) processTypeSwitchStmt(a *ast.TypeSwitchStmt) {
	if !f.cursorIn(a.Body) {
		return
	}
	var prevscope *Scope
	f.scope, prevscope = AdvanceScope(f.scope)

	f.processStmt(a.Init)
	// type var
	var tv *Decl
	if a, ok := a.Assign.(*ast.AssignStmt); ok {
		lhs := a.Lhs
		rhs := a.Rhs
		if lhs != nil && len(lhs) == 1 {
			tvname := lhs[0].(*ast.Ident).Name
			tv = NewDeclVar(tvname, nil, rhs[0], -1, prevscope)
		}
	}

	var lastCursorAfter *ast.CaseClause
	for _, s := range a.Body.List {
		if cc := s.(*ast.CaseClause); f.cursor > f.offset(cc.Colon) {
			lastCursorAfter = cc
		}
	}

	if lastCursorAfter != nil {
		if tv != nil {
			if lastCursorAfter.List != nil && len(lastCursorAfter.List) == 1 {
				tv.Type = lastCursorAfter.List[0]
				tv.Value = nil
			}
			f.scope.addNamedDecl(tv)
		}
		for _, s := range lastCursorAfter.Body {
			f.processStmt(s)
		}
	}
}

func (f *AutoCompleteFile) processSwitchStmt(a *ast.SwitchStmt) {
	if !f.cursorIn(a.Body) {
		return
	}
	f.scope, _ = AdvanceScope(f.scope)

	f.processStmt(a.Init)
	var lastCursorAfter *ast.CaseClause
	for _, s := range a.Body.List {
		if cc := s.(*ast.CaseClause); f.cursor > f.offset(cc.Colon) {
			lastCursorAfter = cc
		}
	}
	if lastCursorAfter != nil {
		for _, s := range lastCursorAfter.Body {
			f.processStmt(s)
		}
	}
}

func (f *AutoCompleteFile) processRangeStmt(a *ast.RangeStmt) {
	if !f.cursorIn(a.Body) {
		return
	}
	var prevscope *Scope
	f.scope, prevscope = AdvanceScope(f.scope)

	if a.Tok == token.DEFINE {
		if t, ok := a.Key.(*ast.Ident); ok {
			d := NewDeclVar(t.Name, nil, a.X, 0, prevscope)
			if d != nil {
				d.Flags |= DECL_RANGEVAR
				f.scope.addNamedDecl(d)
			}
		}

		if a.Value != nil {
			if t, ok := a.Value.(*ast.Ident); ok {
				d := NewDeclVar(t.Name, nil, a.X, 1, prevscope)
				if d != nil {
					d.Flags |= DECL_RANGEVAR
					f.scope.addNamedDecl(d)
				}
			}
		}
	}

	f.processBlockStmt(a.Body)
}

func (f *AutoCompleteFile) processAssignStmt(a *ast.AssignStmt) {
	if a.Tok != token.DEFINE || f.offset(a.TokPos) > f.cursor {
		return
	}

	names := make([]*ast.Ident, len(a.Lhs))
	for i, name := range a.Lhs {
		id, ok := name.(*ast.Ident)
		if !ok {
			// something is wrong, just ignore the whole stmt
			return
		}
		names[i] = id
	}

	var prevscope *Scope
	f.scope, prevscope = AdvanceScope(f.scope)

	pack := declPack{names, nil, a.Rhs}
	for i, name := range pack.names {
		typ, v, vi := pack.typeValueIndex(i, 0)
		d := NewDeclVar(name.Name, typ, v, vi, prevscope)
		if d == nil {
			continue
		}

		f.scope.addNamedDecl(d)
	}
}

func (f *AutoCompleteFile) processFieldList(fieldList *ast.FieldList, s *Scope) {
	if fieldList != nil {
		decls := astFieldListToDecls(fieldList, DECL_VAR, 0, s)
		for _, d := range decls {
			f.scope.addNamedDecl(d)
		}
	}
}

func (f *AutoCompleteFile) cursorIn(block *ast.BlockStmt) bool {
	if f.cursor == -1 || block == nil {
		return false
	}

	if f.cursor >= f.offset(block.Lbrace) && f.cursor <= f.offset(block.Rbrace) {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package hipchat

import (
	"os"
	"regexp"
	"strings"
	"testing"
	"time"

	h "github.com/daneharrigan/hipchat"

	"github.com/keel-hq/keel/approvals"
	b "github.com/keel-hq/keel/bot"
	"github.com/keel-hq/keel/cache/memory"
	"github.com/keel-hq/keel/provider/kubernetes"
	"github.com/keel-hq/keel/types"
	"github.com/keel-hq/keel/util/codecs"

	testutil "github.com/keel-hq/keel/util/testing"

	log "github.com/Sirupsen/logrus"
)

type fakeProvider struct {
	submitted []types.Event
	images    []*types.TrackedImage
}

func (p *fakeProvider) Submit(event types.Event) error {
	p.submitted = append(p.submitted, event)
	return nil
}

func (p *fakeProvider) TrackedImages() ([]*types.TrackedImage, error) {
	return p.images, nil
}

func (p *fakeProvider) List() []string {
	return []string{"fakeprovider"}
}
func (p *fakeProvider) Stop() {
	return
}
func (p *fakeProvider) GetName() string {
	return "fp"
}

var botMessagesChannel chan *b.BotMessage
var approvalsRespCh chan *b.ApprovalResponse

type postedMessage struct {
	channel string
	text    string
}

type fakeXmppImplementer struct {
	postedMessages []postedMessage
	messages       chan *h.Message
}

func (i *fakeXmppImplementer) messageFromChat(message string) {
	i.messages <- &h.Message{
		Body: "@keel " + message,
		From: "111111_approvals@conf.hipchat.com/test",
		To:   "222222_333333@chat.hipchat.com/bot",
	}
}

func (i *fakeXmppImplementer) Say(roomID, name, body string) {
	i.postedMessages = append(i.postedMessages, postedMessage{
		text:    body,
		channel: roomID,
	})
}
func (i *fakeXmppImplementer) Status(s string) {
}
func (i *fakeXmppImplementer) Join(roomID, resource string) {
}
func (i *fakeXmppImplementer) KeepAlive() {
}
func (i *fakeXmppImplementer) Messages() <-chan *h.Message {
	return i.messages
}

func NewBot(k8sImplementer kubernetes.Implementer,
	approvalsManager approvals.Manager, fi XmppImplementer) *Bot {

	approvalsRespCh = make(chan *b.ApprovalResponse)
	botMessagesChannel = make(chan *b.BotMessage)
	fakeBot := &Bot{}
	fakeBot.hipchatClient = fi

	os.Setenv("HIPCHAT_APPROVALS_CHANNEL", "111111_approvals@conf.hipchat.com")
	os.Setenv("HIPCHAT_APPROVALS_BOT_NAME", "keel")
	os.Setenv("HIPCHAT_APPROVALS_USER_NAME", "111111_222222")
	os.Setenv("HIPCHAT_APPROVALS_PASSWORT", "pass")
	os.Setenv("HIPCHAT_CONNECTION_ATTEMPTS", "0")

	b.RegisterBot("fakechat", fakeBot)
	b.Run(k8sImplementer, approvalsManager)
	return fakeBot
}

func init() {
	log.SetLevel(log.DebugLevel)
}

func TestHelpCommand(t *testing.T) {
	f8s := &testutil.FakeK8sImplementer{}
	fi := &fakeXmppImplementer{}
	fi.messages = make(chan *h.Message)
	mem := memory.NewMemoryCache(100*time.Second, 100*time.Second, 10*time.Second)
	am := approvals.New(mem, codecs.DefaultSerializer())

	NewBot(f8s, am, fi)
	defer b.Stop()

	time.Sleep(1 * time.Second)

	if len(fi.postedMessages) != 1 {
		t.Errorf("expected to find 1 message, but got: %d", len(fi.postedMessages))
	}
	if !strings.HasPrefix(fi.postedMessages[0].text, "Keel bot was started") {
		t.Errorf("expected to find greeting message, but got: %s", fi.postedMessages[0].text)
	}

	fi.messageFromChat("help")
	time.Sleep(1 * time.Second)

	if len(fi.postedMessages) != 2 {
		t.Errorf("expected to find 2 messages, but got: %d", len(fi.postedMessages))
	}
	if !strings.HasPrefix(fi.postedMessages[1].text, "/code Here's a list of supported commands") {
		t.Errorf("expected to find help message, but got: %s", fi.postedMessages[1].text)
	}
}

func TestBotAproval(t *testing.T) {
	f8s := &testutil.FakeK8sImplementer{}
	fi := &fakeXmppImplementer{}
	fi.messages = make(chan *h.Message)
	mem := memory.NewMemoryCache(100*time.Second, 100*time.Second, 10*time.Second)
	am := approvals.New(mem, codecs.DefaultSerializer())

	NewBot(f8s, am, fi)
	defer b.Stop()

	time.Sleep(1 * time.Second)

	err := am.Create(&types.Approval{
		Identifier:     "k8s/project/repo:1.2.3",
		VotesRequired:  1,
		CurrentVersion: "2.3.4",
		NewVersion:     "3.4.5",
		Event: &types.Event{
			Repository: types.Repository{
				Name: "project/repo",
				Tag:  "2.3.4",
			},
		},
	})

	if err != nil {
		t.Fatalf("unexpected error while creating : %s", err)
	}

	time.Sleep(1 * time.Second)

	if len(fi.postedMessages) != 2 {
		t.Errorf("expected to find 2 message, but got: %d", len(fi.postedMessages))
	}
	if !strings.HasPrefix(fi.postedMessages[1].text, "/code Approval required!") {
		t.Errorf("expected to find help message, but got: %s", fi.postedMessages[1].text)
	}

	// approve
	fi.messageFromChat("approve k8s/project/repo:1.2.3")
	time.Sleep(1 * time.Second)

	if len(fi.postedMessages) != 3 {
		t.Errorf("expected to find 3 message, but got: %d", len(fi.postedMessages))
	}
	if !strings.HasPrefix(fi.postedMessages[2].text, "/code Update approved!") {
		t.Errorf("expected to find message, but got: %s", fi.postedMessages[2].text)
	}

	// get approvals
	fi.messageFromChat("get approvals")
	time.Sleep(1 * time.Second)
	if len(fi.postedMessages) != 4 {
		t.Errorf("expected to find 4 message, but got: %d", len(fi.postedMessages))
	}
	resp := trimSpaces(fi.postedMessages[3].text)

	if !strings.Contains(resp, "k8s/project/repo:1.2.3 2.3.4 -> 3.4.5 1/1 false") {
		t.Errorf("expected to find message, but got: %s", resp)
	}
}

func TestBotReject(t *testing.T) {
	f8s := &testutil.FakeK8sImplementer{}
	fi := &fakeXmppImplementer{}
	fi.messages = make(chan *h.Message)
	mem := memory.NewMemoryCache(100*time.Second, 100*time.Second, 10*time.Second)
	am := approvals.New(mem, codecs.DefaultSerializer())

	NewBot(f8s, am, fi)
	defer b.Stop()

	time.Sleep(1 * time.Second)

	err := am.Create(&types.Approval{
		Identifier:     "k8s/project/repo:1.2.3",
		VotesRequired:  1,
		CurrentVersion: "2.3.4",
		NewVersion:     "3.4.5",
		Event: &types.Event{
			Repository: types.Repository{
				Name: "project/repo",
				Tag:  "2.3.4",
			},
		},
	})

	if err != nil {
		t.Fatalf("unexpected error while creating : %s", err)
	}

	time.Sleep(1 * time.Second)

	if len(fi.postedMessages) != 2 {
		t.Errorf("expected to find 2 message, but got: %d", len(fi.postedMessages))
	}
	if !strings.HasPrefix(fi.postedMessages[1].text, "/code Approval required!") {
		t.Errorf("expected to find help message, but got: %s", fi.postedMessages[1].text)
	}

	// reject
	fi.messageFromChat("reject k8s/project/repo:1.2.3")
	time.Sleep(1 * time.Second)

	if len(fi.postedMessages) != 3 {
		t.Errorf("expected to find 3 message, but got: %d", len(fi.postedMessages))
	}
	if !strings.HasPrefix(fi.postedMessages[2].text, "/code Change rejected") {
		t.Errorf("expected to find message, but got: %s", fi.postedMessages[2].text)
	}

	// get approvals
	fi.messageFromChat("get approvals")
	time.Sleep(1 * time.Second)
	if len(fi.postedMessages) != 4 {
		t.Errorf("expected to find 4 message, but got: %d", len(fi.postedMessages))
	}
	resp := trimSpaces(fi.postedMessages[3].text)

	if !strings.Contains(resp, "k8s/project/repo:1.2.3 2.3.4 -> 3.4.5 0/1 true") {
		t.Errorf("expected to find message, but got: %s", resp)
	}
}

func trimSpaces(input string) string {
	reLeadcloseWhtsp := regexp.MustCompile(`^[\s\p{Zs}]+|[\s\p{Zs}]+$`)
	reInsideWhtsp := regexp.MustCompile(`[\s\p{Zs}]{2,}`)
	final := reLeadcloseWhtsp.ReplaceAllString(input, "")
	final = reInsideWhtsp.ReplaceAllString(final, " ")
	return final
}
<commit_msg>fixed broken hipchat tests<commit_after>package hipchat

import (
	"os"
	"regexp"
	"strings"
	"testing"
	"time"

	h "github.com/daneharrigan/hipchat"

	"github.com/keel-hq/keel/approvals"
	b "github.com/keel-hq/keel/bot"
	"github.com/keel-hq/keel/cache/memory"
	"github.com/keel-hq/keel/provider/kubernetes"
	"github.com/keel-hq/keel/types"
	"github.com/keel-hq/keel/util/codecs"

	testutil "github.com/keel-hq/keel/util/testing"

	log "github.com/Sirupsen/logrus"
)

type fakeProvider struct {
	submitted []types.Event
	images    []*types.TrackedImage
}

func (p *fakeProvider) Submit(event types.Event) error {
	p.submitted = append(p.submitted, event)
	return nil
}

func (p *fakeProvider) TrackedImages() ([]*types.TrackedImage, error) {
	return p.images, nil
}

func (p *fakeProvider) List() []string {
	return []string{"fakeprovider"}
}
func (p *fakeProvider) Stop() {
	return
}
func (p *fakeProvider) GetName() string {
	return "fp"
}

var botMessagesChannel chan *b.BotMessage
var approvalsRespCh chan *b.ApprovalResponse

type postedMessage struct {
	channel string
	text    string
}

type fakeXmppImplementer struct {
	postedMessages []postedMessage
	messages       chan *h.Message
}

func (i *fakeXmppImplementer) messageFromChat(message string) {
	i.messages <- &h.Message{
		Body: "@keel " + message,
		From: "111111_approvals@conf.hipchat.com/test",
		To:   "222222_333333@chat.hipchat.com/keel",
	}
}

func (i *fakeXmppImplementer) Say(roomID, name, body string) {
	i.postedMessages = append(i.postedMessages, postedMessage{
		text:    body,
		channel: roomID,
	})
}
func (i *fakeXmppImplementer) Status(s string) {
}
func (i *fakeXmppImplementer) Join(roomID, resource string) {
}
func (i *fakeXmppImplementer) KeepAlive() {
}
func (i *fakeXmppImplementer) Messages() <-chan *h.Message {
	return i.messages
}

func NewBot(k8sImplementer kubernetes.Implementer,
	approvalsManager approvals.Manager, fi XmppImplementer) *Bot {

	approvalsRespCh = make(chan *b.ApprovalResponse)
	botMessagesChannel = make(chan *b.BotMessage)
	fakeBot := &Bot{}
	fakeBot.hipchatClient = fi

	os.Setenv("HIPCHAT_APPROVALS_CHANNEL", "111111_approvals@conf.hipchat.com")
	os.Setenv("HIPCHAT_APPROVALS_BOT_NAME", "keel")
	os.Setenv("HIPCHAT_APPROVALS_USER_NAME", "111111_222222")
	os.Setenv("HIPCHAT_APPROVALS_PASSWORT", "pass")
	os.Setenv("HIPCHAT_CONNECTION_ATTEMPTS", "0")

	b.RegisterBot("fakechat", fakeBot)
	b.Run(k8sImplementer, approvalsManager)
	return fakeBot
}

func init() {
	log.SetLevel(log.DebugLevel)
}

func TestHelpCommand(t *testing.T) {
	f8s := &testutil.FakeK8sImplementer{}
	fi := &fakeXmppImplementer{}
	fi.messages = make(chan *h.Message)
	mem := memory.NewMemoryCache(100*time.Second, 100*time.Second, 10*time.Second)
	am := approvals.New(mem, codecs.DefaultSerializer())

	NewBot(f8s, am, fi)
	defer b.Stop()

	time.Sleep(1 * time.Second)

	if len(fi.postedMessages) != 1 {
		t.Errorf("expected to find 1 message, but got: %d", len(fi.postedMessages))
	}
	if !strings.HasPrefix(fi.postedMessages[0].text, "Keel bot was started") {
		t.Errorf("expected to find greeting message, but got: %s", fi.postedMessages[0].text)
	}

	fi.messageFromChat("help")
	time.Sleep(1 * time.Second)

	if len(fi.postedMessages) != 2 {
		t.Errorf("expected to find 2 messages, but got: %d", len(fi.postedMessages))
	}
	if !strings.HasPrefix(fi.postedMessages[1].text, "/code Here's a list of supported commands") {
		t.Errorf("expected to find help message, but got: %s", fi.postedMessages[1].text)
	}
}

func TestBotAproval(t *testing.T) {
	f8s := &testutil.FakeK8sImplementer{}
	fi := &fakeXmppImplementer{}
	fi.messages = make(chan *h.Message)
	mem := memory.NewMemoryCache(100*time.Second, 100*time.Second, 10*time.Second)
	am := approvals.New(mem, codecs.DefaultSerializer())

	NewBot(f8s, am, fi)
	defer b.Stop()

	time.Sleep(1 * time.Second)

	err := am.Create(&types.Approval{
		Identifier:     "k8s/project/repo:1.2.3",
		VotesRequired:  1,
		CurrentVersion: "2.3.4",
		NewVersion:     "3.4.5",
		Event: &types.Event{
			Repository: types.Repository{
				Name: "project/repo",
				Tag:  "2.3.4",
			},
		},
	})

	if err != nil {
		t.Fatalf("unexpected error while creating : %s", err)
	}

	time.Sleep(1 * time.Second)

	if len(fi.postedMessages) != 2 {
		t.Errorf("expected to find 2 message, but got: %d", len(fi.postedMessages))
	}
	if !strings.HasPrefix(fi.postedMessages[1].text, "/code Approval required!") {
		t.Errorf("expected to find help message, but got: %s", fi.postedMessages[1].text)
	}

	// approve
	fi.messageFromChat("approve k8s/project/repo:1.2.3")
	time.Sleep(1 * time.Second)

	if len(fi.postedMessages) != 3 {
		t.Errorf("expected to find 3 message, but got: %d", len(fi.postedMessages))
	}
	if !strings.HasPrefix(fi.postedMessages[2].text, "/code Update approved!") {
		t.Errorf("expected to find message, but got: %s", fi.postedMessages[2].text)
	}

	// get approvals
	fi.messageFromChat("get approvals")
	time.Sleep(1 * time.Second)
	if len(fi.postedMessages) != 4 {
		t.Errorf("expected to find 4 message, but got: %d", len(fi.postedMessages))
	}
	resp := trimSpaces(fi.postedMessages[3].text)

	if !strings.Contains(resp, "k8s/project/repo:1.2.3 2.3.4 -> 3.4.5 1/1 false") {
		t.Errorf("expected to find message, but got: %s", resp)
	}
}

func TestBotReject(t *testing.T) {
	f8s := &testutil.FakeK8sImplementer{}
	fi := &fakeXmppImplementer{}
	fi.messages = make(chan *h.Message)
	mem := memory.NewMemoryCache(100*time.Second, 100*time.Second, 10*time.Second)
	am := approvals.New(mem, codecs.DefaultSerializer())

	NewBot(f8s, am, fi)
	defer b.Stop()

	time.Sleep(1 * time.Second)

	err := am.Create(&types.Approval{
		Identifier:     "k8s/project/repo:1.2.3",
		VotesRequired:  1,
		CurrentVersion: "2.3.4",
		NewVersion:     "3.4.5",
		Event: &types.Event{
			Repository: types.Repository{
				Name: "project/repo",
				Tag:  "2.3.4",
			},
		},
	})

	if err != nil {
		t.Fatalf("unexpected error while creating : %s", err)
	}

	time.Sleep(1 * time.Second)

	if len(fi.postedMessages) != 2 {
		t.Errorf("expected to find 2 message, but got: %d", len(fi.postedMessages))
	}
	if !strings.HasPrefix(fi.postedMessages[1].text, "/code Approval required!") {
		t.Errorf("expected to find help message, but got: %s", fi.postedMessages[1].text)
	}

	// reject
	fi.messageFromChat("reject k8s/project/repo:1.2.3")
	time.Sleep(1 * time.Second)

	if len(fi.postedMessages) != 3 {
		t.Errorf("expected to find 3 message, but got: %d", len(fi.postedMessages))
	}
	if !strings.HasPrefix(fi.postedMessages[2].text, "/code Change rejected") {
		t.Errorf("expected to find message, but got: %s", fi.postedMessages[2].text)
	}

	// get approvals
	fi.messageFromChat("get approvals")
	time.Sleep(1 * time.Second)
	if len(fi.postedMessages) != 4 {
		t.Errorf("expected to find 4 message, but got: %d", len(fi.postedMessages))
	}
	resp := trimSpaces(fi.postedMessages[3].text)

	if !strings.Contains(resp, "k8s/project/repo:1.2.3 2.3.4 -> 3.4.5 0/1 true") {
		t.Errorf("expected to find message, but got: %s", resp)
	}
}

func trimSpaces(input string) string {
	reLeadcloseWhtsp := regexp.MustCompile(`^[\s\p{Zs}]+|[\s\p{Zs}]+$`)
	reInsideWhtsp := regexp.MustCompile(`[\s\p{Zs}]{2,}`)
	final := reLeadcloseWhtsp.ReplaceAllString(input, "")
	final = reInsideWhtsp.ReplaceAllString(final, " ")
	return final
}
<|endoftext|>


<commit_before>package tictactoe

import (
	"github.com/jkomoros/boardgame"
)

type gameState struct {
	Slots *boardgame.SizedStack
}

func (g *gameState) Props() []string {
	return boardgame.PropertyReaderPropsImpl(g)
}

func (g *gameState) Prop(name string) interface{} {
	return boardgame.PropertyReaderPropImpl(g, name)
}

func (g *gameState) Copy() boardgame.GameState {
	var result gameState
	result = *g
	return &result
}

func (g *gameState) JSON() boardgame.JSONObject {
	return g
}

type statePayload struct {
	game *gameState
	//we have no user state because all state is public.
}

func (s *statePayload) Game() boardgame.GameState {
	return s.game
}

func (s *statePayload) Users() []boardgame.UserState {
	return nil
}

func (s *statePayload) JSON() boardgame.JSONObject {
	return boardgame.JSONMap{
		"Game": s.game.JSON(),
	}
}

func (s *statePayload) Copy() boardgame.StatePayload {
	return &statePayload{
		game: s.game.Copy().(*gameState),
	}
}
<commit_msg>Add a userState that holds unused tokens.<commit_after>package tictactoe

import (
	"github.com/jkomoros/boardgame"
)

type gameState struct {
	Slots *boardgame.SizedStack
}

func (g *gameState) Props() []string {
	return boardgame.PropertyReaderPropsImpl(g)
}

func (g *gameState) Prop(name string) interface{} {
	return boardgame.PropertyReaderPropImpl(g, name)
}

func (g *gameState) Copy() boardgame.GameState {
	var result gameState
	result = *g
	return &result
}

func (g *gameState) JSON() boardgame.JSONObject {
	return g
}

type userState struct {
	playerIndex  int
	UnusedTokens *boardgame.GrowableStack
}

func (u *userState) Props() []string {
	return boardgame.PropertyReaderPropsImpl(u)
}

func (u *userState) Prop(name string) interface{} {
	return boardgame.PropertyReaderPropImpl(u, name)
}

func (u *userState) Copy() boardgame.UserState {
	var result userState
	result = *u
	return &result
}

func (u *userState) JSON() boardgame.JSONObject {
	return u
}

func (u *userState) PlayerIndex() int {
	return u.playerIndex
}

type statePayload struct {
	game  *gameState
	users []*userState
}

func (s *statePayload) Game() boardgame.GameState {
	return s.game
}

func (s *statePayload) Users() []boardgame.UserState {
	array := make([]boardgame.UserState, len(s.users))

	for i := 0; i < len(s.users); i++ {
		array[i] = s.users[i]
	}

	return array
}

func (s *statePayload) JSON() boardgame.JSONObject {

	array := make([]boardgame.JSONObject, len(s.users))

	for i, user := range s.users {
		array[i] = user.JSON()
	}

	return boardgame.JSONMap{
		"Game":  s.game.JSON(),
		"Users": array,
	}
}

func (s *statePayload) Copy() boardgame.StatePayload {
	array := make([]*userState, len(s.users))

	for i := 0; i < len(s.users); i++ {
		array[i] = s.users[i].Copy().(*userState)
	}

	return &statePayload{
		game:  s.game.Copy().(*gameState),
		users: array,
	}
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package comm

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/base64"
	"encoding/pem"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"sync"
	"time"

	"github.com/hyperledger/fabric/common/flogging"
	"github.com/hyperledger/fabric/core/config"
	"github.com/spf13/viper"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/grpclog"
)

const defaultTimeout = time.Second * 3

var commLogger = flogging.MustGetLogger("comm")
var caSupport *CASupport
var once sync.Once

// CASupport type manages certificate authorities scoped by channel
type CASupport struct {
	sync.RWMutex
	AppRootCAsByChain     map[string][][]byte
	OrdererRootCAsByChain map[string][][]byte
	ClientRootCAs         [][]byte
	ServerRootCAs         [][]byte
}

// GetCASupport returns the singleton CASupport instance
func GetCASupport() *CASupport {

	once.Do(func() {
		caSupport = &CASupport{
			AppRootCAsByChain:     make(map[string][][]byte),
			OrdererRootCAsByChain: make(map[string][][]byte),
		}
	})
	return caSupport
}

// GetServerRootCAs returns the PEM-encoded root certificates for all of the
// application and orderer organizations defined for all chains.  The root
// certificates returned should be used to set the trusted server roots for
// TLS clients.
func (cas *CASupport) GetServerRootCAs() (appRootCAs, ordererRootCAs [][]byte) {
	cas.RLock()
	defer cas.RUnlock()

	appRootCAs = [][]byte{}
	ordererRootCAs = [][]byte{}

	for _, appRootCA := range cas.AppRootCAsByChain {
		appRootCAs = append(appRootCAs, appRootCA...)
	}

	for _, ordererRootCA := range cas.OrdererRootCAsByChain {
		ordererRootCAs = append(ordererRootCAs, ordererRootCA...)
	}

	// also need to append statically configured root certs
	appRootCAs = append(appRootCAs, cas.ServerRootCAs...)
	return appRootCAs, ordererRootCAs
}

// GetDeliverServiceCredentials returns GRPC transport credentials for given channel to be used by GRPC
// clients which communicate with ordering service endpoints.
// If the channel isn't found, error is returned.
func (cas *CASupport) GetDeliverServiceCredentials(channelID string) (credentials.TransportCredentials, error) {
	cas.RLock()
	defer cas.RUnlock()

	var creds credentials.TransportCredentials
	var tlsConfig = &tls.Config{}
	var certPool = x509.NewCertPool()

	rootCACerts, exists := cas.OrdererRootCAsByChain[channelID]
	if !exists {
		commLogger.Errorf("Attempted to obtain root CA certs of a non existent channel: %s", channelID)
		return nil, fmt.Errorf("didn't find any root CA certs for channel %s", channelID)
	}

	for _, cert := range rootCACerts {
		block, _ := pem.Decode(cert)
		if block != nil {
			cert, err := x509.ParseCertificate(block.Bytes)
			if err == nil {
				certPool.AddCert(cert)
			} else {
				commLogger.Warningf("Failed to add root cert to credentials (%s)", err)
			}
		} else {
			commLogger.Warning("Failed to add root cert to credentials")
		}
	}
	tlsConfig.RootCAs = certPool
	creds = credentials.NewTLS(tlsConfig)
	return creds, nil
}

// GetPeerCredentials returns GRPC transport credentials for use by GRPC
// clients which communicate with remote peer endpoints.
func (cas *CASupport) GetPeerCredentials(tlsCert tls.Certificate) credentials.TransportCredentials {
	var creds credentials.TransportCredentials
	var tlsConfig = &tls.Config{
		Certificates: []tls.Certificate{tlsCert},
	}
	var certPool = x509.NewCertPool()
	// loop through the orderer CAs
	roots, _ := cas.GetServerRootCAs()
	for _, root := range roots {
		block, _ := pem.Decode(root)
		if block != nil {
			cert, err := x509.ParseCertificate(block.Bytes)
			if err == nil {
				certPool.AddCert(cert)
			} else {
				commLogger.Warningf("Failed to add root cert to credentials (%s)", err)
			}
		} else {
			commLogger.Warning("Failed to add root cert to credentials")
		}
	}
	tlsConfig.RootCAs = certPool
	creds = credentials.NewTLS(tlsConfig)
	return creds
}

// GetClientRootCAs returns the PEM-encoded root certificates for all of the
// application and orderer organizations defined for all chains.  The root
// certificates returned should be used to set the trusted client roots for
// TLS servers.
func (cas *CASupport) GetClientRootCAs() (appRootCAs, ordererRootCAs [][]byte) {
	cas.RLock()
	defer cas.RUnlock()

	appRootCAs = [][]byte{}
	ordererRootCAs = [][]byte{}

	for _, appRootCA := range cas.AppRootCAsByChain {
		appRootCAs = append(appRootCAs, appRootCA...)
	}

	for _, ordererRootCA := range cas.OrdererRootCAsByChain {
		ordererRootCAs = append(ordererRootCAs, ordererRootCA...)
	}

	// also need to append statically configured root certs
	appRootCAs = append(appRootCAs, cas.ClientRootCAs...)
	return appRootCAs, ordererRootCAs
}

func getEnv(key, def string) string {
	val := os.Getenv(key)
	if len(val) > 0 {
		return val
	} else {
		return def
	}
}

func GetPeerTestingAddress(port string) string {
	return getEnv("UNIT_TEST_PEER_IP", "localhost") + ":" + port
}

// NewClientConnectionWithAddress Returns a new grpc.ClientConn to the given address.
func NewClientConnectionWithAddress(peerAddress string, block bool, tslEnabled bool, creds credentials.TransportCredentials) (*grpc.ClientConn, error) {
	var opts []grpc.DialOption
	if tslEnabled {
		opts = append(opts, grpc.WithTransportCredentials(creds))
	} else {
		opts = append(opts, grpc.WithInsecure())
	}
	opts = append(opts, grpc.WithTimeout(defaultTimeout))
	if block {
		opts = append(opts, grpc.WithBlock())
	}
	opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(MaxRecvMsgSize()),
		grpc.MaxCallSendMsgSize(MaxSendMsgSize())))
	conn, err := grpc.Dial(peerAddress, opts...)
	if err != nil {
		return nil, err
	}
	return conn, err
}

// InitTLSForPeer returns TLS credentials for peer
func InitTLSForPeer() credentials.TransportCredentials {
	var sn string
	if viper.GetString("peer.tls.serverhostoverride") != "" {
		sn = viper.GetString("peer.tls.serverhostoverride")
	}
	var creds credentials.TransportCredentials
	if config.GetPath("peer.tls.rootcert.file") != "" {
		var err error
		creds, err = credentials.NewClientTLSFromFile(config.GetPath("peer.tls.rootcert.file"), sn)
		if err != nil {
			grpclog.Fatalf("Failed to create TLS credentials %v", err)
		}
	} else {
		creds = credentials.NewClientTLSFromCert(nil, sn)
	}
	return creds
}

func InitTLSForShim(key, certStr string) credentials.TransportCredentials {
	var sn string
	if viper.GetString("peer.tls.serverhostoverride") != "" {
		sn = viper.GetString("peer.tls.serverhostoverride")
	}
	priv, err := base64.StdEncoding.DecodeString(key)
	if err != nil {
		panic(fmt.Errorf("failed decoding private key from base64, string: %s, error: %v", key, err))
	}
	pub, err := base64.StdEncoding.DecodeString(certStr)
	if err != nil {
		panic(fmt.Errorf("failed decoding public key from base64, string: %s, error: %v", certStr, err))
	}
	cert, err := tls.X509KeyPair(pub, priv)
	if err != nil {
		panic(fmt.Errorf("failed loading certificate: %v", err))
	}
	b, err := ioutil.ReadFile(config.GetPath("peer.tls.rootcert.file"))
	if err != nil {
		panic(fmt.Errorf("failed loading root ca cert: %v", err))
	}
	cp := x509.NewCertPool()
	if !cp.AppendCertsFromPEM(b) {
		panic(errors.New("failed to append certificates"))
	}
	return credentials.NewTLS(&tls.Config{
		Certificates: []tls.Certificate{cert},
		RootCAs:      cp,
		ServerName:   sn,
	})
}
<commit_msg>[FAB-5406] Log panic with logger<commit_after>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package comm

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/base64"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"os"
	"sync"
	"time"

	"github.com/hyperledger/fabric/common/flogging"
	"github.com/hyperledger/fabric/core/config"
	"github.com/spf13/viper"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/grpclog"
)

const defaultTimeout = time.Second * 3

var commLogger = flogging.MustGetLogger("comm")
var caSupport *CASupport
var once sync.Once

// CASupport type manages certificate authorities scoped by channel
type CASupport struct {
	sync.RWMutex
	AppRootCAsByChain     map[string][][]byte
	OrdererRootCAsByChain map[string][][]byte
	ClientRootCAs         [][]byte
	ServerRootCAs         [][]byte
}

// GetCASupport returns the singleton CASupport instance
func GetCASupport() *CASupport {

	once.Do(func() {
		caSupport = &CASupport{
			AppRootCAsByChain:     make(map[string][][]byte),
			OrdererRootCAsByChain: make(map[string][][]byte),
		}
	})
	return caSupport
}

// GetServerRootCAs returns the PEM-encoded root certificates for all of the
// application and orderer organizations defined for all chains.  The root
// certificates returned should be used to set the trusted server roots for
// TLS clients.
func (cas *CASupport) GetServerRootCAs() (appRootCAs, ordererRootCAs [][]byte) {
	cas.RLock()
	defer cas.RUnlock()

	appRootCAs = [][]byte{}
	ordererRootCAs = [][]byte{}

	for _, appRootCA := range cas.AppRootCAsByChain {
		appRootCAs = append(appRootCAs, appRootCA...)
	}

	for _, ordererRootCA := range cas.OrdererRootCAsByChain {
		ordererRootCAs = append(ordererRootCAs, ordererRootCA...)
	}

	// also need to append statically configured root certs
	appRootCAs = append(appRootCAs, cas.ServerRootCAs...)
	return appRootCAs, ordererRootCAs
}

// GetDeliverServiceCredentials returns GRPC transport credentials for given channel to be used by GRPC
// clients which communicate with ordering service endpoints.
// If the channel isn't found, error is returned.
func (cas *CASupport) GetDeliverServiceCredentials(channelID string) (credentials.TransportCredentials, error) {
	cas.RLock()
	defer cas.RUnlock()

	var creds credentials.TransportCredentials
	var tlsConfig = &tls.Config{}
	var certPool = x509.NewCertPool()

	rootCACerts, exists := cas.OrdererRootCAsByChain[channelID]
	if !exists {
		commLogger.Errorf("Attempted to obtain root CA certs of a non existent channel: %s", channelID)
		return nil, fmt.Errorf("didn't find any root CA certs for channel %s", channelID)
	}

	for _, cert := range rootCACerts {
		block, _ := pem.Decode(cert)
		if block != nil {
			cert, err := x509.ParseCertificate(block.Bytes)
			if err == nil {
				certPool.AddCert(cert)
			} else {
				commLogger.Warningf("Failed to add root cert to credentials (%s)", err)
			}
		} else {
			commLogger.Warning("Failed to add root cert to credentials")
		}
	}
	tlsConfig.RootCAs = certPool
	creds = credentials.NewTLS(tlsConfig)
	return creds, nil
}

// GetPeerCredentials returns GRPC transport credentials for use by GRPC
// clients which communicate with remote peer endpoints.
func (cas *CASupport) GetPeerCredentials(tlsCert tls.Certificate) credentials.TransportCredentials {
	var creds credentials.TransportCredentials
	var tlsConfig = &tls.Config{
		Certificates: []tls.Certificate{tlsCert},
	}
	var certPool = x509.NewCertPool()
	// loop through the orderer CAs
	roots, _ := cas.GetServerRootCAs()
	for _, root := range roots {
		block, _ := pem.Decode(root)
		if block != nil {
			cert, err := x509.ParseCertificate(block.Bytes)
			if err == nil {
				certPool.AddCert(cert)
			} else {
				commLogger.Warningf("Failed to add root cert to credentials (%s)", err)
			}
		} else {
			commLogger.Warning("Failed to add root cert to credentials")
		}
	}
	tlsConfig.RootCAs = certPool
	creds = credentials.NewTLS(tlsConfig)
	return creds
}

// GetClientRootCAs returns the PEM-encoded root certificates for all of the
// application and orderer organizations defined for all chains.  The root
// certificates returned should be used to set the trusted client roots for
// TLS servers.
func (cas *CASupport) GetClientRootCAs() (appRootCAs, ordererRootCAs [][]byte) {
	cas.RLock()
	defer cas.RUnlock()

	appRootCAs = [][]byte{}
	ordererRootCAs = [][]byte{}

	for _, appRootCA := range cas.AppRootCAsByChain {
		appRootCAs = append(appRootCAs, appRootCA...)
	}

	for _, ordererRootCA := range cas.OrdererRootCAsByChain {
		ordererRootCAs = append(ordererRootCAs, ordererRootCA...)
	}

	// also need to append statically configured root certs
	appRootCAs = append(appRootCAs, cas.ClientRootCAs...)
	return appRootCAs, ordererRootCAs
}

func getEnv(key, def string) string {
	val := os.Getenv(key)
	if len(val) > 0 {
		return val
	} else {
		return def
	}
}

func GetPeerTestingAddress(port string) string {
	return getEnv("UNIT_TEST_PEER_IP", "localhost") + ":" + port
}

// NewClientConnectionWithAddress Returns a new grpc.ClientConn to the given address.
func NewClientConnectionWithAddress(peerAddress string, block bool, tslEnabled bool, creds credentials.TransportCredentials) (*grpc.ClientConn, error) {
	var opts []grpc.DialOption
	if tslEnabled {
		opts = append(opts, grpc.WithTransportCredentials(creds))
	} else {
		opts = append(opts, grpc.WithInsecure())
	}
	opts = append(opts, grpc.WithTimeout(defaultTimeout))
	if block {
		opts = append(opts, grpc.WithBlock())
	}
	opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(MaxRecvMsgSize()),
		grpc.MaxCallSendMsgSize(MaxSendMsgSize())))
	conn, err := grpc.Dial(peerAddress, opts...)
	if err != nil {
		return nil, err
	}
	return conn, err
}

// InitTLSForPeer returns TLS credentials for peer
func InitTLSForPeer() credentials.TransportCredentials {
	var sn string
	if viper.GetString("peer.tls.serverhostoverride") != "" {
		sn = viper.GetString("peer.tls.serverhostoverride")
	}
	var creds credentials.TransportCredentials
	if config.GetPath("peer.tls.rootcert.file") != "" {
		var err error
		creds, err = credentials.NewClientTLSFromFile(config.GetPath("peer.tls.rootcert.file"), sn)
		if err != nil {
			grpclog.Fatalf("Failed to create TLS credentials %v", err)
		}
	} else {
		creds = credentials.NewClientTLSFromCert(nil, sn)
	}
	return creds
}

func InitTLSForShim(key, certStr string) credentials.TransportCredentials {
	var sn string
	if viper.GetString("peer.tls.serverhostoverride") != "" {
		sn = viper.GetString("peer.tls.serverhostoverride")
	}
	priv, err := base64.StdEncoding.DecodeString(key)
	if err != nil {
		commLogger.Panicf("failed decoding private key from base64, string: %s, error: %v", key, err)
	}
	pub, err := base64.StdEncoding.DecodeString(certStr)
	if err != nil {
		commLogger.Panicf("failed decoding public key from base64, string: %s, error: %v", certStr, err)
	}
	cert, err := tls.X509KeyPair(pub, priv)
	if err != nil {
		commLogger.Panicf("failed loading certificate: %v", err)
	}
	b, err := ioutil.ReadFile(config.GetPath("peer.tls.rootcert.file"))
	if err != nil {
		commLogger.Panicf("failed loading root ca cert: %v", err)
	}
	cp := x509.NewCertPool()
	if !cp.AppendCertsFromPEM(b) {
		commLogger.Panicf("failed to append certificates")
	}
	return credentials.NewTLS(&tls.Config{
		Certificates: []tls.Certificate{cert},
		RootCAs:      cp,
		ServerName:   sn,
	})
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"net/mail"
	"os"
	"os/signal"
	"time"
)

const (
	tasklist_template = "ui_template.html"
)

type Task struct {
	Description string
	Frequency   time.Duration
	Users       []User // already a list (future feature)
}
type User mail.Address
type TaskList map[string]Task

func main() {

	file := "tasks.json"
	tasks := loadFromJson(file)
	defer saveToJson(file, tasks)

	go uiServer(8080, tasks)
	// go handleGlobalContext(&ctx)

	sig := make(chan os.Signal)
	signal.Notify(sig, os.Interrupt)
	select {
	case <-sig:
		saveToJson(file, tasks)
		fmt.Println("\nExiting …")
	}
}

func uiServer(port int, tasks TaskList) {
	http.Handle("/", tasks)
	http.HandleFunc("/static/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, r.URL.Path[1:])
	})
	http.ListenAndServe(fmt.Sprintf(":%d", port), nil)

}

func (tasks TaskList) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	switch req.Method {
	case "GET":
		if req.URL.Path == "/" {
			w.Header()["Content-Type"] = []string{"text/html"}
			ts, err := ioutil.ReadFile(tasklist_template)
			logFatal(err)
			t := template.Must(template.New("tasklist").Parse(string(ts)))

			t.Execute(w, tasks)
		}

	case "POST":
		err := req.ParseForm()
		if err == nil {
			if req.URL.Path == "/commit" {
				/* What we get via POST:
				E-Mail: email
				Name: name
				each checked task is transmitted as one key
				(see for-loop below)
				req.Form looks like this:
				map[name:[sternenseemann] Foobar:[do] submit:[Commit] email:[foo@foo.de]]
				*/
				for taskname, _ := range tasks {
					if req.Form[taskname] != nil {
						// TODO: check for existance of the fields
					}
				}
				http.Redirect(w, req, "/", http.StatusFound)
			}
		} else {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
	}

}

func loadFromJson(file string) TaskList {
	b, err := ioutil.ReadFile(file)
	if err != nil {
		l := make(TaskList)
		return l
	}
	var tasks TaskList
	err = json.Unmarshal(b, &tasks)
	logFatal(err)
	return tasks
}

func saveToJson(file string, tasks TaskList) {
	b, err := json.Marshal(tasks)

	logFatal(err)
	err = ioutil.WriteFile(file, b, 0644)
	logFatal(err)
}

func logFatal(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>using pointers again for less copies of memory<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"net/mail"
	"os"
	"os/signal"
	"time"
)

const (
	tasklist_template = "ui_template.html"
)

type Task struct {
	Description string
	Frequency   time.Duration
	Users       []User // already a list (future feature)
}
type User mail.Address
type TaskList map[string]Task

func main() {

	file := "tasks.json"
	tasks := loadFromJson(file)
	defer saveToJson(file, tasks)

	go uiServer(8080, tasks)
	// go handleGlobalContext(&ctx)

	sig := make(chan os.Signal)
	signal.Notify(sig, os.Interrupt)
	select {
	case <-sig:
		saveToJson(file, tasks)
		fmt.Println("\nExiting …")
	}
}

func uiServer(port int, tasks *TaskList) {
	http.Handle("/", tasks)
	http.HandleFunc("/static/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, r.URL.Path[1:])
	})
	http.ListenAndServe(fmt.Sprintf(":%d", port), nil)

}

func (tasks *TaskList) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	switch req.Method {
	case "GET":
		if req.URL.Path == "/" {
			w.Header()["Content-Type"] = []string{"text/html"}
			ts, err := ioutil.ReadFile(tasklist_template)
			logFatal(err)
			t := template.Must(template.New("tasklist").Parse(string(ts)))

			t.Execute(w, tasks)
		}

	case "POST":
		err := req.ParseForm()
		if err == nil {
			if req.URL.Path == "/commit" {
				/* What we get via POST:
				E-Mail: email
				Name: name
				each checked task is transmitted as one key
				(see for-loop below)
				req.Form looks like this:
				map[name:[sternenseemann] Foobar:[do] submit:[Commit] email:[foo@foo.de]]
				*/
				for taskname, _ := range *tasks {
					if req.Form[taskname] != nil {
						// TODO: check for existance of the fields
						fmt.Println("The User", req.Form["name"][0], "with email", req.Form["email"][0],
							"commited theirselves the task", taskname)
					}
				}
				http.Redirect(w, req, "/", http.StatusFound)
			}
		} else {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
	}

}

func loadFromJson(file string) *TaskList {
	b, err := ioutil.ReadFile(file)
	if err != nil {
		l := make(TaskList)
		return &l
	}
	var tasks TaskList
	err = json.Unmarshal(b, &tasks)
	logFatal(err)
	return &tasks
}

func saveToJson(file string, tasks *TaskList) {
	b, err := json.Marshal(tasks)

	logFatal(err)
	err = ioutil.WriteFile(file, b, 0644)
	logFatal(err)
}

func logFatal(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>/*
Package iterbool provides tools for creating iterators for the bool type.

These iterators are intentionally made to resemble *sql.Rows from the "database/sql" package.
Including having the same Close, Err, Next, and Scan methods.

(Note that to turn something into an actual *sql.Rows from the "database/sql" package, instead of just resembling it,
use https://github.com/reiver/go-shunt instead.)

For example, we can turn a slice into an iterator, with code like the following:

	var slice []string = []string {
		false,
		true,
		true,
		false,
		true,
	}

	iterator := iterbool.Slice{
		Slice: slice,
	}

	defer iterator.Close()

	for iterator.Next() {

		var datum bool // Could have also used: var datum interface{}

		if err := iterator.Decode(&datum); nil != err {
			return err
		}

		fmt.Printf("Next datum: %v \n", datum)
	}
	if err := iterator.Err(); nil != err {
		return err
	}

This can help to enable us to write more (run-time oriented) generic code.

To be able to distinguish one iterator type from another, in a (run-time oriented)
generic way, that is not so specific to this package, we can use the Type method.

For example:

	switch reflect.Zero( iterator.Type() ).Interface().(type) {
	case bool:
		//@TODO
	case byte:
		//@TODO
	case complex64:
		//@TODO
	case complex128:
		//@TODO
	case float32:
		//@TODO
	case float64:
		//@TODO
	case [2]float64:
		//@TODO
	case [3]float64:
		//@TODO
	case [4]float64:
		//@TODO
	case int8:
		//@TODO
	case int16:
		//@TODO
	case int32:
		//@TODO
	case int64:
		//@TODO
	case rune:
		//@TODO
	case string:
		//@TODO
	case struct{}:
		//@TODO
	case time.Time:
		//@TODO
	case uint8:
		//@TODO
	case uint16:
		//@TODO
	case uint32:
		//@TODO
	case uint64:
		//@TODO
	default:
		//@TODO
	}
*/
package iterbool
<commit_msg>added docs<commit_after>/*
Package iterbool provides tools for creating iterators for the bool type.

These iterators are intentionally made to resemble *sql.Rows from the "database/sql" package.
Including having the same Close, Err, Next, and Scan methods.

(Note that to turn something into an actual *sql.Rows from the "database/sql" package, instead of just resembling it,
use https://github.com/reiver/go-shunt instead.)

For example, we can turn a slice into an iterator, with code like the following:

	var slice []bool = []bool {
		false,
		true,
		true,
		false,
		true,
	}

	iterator := iterbool.Slice{
		Slice: slice,
	}

	defer iterator.Close()

	for iterator.Next() {

		var datum bool // Could have also used: var datum interface{}

		if err := iterator.Decode(&datum); nil != err {
			return err
		}

		fmt.Printf("Next datum: %v \n", datum)
	}
	if err := iterator.Err(); nil != err {
		return err
	}

This can help to enable us to write more (run-time oriented) generic code.

To be able to distinguish one iterator type from another, in a (run-time oriented)
generic way, that is not so specific to this package, we can use the Type method.

For example:

	switch reflect.Zero( iterator.Type() ).Interface().(type) {
	case bool:
		//@TODO
	case byte:
		//@TODO
	case complex64:
		//@TODO
	case complex128:
		//@TODO
	case float32:
		//@TODO
	case float64:
		//@TODO
	case [2]float64:
		//@TODO
	case [3]float64:
		//@TODO
	case [4]float64:
		//@TODO
	case int8:
		//@TODO
	case int16:
		//@TODO
	case int32:
		//@TODO
	case int64:
		//@TODO
	case rune:
		//@TODO
	case string:
		//@TODO
	case struct{}:
		//@TODO
	case time.Time:
		//@TODO
	case uint8:
		//@TODO
	case uint16:
		//@TODO
	case uint32:
		//@TODO
	case uint64:
		//@TODO
	default:
		//@TODO
	}
*/
package iterbool
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"sync"
	"syscall"

	"github.com/golang/glog"
	_ "github.com/hyperhq/runv/cli/nsenter"
	"github.com/hyperhq/runv/hyperstart/libhyperstart"
	"github.com/hyperhq/runv/lib/term"
	"github.com/kardianos/osext"
	"github.com/kr/pty"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/urfave/cli"
)

var shimCommand = cli.Command{
	Name:     "shim",
	Usage:    "[internal command] proxy operations(io, signal ...) to the container/process",
	HideHelp: true,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name: "container",
		},
		cli.StringFlag{
			Name: "process",
		},
		cli.BoolFlag{
			Name: "proxy-exit-code",
		},
		cli.BoolFlag{
			Name: "proxy-stdio",
		},
		cli.BoolFlag{
			Name: "proxy-signal",
		},
		cli.BoolFlag{
			Name: "proxy-winsize",
		},
	},
	Before: func(context *cli.Context) error {
		return cmdPrepare(context, false, false)
	},
	Action: func(context *cli.Context) error {
		container := context.String("container")
		process := context.String("process")

		h, err := libhyperstart.NewGrpcBasedHyperstart(filepath.Join(context.GlobalString("root"), container, "sandbox", "hyperstartgrpc.sock"))
		if err != nil {
			return cli.NewExitError(fmt.Sprintf("failed to connect to hyperstart proxy: %v", err), -1)
		}

		if process == "init" {
			waitSigUsr1 := make(chan os.Signal, 1)
			signal.Notify(waitSigUsr1, syscall.SIGUSR1)
			<-waitSigUsr1
			signal.Stop(waitSigUsr1)
		}

		if context.Bool("proxy-stdio") {
			wg := &sync.WaitGroup{}
			proxyStdio(h, container, process, wg)
			defer wg.Wait()
		}

		if context.Bool("proxy-winsize") {
			glog.V(3).Infof("using shim to proxy winsize")
			s, err := term.SetRawTerminal(os.Stdin.Fd())
			if err != nil {
				return cli.NewExitError(fmt.Sprintf("failed to set raw terminal: %v", err), -1)
			}
			defer term.RestoreTerminal(os.Stdin.Fd(), s)
			monitorTtySize(h, container, process)
		}

		if context.Bool("proxy-signal") {
			glog.V(3).Infof("using shim to proxy signal")
			sigc := forwardAllSignals(h, container, process)
			defer signal.Stop(sigc)
		}

		// wait until exit
		exitcode := h.WaitProcess(container, process)
		if context.Bool("proxy-exit-code") {
			glog.V(3).Infof("using shim to proxy exit code: %d", exitcode)
			if exitcode != 0 {
				cli.NewExitError("process returns non zero exit code", exitcode)
			}
			return nil
		}

		return nil
	},
}

func proxyStdio(h libhyperstart.Hyperstart, container, process string, wg *sync.WaitGroup) {
	// don't wait the copying of the stdin, because `io.Copy(inPipe, os.Stdin)`
	// can't terminate when no input. todo: find a better way.
	wg.Add(2)
	inPipe, outPipe, errPipe := libhyperstart.StdioPipe(h, container, process)
	go func() {
		_, err1 := io.Copy(inPipe, os.Stdin)
		err2 := h.CloseStdin(container, process)
		glog.V(3).Infof("copy stdin %#v %#v", err1, err2)
	}()

	go func() {
		_, err := io.Copy(os.Stdout, outPipe)
		glog.V(3).Infof("copy stdout %#v", err)
		wg.Done()
	}()

	go func() {
		_, err := io.Copy(os.Stderr, errPipe)
		glog.V(3).Infof("copy stderr %#v", err)
		wg.Done()
	}()
}

func forwardAllSignals(h libhyperstart.Hyperstart, container, process string) chan os.Signal {
	sigc := make(chan os.Signal, 2048)
	// handle all signals for the process.
	signal.Notify(sigc)
	signal.Ignore(syscall.SIGCHLD, syscall.SIGPIPE)

	go func() {
		for s := range sigc {
			if s == syscall.SIGCHLD || s == syscall.SIGPIPE || s == syscall.SIGWINCH {
				//ignore these
				continue
			}
			// forward this signal to container
			sysSig, ok := s.(syscall.Signal)
			if !ok {
				err := fmt.Errorf("can't forward unknown signal %q", s.String())
				fmt.Fprintf(os.Stderr, "%v", err)
				glog.Errorf("%v", err)
				continue
			}
			if err := h.SignalProcess(container, process, sysSig); err != nil {
				err = fmt.Errorf("forward signal %q failed: %v", s.String(), err)
				fmt.Fprintf(os.Stderr, "%v", err)
				glog.Errorf("%v", err)
			}
		}
	}()
	return sigc
}

func createShim(options runvOptions, container, process string, spec *specs.Process) (*os.Process, error) {
	path, err := osext.Executable()
	if err != nil {
		return nil, fmt.Errorf("cannot find self executable path for %s: %v", os.Args[0], err)
	}

	var ptymaster, tty *os.File
	if options.String("console") != "" {
		tty, err = os.OpenFile(options.String("console"), os.O_RDWR, 0)
		if err != nil {
			return nil, err
		}
	} else if options.String("console-socket") != "" {
		ptymaster, tty, err = pty.Open()
		if err != nil {
			return nil, err
		}
		if err = sendtty(options.String("console-socket"), ptymaster); err != nil {
			return nil, err
		}
		ptymaster.Close()
	}

	args := []string{"runv", "--root", options.GlobalString("root")}
	if options.GlobalString("log_dir") != "" {
		args = append(args, "--log_dir", filepath.Join(options.GlobalString("log_dir"), "shim-"+container))
	}
	if options.GlobalBool("debug") {
		args = append(args, "--debug")
	}
	args = append(args, "shim", "--container", container, "--process", process)
	args = append(args, "--proxy-stdio", "--proxy-exit-code", "--proxy-signal")
	if spec.Terminal {
		args = append(args, "--proxy-winsize")
	}

	cmd := exec.Cmd{
		Path: path,
		Args: args,
		Dir:  "/",
		SysProcAttr: &syscall.SysProcAttr{
			Setctty: tty != nil,
			Setsid:  tty != nil || !options.attach,
		},
	}
	if options.withContainer == nil {
		cmd.SysProcAttr.Cloneflags = syscall.CLONE_NEWNET
	} else {
		cmd.Env = append(os.Environ(), fmt.Sprintf("_RUNVNETNSPID=%d", options.withContainer.Pid))
	}
	if tty == nil {
		// inherit stdio/tty
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	} else {
		defer tty.Close()
		cmd.Stdin = tty
		cmd.Stdout = tty
		cmd.Stderr = tty
	}

	err = cmd.Start()
	if err != nil {
		return nil, err
	}

	if options.String("pid-file") != "" {
		err = createPidFile(options.String("pid-file"), cmd.Process.Pid)
		if err != nil {
			cmd.Process.Kill()
			return nil, err
		}
	}

	return cmd.Process, nil
}

// createPidFile creates a file with the processes pid inside it atomically
// it creates a temp file with the paths filename + '.' infront of it
// then renames the file
func createPidFile(path string, pid int) error {
	var (
		tmpDir  = filepath.Dir(path)
		tmpName = filepath.Join(tmpDir, fmt.Sprintf(".%s", filepath.Base(path)))
	)
	f, err := os.OpenFile(tmpName, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666)
	if err != nil {
		return err
	}
	_, err = fmt.Fprintf(f, "%d", pid)
	f.Close()
	if err != nil {
		return err
	}
	return os.Rename(tmpName, path)
}
<commit_msg>shim: return error when proxy-exit-code if process fails<commit_after>package main

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"sync"
	"syscall"

	"github.com/golang/glog"
	_ "github.com/hyperhq/runv/cli/nsenter"
	"github.com/hyperhq/runv/hyperstart/libhyperstart"
	"github.com/hyperhq/runv/lib/term"
	"github.com/kardianos/osext"
	"github.com/kr/pty"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/urfave/cli"
)

var shimCommand = cli.Command{
	Name:     "shim",
	Usage:    "[internal command] proxy operations(io, signal ...) to the container/process",
	HideHelp: true,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name: "container",
		},
		cli.StringFlag{
			Name: "process",
		},
		cli.BoolFlag{
			Name: "proxy-exit-code",
		},
		cli.BoolFlag{
			Name: "proxy-stdio",
		},
		cli.BoolFlag{
			Name: "proxy-signal",
		},
		cli.BoolFlag{
			Name: "proxy-winsize",
		},
	},
	Before: func(context *cli.Context) error {
		return cmdPrepare(context, false, false)
	},
	Action: func(context *cli.Context) error {
		container := context.String("container")
		process := context.String("process")

		h, err := libhyperstart.NewGrpcBasedHyperstart(filepath.Join(context.GlobalString("root"), container, "sandbox", "hyperstartgrpc.sock"))
		if err != nil {
			return cli.NewExitError(fmt.Sprintf("failed to connect to hyperstart proxy: %v", err), -1)
		}

		if process == "init" {
			waitSigUsr1 := make(chan os.Signal, 1)
			signal.Notify(waitSigUsr1, syscall.SIGUSR1)
			<-waitSigUsr1
			signal.Stop(waitSigUsr1)
		}

		if context.Bool("proxy-stdio") {
			wg := &sync.WaitGroup{}
			proxyStdio(h, container, process, wg)
			defer wg.Wait()
		}

		if context.Bool("proxy-winsize") {
			glog.V(3).Infof("using shim to proxy winsize")
			s, err := term.SetRawTerminal(os.Stdin.Fd())
			if err != nil {
				return cli.NewExitError(fmt.Sprintf("failed to set raw terminal: %v", err), -1)
			}
			defer term.RestoreTerminal(os.Stdin.Fd(), s)
			monitorTtySize(h, container, process)
		}

		if context.Bool("proxy-signal") {
			glog.V(3).Infof("using shim to proxy signal")
			sigc := forwardAllSignals(h, container, process)
			defer signal.Stop(sigc)
		}

		// wait until exit
		exitcode := h.WaitProcess(container, process)
		if context.Bool("proxy-exit-code") {
			glog.V(3).Infof("using shim to proxy exit code: %d", exitcode)
			if exitcode != 0 {
				return cli.NewExitError("process returns non zero exit code", exitcode)
			}
			return nil
		}

		return nil
	},
}

func proxyStdio(h libhyperstart.Hyperstart, container, process string, wg *sync.WaitGroup) {
	// don't wait the copying of the stdin, because `io.Copy(inPipe, os.Stdin)`
	// can't terminate when no input. todo: find a better way.
	wg.Add(2)
	inPipe, outPipe, errPipe := libhyperstart.StdioPipe(h, container, process)
	go func() {
		_, err1 := io.Copy(inPipe, os.Stdin)
		err2 := h.CloseStdin(container, process)
		glog.V(3).Infof("copy stdin %#v %#v", err1, err2)
	}()

	go func() {
		_, err := io.Copy(os.Stdout, outPipe)
		glog.V(3).Infof("copy stdout %#v", err)
		wg.Done()
	}()

	go func() {
		_, err := io.Copy(os.Stderr, errPipe)
		glog.V(3).Infof("copy stderr %#v", err)
		wg.Done()
	}()
}

func forwardAllSignals(h libhyperstart.Hyperstart, container, process string) chan os.Signal {
	sigc := make(chan os.Signal, 2048)
	// handle all signals for the process.
	signal.Notify(sigc)
	signal.Ignore(syscall.SIGCHLD, syscall.SIGPIPE)

	go func() {
		for s := range sigc {
			if s == syscall.SIGCHLD || s == syscall.SIGPIPE || s == syscall.SIGWINCH {
				//ignore these
				continue
			}
			// forward this signal to container
			sysSig, ok := s.(syscall.Signal)
			if !ok {
				err := fmt.Errorf("can't forward unknown signal %q", s.String())
				fmt.Fprintf(os.Stderr, "%v", err)
				glog.Errorf("%v", err)
				continue
			}
			if err := h.SignalProcess(container, process, sysSig); err != nil {
				err = fmt.Errorf("forward signal %q failed: %v", s.String(), err)
				fmt.Fprintf(os.Stderr, "%v", err)
				glog.Errorf("%v", err)
			}
		}
	}()
	return sigc
}

func createShim(options runvOptions, container, process string, spec *specs.Process) (*os.Process, error) {
	path, err := osext.Executable()
	if err != nil {
		return nil, fmt.Errorf("cannot find self executable path for %s: %v", os.Args[0], err)
	}

	var ptymaster, tty *os.File
	if options.String("console") != "" {
		tty, err = os.OpenFile(options.String("console"), os.O_RDWR, 0)
		if err != nil {
			return nil, err
		}
	} else if options.String("console-socket") != "" {
		ptymaster, tty, err = pty.Open()
		if err != nil {
			return nil, err
		}
		if err = sendtty(options.String("console-socket"), ptymaster); err != nil {
			return nil, err
		}
		ptymaster.Close()
	}

	args := []string{"runv", "--root", options.GlobalString("root")}
	if options.GlobalString("log_dir") != "" {
		args = append(args, "--log_dir", filepath.Join(options.GlobalString("log_dir"), "shim-"+container))
	}
	if options.GlobalBool("debug") {
		args = append(args, "--debug")
	}
	args = append(args, "shim", "--container", container, "--process", process)
	args = append(args, "--proxy-stdio", "--proxy-exit-code", "--proxy-signal")
	if spec.Terminal {
		args = append(args, "--proxy-winsize")
	}

	cmd := exec.Cmd{
		Path: path,
		Args: args,
		Dir:  "/",
		SysProcAttr: &syscall.SysProcAttr{
			Setctty: tty != nil,
			Setsid:  tty != nil || !options.attach,
		},
	}
	if options.withContainer == nil {
		cmd.SysProcAttr.Cloneflags = syscall.CLONE_NEWNET
	} else {
		cmd.Env = append(os.Environ(), fmt.Sprintf("_RUNVNETNSPID=%d", options.withContainer.Pid))
	}
	if tty == nil {
		// inherit stdio/tty
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	} else {
		defer tty.Close()
		cmd.Stdin = tty
		cmd.Stdout = tty
		cmd.Stderr = tty
	}

	err = cmd.Start()
	if err != nil {
		return nil, err
	}

	if options.String("pid-file") != "" {
		err = createPidFile(options.String("pid-file"), cmd.Process.Pid)
		if err != nil {
			cmd.Process.Kill()
			return nil, err
		}
	}

	return cmd.Process, nil
}

// createPidFile creates a file with the processes pid inside it atomically
// it creates a temp file with the paths filename + '.' infront of it
// then renames the file
func createPidFile(path string, pid int) error {
	var (
		tmpDir  = filepath.Dir(path)
		tmpName = filepath.Join(tmpDir, fmt.Sprintf(".%s", filepath.Base(path)))
	)
	f, err := os.OpenFile(tmpName, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666)
	if err != nil {
		return err
	}
	_, err = fmt.Fprintf(f, "%d", pid)
	f.Close()
	if err != nil {
		return err
	}
	return os.Rename(tmpName, path)
}
<|endoftext|>


<commit_before>// Copyright 2017 The StudyGolang Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// http://studygolang.com
// Author:polaris	polaris@studygolang.com

package logic

import (
	"model"
	"net/url"
	"strings"
	"time"

	. "db"

	"github.com/polaris1119/logger"
	"github.com/sundy-li/html2article"
)

func (self ArticleLogic) ParseArticleByAccuracy(articleUrl string) (*model.Article, error) {
	htmlArticle, err := html2article.FromUrl(articleUrl)
	if err != nil {
		logger.Errorln("html2article from url:", articleUrl, "error:", err)
		return nil, err
	}

	urlTyp, err := url.Parse(articleUrl)
	if err != nil {
		logger.Errorln("html2article parse url:", articleUrl, "error:", err)
		return nil, err
	}

	var (
		title = htmlArticle.Title
		name  = urlTyp.Hostname()
	)
	pos := strings.LastIndex(htmlArticle.Title, "-")
	if pos == -1 {
		pos = strings.LastIndex(htmlArticle.Title, "|")
	}

	if pos != -1 {
		title = strings.TrimSpace(htmlArticle.Title[:pos])
		name = strings.TrimSpace(htmlArticle.Title[pos+1:])
	}

	pubDate := time.Now().Format("2006-02-01 15:04")
	if htmlArticle.Publishtime > 0 {
		pubDate = time.Unix(htmlArticle.Publishtime, 0).UTC().Format("2006-02-01 15:04")
	}
	article := &model.Article{
		Domain:  urlTyp.Hostname(),
		Name:    name,
		Title:   title,
		Content: htmlArticle.Html,
		Txt:     htmlArticle.Content,
		PubDate: pubDate,
		Url:     articleUrl,
	}

	_, err = MasterDB.Insert(article)
	if err != nil {
		logger.Errorln("insert article error:", err)
		return nil, err
	}

	return article, nil
}
<commit_msg>网站名称当成作者<commit_after>// Copyright 2017 The StudyGolang Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// http://studygolang.com
// Author:polaris	polaris@studygolang.com

package logic

import (
	"model"
	"net/url"
	"strings"
	"time"

	. "db"

	"github.com/polaris1119/logger"
	"github.com/sundy-li/html2article"
)

func (self ArticleLogic) ParseArticleByAccuracy(articleUrl string) (*model.Article, error) {
	htmlArticle, err := html2article.FromUrl(articleUrl)
	if err != nil {
		logger.Errorln("html2article from url:", articleUrl, "error:", err)
		return nil, err
	}

	urlTyp, err := url.Parse(articleUrl)
	if err != nil {
		logger.Errorln("html2article parse url:", articleUrl, "error:", err)
		return nil, err
	}

	var (
		title = htmlArticle.Title
		name  = urlTyp.Hostname()
	)
	pos := strings.LastIndex(htmlArticle.Title, "-")
	if pos == -1 {
		pos = strings.LastIndex(htmlArticle.Title, "|")
	}

	if pos != -1 {
		title = strings.TrimSpace(htmlArticle.Title[:pos])
		name = strings.TrimSpace(htmlArticle.Title[pos+1:])
	}

	pubDate := time.Now().Format("2006-02-01 15:04")
	if htmlArticle.Publishtime > 0 {
		pubDate = time.Unix(htmlArticle.Publishtime, 0).UTC().Format("2006-02-01 15:04")
	}
	article := &model.Article{
		Domain:    urlTyp.Hostname(),
		Name:      name,
		Title:     title,
		Author:    name,
		AuthorTxt: name,
		Content:   htmlArticle.Html,
		Txt:       htmlArticle.Content,
		PubDate:   pubDate,
		Url:       articleUrl,
	}

	_, err = MasterDB.Insert(article)
	if err != nil {
		logger.Errorln("insert article error:", err)
		return nil, err
	}

	return article, nil
}
<|endoftext|>