<commit_before>package update

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"runtime"

	"github.com/blang/semver"
	"github.com/cheggaaa/pb"
	"github.com/containerum/chkit/pkg/chkitErrors"
	"github.com/sirupsen/logrus"
	"gopkg.in/resty.v1"
	"gopkg.in/urfave/cli.v2"
)

type LatestChecker interface {
	LatestVersion() (semver.Version, error)
}

type LatestDownloader interface {
	LatestDownload() (io.ReadCloser, error)
}

type LatestCheckerDownloader interface {
	LatestChecker
	LatestDownloader
}

const (
	ErrUpdateCheck    = chkitErrors.Err("unable to check latest version")
	ErrUpdateDownload = chkitErrors.Err("unable to download latest version")
)

const (
	MaxFileSize = 50 * (1 << 20) // 50 megabytes
)

func DownloadFileName(version semver.Version) string {
	extension := "tar.gz"
	if runtime.GOOS == "windows" {
		extension = "zip"
	}
	return fmt.Sprintf("chkit_%s_%s_v%s.%s", runtime.GOOS, runtime.GOARCH, version, extension)
}

type GithubLatestCheckerDownloader struct {
	client      *resty.Client
	ctx         *cli.Context
	downloadUrl string
}

func NewGithubLatestCheckerDownloader(ctx *cli.Context, owner, repo string) *GithubLatestCheckerDownloader {
	re := resty.New().
		SetHostURL(
			fmt.Sprintf("https://api.github.com/repos/%s/%s/releases",
				owner,
				repo))
	if ctx.Bool("debug-requests") {
		re.SetLogger(logrus.StandardLogger().
			WriterLevel(logrus.DebugLevel)).
			SetDebug(true)
	}
	return &GithubLatestCheckerDownloader{
		ctx:         ctx,
		client:      re,
		downloadUrl: fmt.Sprintf("https://github.com/%s/%s/releases/download", owner, repo),
	}
}

func (gh *GithubLatestCheckerDownloader) LatestVersion() (semver.Version, error) {
	logrus.Debug("get latest version from github")

	var latestVersionResp struct {
		LatestVersion string `json:"tag_name"`
	}

	_, err := gh.client.R().SetResult(&latestVersionResp).Get("/latest")
	if err != nil {
		return semver.MustParse("0.0.1-alpha"), chkitErrors.Wrap(ErrUpdateCheck, err)
	}
	vers, err := semver.ParseTolerant(latestVersionResp.LatestVersion)
	if err != nil {
		return semver.MustParse("0.0.1-alpha"), chkitErrors.Wrap(ErrUpdateCheck, err)
	}
	return vers, nil
}

func (gh *GithubLatestCheckerDownloader) LatestDownload() (io.ReadCloser, error) {
	logrus.Debug("download update")

	latestVersion, err := gh.LatestVersion()
	if err != nil {
		return nil, err
	}

	resp, err := http.Get(fmt.Sprintf("%s/%s/%s", gh.downloadUrl, latestVersion, DownloadFileName(latestVersion)))
	if err != nil {
		return nil, chkitErrors.Wrap(ErrUpdateDownload, err)
	}

	bar := pb.New64(resp.ContentLength)

	return bar.NewProxyReader(resp.Body), nil
}

type FileSystemLatestCheckerDownloader struct {
	baseDir string
	log     *logrus.Logger
	ctx     *cli.Context
}

func NewFileSystemLatestCheckerDownloader(ctx *cli.Context, baseDir string) *FileSystemLatestCheckerDownloader {
	return &FileSystemLatestCheckerDownloader{
		baseDir: baseDir,
		ctx:     ctx,
	}
}

func (fs *FileSystemLatestCheckerDownloader) LatestVersion() (semver.Version, error) {
	fs.log.Debug("get latest version from filesystem")

	verFile, err := os.Open(path.Join(fs.baseDir, "version"))
	if err != nil {
		return semver.MustParse("0.0.1-alpha"), chkitErrors.Wrap(ErrUpdateCheck, err)
	}
	defer verFile.Close()

	ver, err := ioutil.ReadAll(verFile)
	if err != nil {
		return semver.MustParse("0.0.1-alpha"), chkitErrors.Wrap(ErrUpdateCheck, err)
	}

	sv, err := semver.ParseTolerant(string(ver))
	if err != nil {
		return semver.MustParse("0.0.1-alpha"), chkitErrors.Wrap(ErrUpdateCheck, err)
	}

	return sv, nil
}

func (fs *FileSystemLatestCheckerDownloader) LatestDownload() (io.ReadCloser, error) {
	fs.log.Debug("get latest version package from filesystem")

	latestVersion, err := fs.LatestVersion()
	if err != nil {
		return nil, err
	}

	pkg, err := os.Open(path.Join(fs.baseDir, DownloadFileName(latestVersion)))
	if err != nil {
		return nil, chkitErrors.Wrap(ErrUpdateDownload, err)
	}

	return pkg, nil
}
<commit_msg>add logs<commit_after>package update

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"runtime"

	"github.com/blang/semver"
	"github.com/cheggaaa/pb"
	"github.com/containerum/chkit/pkg/chkitErrors"
	"github.com/sirupsen/logrus"
	"gopkg.in/resty.v1"
	"gopkg.in/urfave/cli.v2"
)

type LatestChecker interface {
	LatestVersion() (semver.Version, error)
}

type LatestDownloader interface {
	LatestDownload() (io.ReadCloser, error)
}

type LatestCheckerDownloader interface {
	LatestChecker
	LatestDownloader
}

const (
	ErrUpdateCheck    = chkitErrors.Err("unable to check latest version")
	ErrUpdateDownload = chkitErrors.Err("unable to download latest version")
)

const (
	MaxFileSize = 50 * (1 << 20) // 50 megabytes
)

func DownloadFileName(version semver.Version) string {
	extension := "tar.gz"
	if runtime.GOOS == "windows" {
		extension = "zip"
	}
	return fmt.Sprintf("chkit_%s_%s_v%s.%s", runtime.GOOS, runtime.GOARCH, version, extension)
}

type GithubLatestCheckerDownloader struct {
	client      *resty.Client
	ctx         *cli.Context
	downloadUrl string
}

func NewGithubLatestCheckerDownloader(ctx *cli.Context, owner, repo string) *GithubLatestCheckerDownloader {
	re := resty.New().
		SetHostURL(
			fmt.Sprintf("https://api.github.com/repos/%s/%s/releases",
				owner,
				repo))
	if ctx.Bool("debug-requests") {
		re.SetLogger(logrus.StandardLogger().
			WriterLevel(logrus.DebugLevel)).
			SetDebug(true)
	}
	return &GithubLatestCheckerDownloader{
		ctx:         ctx,
		client:      re,
		downloadUrl: fmt.Sprintf("https://github.com/%s/%s/releases/download", owner, repo),
	}
}

func (gh *GithubLatestCheckerDownloader) LatestVersion() (semver.Version, error) {
	logrus.Debug("get latest version from github")

	var latestVersionResp struct {
		LatestVersion string `json:"tag_name"`
	}

	_, err := gh.client.R().SetResult(&latestVersionResp).Get("/latest")
	if err != nil {
		logrus.WithError(err).Errorf("error while getting latest version from github")
		return semver.MustParse("0.0.1-alpha"), chkitErrors.Wrap(ErrUpdateCheck, err)
	}

	vers, err := semver.ParseTolerant(latestVersionResp.LatestVersion)
	if err != nil {
		logrus.WithError(err).Errorf("error while parsing latest version tag")
		return semver.MustParse("0.0.1-alpha"), chkitErrors.Wrap(ErrUpdateCheck, err)
	}
	return vers, nil
}

func (gh *GithubLatestCheckerDownloader) LatestDownload() (io.ReadCloser, error) {
	logrus.Debug("download update")

	latestVersion, err := gh.LatestVersion()
	if err != nil {
		return nil, err
	}

	resp, err := http.Get(fmt.Sprintf("%s/%s/%s", gh.downloadUrl, latestVersion, DownloadFileName(latestVersion)))
	if err != nil {
		return nil, chkitErrors.Wrap(ErrUpdateDownload, err)
	}

	bar := pb.New64(resp.ContentLength)

	return bar.NewProxyReader(resp.Body), nil
}

type FileSystemLatestCheckerDownloader struct {
	baseDir string
	log     *logrus.Logger
	ctx     *cli.Context
}

func NewFileSystemLatestCheckerDownloader(ctx *cli.Context, baseDir string) *FileSystemLatestCheckerDownloader {
	return &FileSystemLatestCheckerDownloader{
		baseDir: baseDir,
		ctx:     ctx,
	}
}

func (fs *FileSystemLatestCheckerDownloader) LatestVersion() (semver.Version, error) {
	fs.log.Debug("get latest version from filesystem")

	verFile, err := os.Open(path.Join(fs.baseDir, "version"))
	if err != nil {
		return semver.MustParse("0.0.1-alpha"), chkitErrors.Wrap(ErrUpdateCheck, err)
	}
	defer verFile.Close()

	ver, err := ioutil.ReadAll(verFile)
	if err != nil {
		return semver.MustParse("0.0.1-alpha"), chkitErrors.Wrap(ErrUpdateCheck, err)
	}

	sv, err := semver.ParseTolerant(string(ver))
	if err != nil {
		return semver.MustParse("0.0.1-alpha"), chkitErrors.Wrap(ErrUpdateCheck, err)
	}

	return sv, nil
}

func (fs *FileSystemLatestCheckerDownloader) LatestDownload() (io.ReadCloser, error) {
	fs.log.Debug("get latest version package from filesystem")

	latestVersion, err := fs.LatestVersion()
	if err != nil {
		return nil, err
	}

	pkg, err := os.Open(path.Join(fs.baseDir, DownloadFileName(latestVersion)))
	if err != nil {
		return nil, chkitErrors.Wrap(ErrUpdateDownload, err)
	}

	return pkg, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import (
	"bytes"
	"log"
	"sync"
	"sync/atomic"

	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/boltdb/bolt"
)

type BatchTx interface {
	Lock()
	Unlock()
	UnsafeCreateBucket(name []byte)
	UnsafePut(bucketName []byte, key []byte, value []byte)
	UnsafeRange(bucketName []byte, key, endKey []byte, limit int64) (keys [][]byte, vals [][]byte)
	UnsafeDelete(bucketName []byte, key []byte)
	Commit()
	CommitAndStop()
}

type batchTx struct {
	sync.Mutex
	tx      *bolt.Tx
	backend *backend
	pending int
}

func newBatchTx(backend *backend) *batchTx {
	tx := &batchTx{backend: backend}
	tx.Commit()
	return tx
}

func (t *batchTx) UnsafeCreateBucket(name []byte) {
	_, err := t.tx.CreateBucket(name)
	if err != nil && err != bolt.ErrBucketExists {
		log.Fatalf("storage: cannot create bucket %s (%v)", string(name), err)
	}
}

// before calling unsafePut, the caller MUST hold the lock on tx.
func (t *batchTx) UnsafePut(bucketName []byte, key []byte, value []byte) {
	bucket := t.tx.Bucket(bucketName)
	if bucket == nil {
		log.Fatalf("storage: bucket %s does not exist", string(bucketName))
	}
	if err := bucket.Put(key, value); err != nil {
		log.Fatalf("storage: cannot put key into bucket (%v)", err)
	}
	t.pending++
}

// before calling unsafeRange, the caller MUST hold the lock on tx.
func (t *batchTx) UnsafeRange(bucketName []byte, key, endKey []byte, limit int64) (keys [][]byte, vs [][]byte) {
	bucket := t.tx.Bucket(bucketName)
	if bucket == nil {
		log.Fatalf("storage: bucket %s does not exist", string(bucketName))
	}

	if len(endKey) == 0 {
		if v := bucket.Get(key); v == nil {
			return keys, vs
		} else {
			return append(keys, key), append(vs, v)
		}
	}

	c := bucket.Cursor()
	for ck, cv := c.Seek(key); ck != nil && bytes.Compare(ck, endKey) < 0; ck, cv = c.Next() {
		vs = append(vs, cv)
		keys = append(keys, ck)
		if limit > 0 && limit == int64(len(keys)) {
			break
		}
	}

	return keys, vs
}

// before calling unsafeDelete, the caller MUST hold the lock on tx.
func (t *batchTx) UnsafeDelete(bucketName []byte, key []byte) {
	bucket := t.tx.Bucket(bucketName)
	if bucket == nil {
		log.Fatalf("storage: bucket %s does not exist", string(bucketName))
	}
	err := bucket.Delete(key)
	if err != nil {
		log.Fatalf("storage: cannot delete key from bucket (%v)", err)
	}
	t.pending++
}

// Commit commits a previous tx and begins a new writable one.
func (t *batchTx) Commit() {
	t.Lock()
	defer t.Unlock()
	t.commit(false)
}

// CommitAndStop commits the previous tx and do not create a new one.
func (t *batchTx) CommitAndStop() {
	t.Lock()
	defer t.Unlock()
	t.commit(true)
}

func (t *batchTx) Unlock() {
	if t.pending >= t.backend.batchLimit {
		t.commit(false)
		t.pending = 0
	}
	t.Mutex.Unlock()
}

func (t *batchTx) commit(stop bool) {
	var err error
	// commit the last tx
	if t.tx != nil {
		if t.pending == 0 && !stop {
			return
		}
		err = t.tx.Commit()
		t.pending = 0
		if err != nil {
			log.Fatalf("storage: cannot commit tx (%s)", err)
		}
	}

	if stop {
		return
	}

	// begin a new tx
	t.tx, err = t.backend.db.Begin(true)
	if err != nil {
		log.Fatalf("storage: cannot begin tx (%s)", err)
	}
	atomic.StoreInt64(&t.backend.size, t.tx.Size())
}
<commit_msg>backend: create bucket should increase pending<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import (
	"bytes"
	"log"
	"sync"
	"sync/atomic"

	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/boltdb/bolt"
)

type BatchTx interface {
	Lock()
	Unlock()
	UnsafeCreateBucket(name []byte)
	UnsafePut(bucketName []byte, key []byte, value []byte)
	UnsafeRange(bucketName []byte, key, endKey []byte, limit int64) (keys [][]byte, vals [][]byte)
	UnsafeDelete(bucketName []byte, key []byte)
	Commit()
	CommitAndStop()
}

type batchTx struct {
	sync.Mutex
	tx      *bolt.Tx
	backend *backend
	pending int
}

func newBatchTx(backend *backend) *batchTx {
	tx := &batchTx{backend: backend}
	tx.Commit()
	return tx
}

func (t *batchTx) UnsafeCreateBucket(name []byte) {
	_, err := t.tx.CreateBucket(name)
	if err != nil && err != bolt.ErrBucketExists {
		log.Fatalf("storage: cannot create bucket %s (%v)", string(name), err)
	}
	t.pending++
}

// before calling unsafePut, the caller MUST hold the lock on tx.
func (t *batchTx) UnsafePut(bucketName []byte, key []byte, value []byte) {
	bucket := t.tx.Bucket(bucketName)
	if bucket == nil {
		log.Fatalf("storage: bucket %s does not exist", string(bucketName))
	}
	if err := bucket.Put(key, value); err != nil {
		log.Fatalf("storage: cannot put key into bucket (%v)", err)
	}
	t.pending++
}

// before calling unsafeRange, the caller MUST hold the lock on tx.
func (t *batchTx) UnsafeRange(bucketName []byte, key, endKey []byte, limit int64) (keys [][]byte, vs [][]byte) {
	bucket := t.tx.Bucket(bucketName)
	if bucket == nil {
		log.Fatalf("storage: bucket %s does not exist", string(bucketName))
	}

	if len(endKey) == 0 {
		if v := bucket.Get(key); v == nil {
			return keys, vs
		} else {
			return append(keys, key), append(vs, v)
		}
	}

	c := bucket.Cursor()
	for ck, cv := c.Seek(key); ck != nil && bytes.Compare(ck, endKey) < 0; ck, cv = c.Next() {
		vs = append(vs, cv)
		keys = append(keys, ck)
		if limit > 0 && limit == int64(len(keys)) {
			break
		}
	}

	return keys, vs
}

// before calling unsafeDelete, the caller MUST hold the lock on tx.
func (t *batchTx) UnsafeDelete(bucketName []byte, key []byte) {
	bucket := t.tx.Bucket(bucketName)
	if bucket == nil {
		log.Fatalf("storage: bucket %s does not exist", string(bucketName))
	}
	err := bucket.Delete(key)
	if err != nil {
		log.Fatalf("storage: cannot delete key from bucket (%v)", err)
	}
	t.pending++
}

// Commit commits a previous tx and begins a new writable one.
func (t *batchTx) Commit() {
	t.Lock()
	defer t.Unlock()
	t.commit(false)
}

// CommitAndStop commits the previous tx and do not create a new one.
func (t *batchTx) CommitAndStop() {
	t.Lock()
	defer t.Unlock()
	t.commit(true)
}

func (t *batchTx) Unlock() {
	if t.pending >= t.backend.batchLimit {
		t.commit(false)
		t.pending = 0
	}
	t.Mutex.Unlock()
}

func (t *batchTx) commit(stop bool) {
	var err error
	// commit the last tx
	if t.tx != nil {
		if t.pending == 0 && !stop {
			return
		}
		err = t.tx.Commit()
		t.pending = 0
		if err != nil {
			log.Fatalf("storage: cannot commit tx (%s)", err)
		}
	}

	if stop {
		return
	}

	// begin a new tx
	t.tx, err = t.backend.db.Begin(true)
	if err != nil {
		log.Fatalf("storage: cannot begin tx (%s)", err)
	}
	atomic.StoreInt64(&t.backend.size, t.tx.Size())
}
<|endoftext|>


<commit_before>// Copyright ©2013 The nlpt Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package nlptoken

// Encodings is the preferred way to build the alphabet.
// It defines a scope function for ordered unicode code points.
// See codepoints.go of list.
type Encodings interface {
	scope() (rune, rune)
}

// CodePoint is the struct for unicode.go,
// it contains the order of the range code points
type CodePoint struct {
	order []rune
}

// scope implements the Encodings interface through
// CodePoint structs defined over ordered range of
// Unicode Code Points.
func (cp CodePoint) scope() (startidx, stopidx rune) {
	return cp.order[0], cp.order[1]
}

// UnicodeSet builds a slice of runes based on unicode ranges
// for any Encodings struct that has the Unicode Code Points range
// defined.
// fmt.Println("BasicLatin:", UnicodeAlphabet(BasicLatin,Cyrllic))
func UnicodeAlphabet(sets ...Encodings) []rune {
	var uniset []rune
	for _, s := range sets {
		startidx, stopidx := s.scope()
		tmp := make([]rune, stopidx, stopidx)
		for i := startidx; i < stopidx; i++ {
			//fmt.Println(tmp[i])
			tmp[i] = rune(i)
		}
		uniset = append(uniset, tmp[startidx:]...)
	}
	return uniset
}

// TokenBase is the Interface to build() with different sets of rune Structs for tokenizing text
// It's been locked down to 2 rune slices per struct.
type TokenBase interface {
	build() ([]rune, []rune)
}

// Parametric function to group all rune Structs into one rune slice
func SimpleAlphabet(tks ...TokenBase) []rune {
	var result []rune
	//range over types, collect multiple return values, append to new slice, append final slice
	for _, tok := range tks {
		//return both rune slices
		s, t := tok.build()
		// append both rune slices and assign to new slice
		u := append(s, t...)
		//final append to result slice
		result = append(result, u...)
	}
	return result
}

// The Latin Struct for the tokenizer
// Languge Ex: English
type Latin struct {
	Lower, Upper []rune
}

// The LatinLigature Struct defines a common set extra unicode characters for the Latin alphabet
// A Ligature is the combination of two or more graphemes/characters
// NOTE: For Slavic languages using the Cyrillic Struct
// Languge Ex: French
type LatinLigature struct {
	LigatureUpper, LigatureLower []rune
}

// The LatinDiacritc Struct defines a common set extra unicode characters for the Latin alphabet
// A Diacritic is mark above, below, through, or between alphabetic characters generally signfying pronunciation
// NOTE: For Slavic languages use the Cyrillic Struct
// Languge Ex: Portuguese (Brazilian and Portugal), Spanish, French, German
type LatinDiacritic struct {
	DiacriticUpper, DiacriticLower []rune
}

// The Punctiation and Number Struct for the tokenizer
type PunctNum struct {
	Punctuation, Number []rune
}

// The Space and various Characters Struct for the tokenizer
type SpaceChar struct {
	Space, Character []rune
}

// The Tab and New Line Struct for the tokenizer
type TabLine struct {
	Tab, NewLine []rune
}

// The Greek letter Struct for the tokenizer
// Languge Ex: Greek
type Greek struct {
	Upper, Lower []rune
}

// The Cyrillic letter Struct for the tokenizer
// Languge Ex: Russian, Belarusian, Ukrainian, Rusyn, Serbian, Bulgarin, Macedonian, Chechen, and other Slavic langauges.
type Cyrillic struct {
	Upper, Lower []rune
}

// Small set of Mandarin characters and pinyin for the tokenizer
// Languge Ex: Chinese Mandarin basic ideograms and pinyin
type Mandarin struct {
	Ideogram, Pinyin []rune
}

// build() methods for TokenBase interface using pointers.

func (l *Latin) build() ([]rune, []rune) {
	l.Lower = []rune("abcdefghijklmnopqrstuvwxyz")
	l.Upper = []rune("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
	return l.Lower, l.Upper
}
func (llg *LatinLigature) build() ([]rune, []rune) {
	llg.LigatureLower = []rune("œæ")
	llg.LigatureUpper = []rune("ŒÆ")
	return llg.LigatureLower, llg.LigatureUpper
}
func (ld *LatinDiacritic) build() ([]rune, []rune) {
	ld.DiacriticLower = []rune("áâãàäçèéêíóôõöúüñÿ")
	ld.DiacriticUpper = []rune("ÁÄÂÃÀßÇÉÊÍÓÔÕÖÚÜÑŸ")
	return ld.DiacriticLower, ld.DiacriticUpper
}
func (pn *PunctNum) build() ([]rune, []rune) {
	pn.Punctuation = []rune(".,?!")
	pn.Number = []rune("0123456789")
	return pn.Punctuation, pn.Number
}
func (sc *SpaceChar) build() ([]rune, []rune) {
	sc.Space = []rune(" ")
	sc.Character = []rune("\"'`@#$-+=/*&%[]{}()")
	return sc.Character, sc.Space
}
func (tl *TabLine) build() ([]rune, []rune) {
	tl.NewLine = []rune("\n")
	tl.Tab = []rune("\t")
	return tl.Tab, tl.NewLine
}
func (g *Greek) build() ([]rune, []rune) {
	g.Lower = []rune("αβγδεζηθϑικλμνξοπϖρςστυφχψω")
	g.Upper = []rune("ΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥϒΦΧΨΩ")
	return g.Lower, g.Upper
}
func (c *Cyrillic) build() ([]rune, []rune) {
	c.Lower = []rune("аӑӓәӛӕбвгґѓғӷҕдђеѐёӗҽҿєжӂҗӝзҙӟӡѕиѝӥӣіїӀйҋјкқҟҡӄҝлӆљмӎнӊңӈҥњоӧөӫҩпҧрҏсҫтҭћќуўӳӱӯүұфхҳһцҵчӵҷӌҹџшщъыӹьҍэӭюя")
	c.Upper = []rune("АӐӒӘӚӔБВГҐЃҒӶҔДЂЕЀЁӖҼҾЄЖӁҖӜЗҘӞӠЅИЍӤӢІЇӀЙҊЈКҚҞҠӃҜЛӅЉМӍНӉҢӇҤЊОӦӨӪҨПҦРҎСҪТҬЋЌУЎӲӰӮҮҰФХҲҺЦҴЧӴҶӋҸЏШЩЪЫӸЬҌЭӬЮЯ")
	return c.Lower, c.Upper
}
<commit_msg>rename Cyrillic struct to CyrillicSimple<commit_after>// Copyright ©2013 The nlpt Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package nlptoken

// Encodings is the preferred way to build the alphabet.
// It defines a scope function for ordered unicode code points.
// See codepoints.go of list.
type Encodings interface {
	scope() (rune, rune)
}

// CodePoint is the struct for unicode.go,
// it contains the order of the range code points
type CodePoint struct {
	order []rune
}

// scope implements the Encodings interface through
// CodePoint structs defined over ordered range of
// Unicode Code Points.
func (cp CodePoint) scope() (startidx, stopidx rune) {
	return cp.order[0], cp.order[1]
}

// UnicodeSet builds a slice of runes based on unicode ranges
// for any Encodings struct that has the Unicode Code Points range
// defined.
// fmt.Println("BasicLatin:", UnicodeAlphabet(BasicLatin,Cyrllic))
func UnicodeAlphabet(sets ...Encodings) []rune {
	var uniset []rune
	for _, s := range sets {
		startidx, stopidx := s.scope()
		tmp := make([]rune, stopidx, stopidx)
		for i := startidx; i < stopidx; i++ {
			//fmt.Println(tmp[i])
			tmp[i] = rune(i)
		}
		uniset = append(uniset, tmp[startidx:]...)
	}
	return uniset
}

// TokenBase is the Interface to build() with different sets of rune
// structs for tokenizing text.
// It's been locked down to 2 rune slices per struct.
type TokenBase interface {
	build() ([]rune, []rune)
}

// Parametric function to group all rune Structs into one rune slice
func SimpleAlphabet(tks ...TokenBase) []rune {
	var result []rune
	//range over types, collect multiple return values, append to new slice, append final slice
	for _, tok := range tks {
		//return both rune slices
		s, t := tok.build()
		// append both rune slices and assign to new slice
		u := append(s, t...)
		//final append to result slice
		result = append(result, u...)
	}
	return result
}

// The Latin Struct for the tokenizer
// Languge Ex: English
type Latin struct {
	Lower, Upper []rune
}

// The LatinLigature Struct defines a common set extra unicode characters for the Latin alphabet
// A Ligature is the combination of two or more graphemes/characters
// NOTE: For Slavic languages using the Cyrillic Struct
// Languge Ex: French
type LatinLigature struct {
	LigatureUpper, LigatureLower []rune
}

// The LatinDiacritc Struct defines a common set extra unicode characters for the Latin alphabet
// A Diacritic is mark above, below, through, or between alphabetic characters generally signfying pronunciation
// NOTE: For Slavic languages use the Cyrillic Struct
// Languge Ex: Portuguese (Brazilian and Portugal), Spanish, French, German
type LatinDiacritic struct {
	DiacriticUpper, DiacriticLower []rune
}

// The Punctiation and Number Struct for the tokenizer
type PunctNum struct {
	Punctuation, Number []rune
}

// The Space and various Characters Struct for the tokenizer
type SpaceChar struct {
	Space, Character []rune
}

// The Tab and New Line Struct for the tokenizer
type TabLine struct {
	Tab, NewLine []rune
}

// The Greek letter Struct for the tokenizer
// Languge Ex: Greek
type Greek struct {
	Upper, Lower []rune
}

// The CyrillicSimple letter Struct for the tokenizer
// Languge Ex: Russian, Belarusian, Ukrainian, Rusyn, Serbian, Bulgarin, Macedonian, Chechen, and other Slavic langauges.
type CyrillicSimple struct {
	Upper, Lower []rune
}

// build() methods for TokenBase interface using pointers.

func (l *Latin) build() ([]rune, []rune) {
	l.Lower = []rune("abcdefghijklmnopqrstuvwxyz")
	l.Upper = []rune("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
	return l.Lower, l.Upper
}
func (llg *LatinLigature) build() ([]rune, []rune) {
	llg.LigatureLower = []rune("œæ")
	llg.LigatureUpper = []rune("ŒÆ")
	return llg.LigatureLower, llg.LigatureUpper
}
func (ld *LatinDiacritic) build() ([]rune, []rune) {
	ld.DiacriticLower = []rune("áâãàäçèéêíóôõöúüñÿ")
	ld.DiacriticUpper = []rune("ÁÄÂÃÀßÇÉÊÍÓÔÕÖÚÜÑŸ")
	return ld.DiacriticLower, ld.DiacriticUpper
}
func (pn *PunctNum) build() ([]rune, []rune) {
	pn.Punctuation = []rune(".,?!")
	pn.Number = []rune("0123456789")
	return pn.Punctuation, pn.Number
}
func (sc *SpaceChar) build() ([]rune, []rune) {
	sc.Space = []rune(" ")
	sc.Character = []rune("\"'`@#$-+=/*&%[]{}()")
	return sc.Character, sc.Space
}
func (tl *TabLine) build() ([]rune, []rune) {
	tl.NewLine = []rune("\n")
	tl.Tab = []rune("\t")
	return tl.Tab, tl.NewLine
}
func (g *Greek) build() ([]rune, []rune) {
	g.Lower = []rune("αβγδεζηθϑικλμνξοπϖρςστυφχψω")
	g.Upper = []rune("ΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥϒΦΧΨΩ")
	return g.Lower, g.Upper
}
func (c *CyrillicSimple) build() ([]rune, []rune) {
	c.Lower = []rune("аӑӓәӛӕбвгґѓғӷҕдђеѐёӗҽҿєжӂҗӝзҙӟӡѕиѝӥӣіїӀйҋјкқҟҡӄҝлӆљмӎнӊңӈҥњоӧөӫҩпҧрҏсҫтҭћќуўӳӱӯүұфхҳһцҵчӵҷӌҹџшщъыӹьҍэӭюя")
	c.Upper = []rune("АӐӒӘӚӔБВГҐЃҒӶҔДЂЕЀЁӖҼҾЄЖӁҖӜЗҘӞӠЅИЍӤӢІЇӀЙҊЈКҚҞҠӃҜЛӅЉМӍНӉҢӇҤЊОӦӨӪҨПҦРҎСҪТҬЋЌУЎӲӰӮҮҰФХҲҺЦҴЧӴҶӋҸЏШЩЪЫӸЬҌЭӬЮЯ")
	return c.Lower, c.Upper
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"regexp"

	"github.com/majormjr/rcon"
)

type FactorioServer struct {
	Cmd            *exec.Cmd              `json:"-"`
	Savefile       string                 `json:"savefile"`
	Latency        int                    `json:"latency"`
	BindIP         string                 `json:"bindip"`
	Port           int                    `json:"port"`
	Running        bool                   `json:"running"`
	Version        Version                `json:"fac_version"`
	BaseModVersion string                 `json:"base_mod_version"`
	StdOut         io.ReadCloser          `json:"-"`
	StdErr         io.ReadCloser          `json:"-"`
	StdIn          io.WriteCloser         `json:"-"`
	Settings       map[string]interface{} `json:"-"`
	Rcon           *rcon.RemoteConsole    `json:"-"`
	LogChan        chan []string          `json:"-"`
}

func randomPort() int {
	// Returns random port to use for rcon connection
	return rand.Intn(45000-40000) + 40000
}

func initFactorio() (f *FactorioServer, err error) {
	f = new(FactorioServer)
	f.Settings = make(map[string]interface{})

	if err = os.MkdirAll(config.FactorioConfigDir, 0755); err != nil {
		return nil, fmt.Errorf("failed to create config directory: %v", err)
	}

	settingsPath := filepath.Join(config.FactorioConfigDir, config.SettingsFile)
	var settings *os.File

	if _, err := os.Stat(settingsPath); os.IsNotExist(err) {
		// copy example settings to supplied settings file, if not exists
		log.Printf("Server settings at %s not found, copying example server settings.\n", settingsPath)

		examplePath := filepath.Join(config.FactorioDir, "data", "server-settings.example.json")

		example, err := os.Open(examplePath)
		if err != nil {
			return nil, fmt.Errorf("failed to open example server settings: %v", err)
		}
		defer example.Close()

		settings, err = os.Create(settingsPath)
		if err != nil {
			return nil, fmt.Errorf("failed to create server settings file: %v", err)
		}
		defer settings.Close()

		_, err = io.Copy(settings, example)
		if err != nil {
			return nil, fmt.Errorf("failed to copy example server settings: %v", err)
		}
	} else {
		// otherwise, open file normally
		settings, err = os.Open(settingsPath)
		if err != nil {
			return nil, fmt.Errorf("failed to open server settings file: %v", err)
		}
		defer settings.Close()
	}

	// before reading reset offset
	if _, err = settings.Seek(0, 0); err != nil {
		return nil, fmt.Errorf("error while seeking in settings file: %v", err)
	}

	if err = json.NewDecoder(settings).Decode(&f.Settings); err != nil {
		return nil, fmt.Errorf("error reading %s: %v", settingsPath, err)
	}

	log.Printf("Loaded Factorio settings from %s\n", settingsPath)

	//Load factorio version
	out, err := exec.Command(config.FactorioBinary, "--version").Output()
	if err != nil {
		log.Printf("error on loading factorio version: %s", err)
		return
	}

	reg := regexp.MustCompile("Version.*?((\\d+\\.)?(\\d+\\.)?(\\*|\\d+)+)")
	found := reg.FindStringSubmatch(string(out))
	err = f.Version.UnmarshalText([]byte(found[1]))
	if err != nil {
		log.Printf("could not parse version: %v", err)
		return
	}

	//Load baseMod version
	baseModInfoFile := filepath.Join(config.FactorioDir, "data", "base", "info.json")
	bmifBa, err := ioutil.ReadFile(baseModInfoFile)
	if err != nil {
		log.Printf("couldn't open baseMods info.json: %s", err)
		return
	}
	var modInfo ModInfo
	err = json.Unmarshal(bmifBa, &modInfo)
	if err != nil {
		log.Printf("error unmarshalling baseMods info.json to a modInfo: %s", err)
		return
	}

	f.BaseModVersion = modInfo.Version

	return
}

func (f *FactorioServer) Run() error {
	var err error

	data, err := json.MarshalIndent(f.Settings, "", "  ")
	if err != nil {
		log.Println("Failed to marshal FactorioServerSettings: ", err)
	} else {
		ioutil.WriteFile(filepath.Join(config.FactorioConfigDir, config.SettingsFile), data, 0644)
	}

	args := []string{
		"--bind", (f.BindIP),
		"--port", strconv.Itoa(f.Port),
		"--server-settings", filepath.Join(config.FactorioConfigDir, "server-settings.json"),
		"--rcon-port", strconv.Itoa(config.FactorioRconPort),
		"--rcon-password", config.FactorioRconPass,
	}

	if f.Savefile == "Load Latest" {
		args = append(args, "--start-server-load-latest")
	} else {
		args = append(args, "--start-server", filepath.Join(config.FactorioSavesDir, f.Savefile))
	}

	log.Println("Starting server with command: ", config.FactorioBinary, args)

	f.Cmd = exec.Command(config.FactorioBinary, args...)

	f.StdOut, err = f.Cmd.StdoutPipe()
	if err != nil {
		log.Printf("Error opening stdout pipe: %s", err)
		return err
	}

	f.StdIn, err = f.Cmd.StdinPipe()
	if err != nil {
		log.Printf("Error opening stdin pipe: %s", err)
		return err
	}

	f.StdErr, err = f.Cmd.StderrPipe()
	if err != nil {
		log.Printf("Error opening stderr pipe: %s", err)
		return err
	}

	go f.parseRunningCommand(f.StdOut)
	go f.parseRunningCommand(f.StdErr)

	err = f.Cmd.Start()
	if err != nil {
		log.Printf("Factorio process failed to start: %s", err)
		return err
	}
	f.Running = true

	err = f.Cmd.Wait()
	if err != nil {
		log.Printf("Factorio process exited with error: %s", err)
		f.Running = false
		return err
	}

	return nil
}

func (f *FactorioServer) parseRunningCommand(std io.ReadCloser) (err error) {
	stdScanner := bufio.NewScanner(std)
	for stdScanner.Scan() {
		log.Printf("Factorio Server: %s", stdScanner.Text())
		if err := f.writeLog(stdScanner.Text()); err != nil {
			log.Printf("Error: %s", err)
		}

		line := strings.Fields(stdScanner.Text())
		// Ensure logline slice is in bounds
		if len(line) > 1 {
			// Check if Factorio Server reports any errors if so handle it
			if line[1] == "Error" {
				err := f.checkLogError(line)
				if err != nil {
					log.Printf("Error checking Factorio Server Error: %s", err)
				}
			}
			// If rcon port opens indicated in log connect to rcon
			rconLog := "Starting RCON interface at port " + strconv.Itoa(config.FactorioRconPort)
			// check if slice index is greater than 2 to prevent panic
			if len(line) > 2 {
				// log line for opened rcon connection
				if strings.Join(line[3:], " ") == rconLog {
					log.Printf("Rcon running on Factorio Server")
					err = connectRC()
					if err != nil {
						log.Printf("Error: %s", err)
					}
				}
			}
		}
	}
	if err := stdScanner.Err(); err != nil {
		log.Printf("Error reading std buffer: %s", err)
		return err
	}
	return nil
}

func (f *FactorioServer) writeLog(logline string) error {
	logfileName := config.FactorioDir + "factorio-server-console.log"
	file, err := os.OpenFile(logfileName, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		log.Printf("Cannot open logfile for appending Factorio Server output: %s", err)
		return err
	}
	defer file.Close()

	logline = logline + "\n"

	if _, err = file.WriteString(logline); err != nil {
		log.Printf("Error appending to factorio-server-console.log: %s", err)
		return err
	}

	return nil
}

func (f *FactorioServer) checkLogError(logline []string) error {
	// TODO Handle errors generated by running Factorio Server
	log.Println(logline)

	return nil
}

func (f *FactorioServer) Stop() error {
	if runtime.GOOS == "windows" {

		// Disable our own handling of CTRL+C, so we don't close when we send it to the console.
		setCtrlHandlingIsDisabledForThisProcess(true)

		// Send CTRL+C to all processes attached to the console (ourself, and the factorio server instance)
		sendCtrlCToPid(0)
		log.Println("Sent SIGINT to Factorio process. Factorio shutting down...")

		// Somehow, the Factorio devs managed to code the game to react appropriately to CTRL+C, including
		// saving the game, but not actually exit. So, we still have to manually kill the process, and
		// for extra fun, there's no way to know when the server save has actually completed (unless we want
		// to inject filesystem logic into what should be a process-level Stop() routine), so our best option
		// is to just wait an arbitrary amount of time and hope that the save is successful in that time.
		time.Sleep(2 * time.Second)
		f.Cmd.Process.Signal(os.Kill)

		// Re-enable handling of CTRL+C after we're sure that the factrio server is shut down.
		setCtrlHandlingIsDisabledForThisProcess(false)

		f.Running = false
		return nil
	}

	err := f.Cmd.Process.Signal(os.Interrupt)
	if err != nil {
		if err.Error() == "os: process already finished" {
			f.Running = false
			return err
		}
		log.Printf("Error sending SIGINT to Factorio process: %s", err)
		return err
	}
	f.Running = false
	log.Printf("Sent SIGINT to Factorio process. Factorio shutting down...")

	err = f.Rcon.Close()
	if err != nil {
		log.Printf("Error close rcon connection: %s", err)
	}

	return nil
}

func (f *FactorioServer) Kill() error {
	if runtime.GOOS == "windows" {

		err := f.Cmd.Process.Signal(os.Kill)
		if err != nil {
			if err.Error() == "os: process already finished" {
				f.Running = false
				return err
			}
			log.Printf("Error sending SIGKILL to Factorio process: %s", err)
			return err
		}
		f.Running = false
		log.Println("Sent SIGKILL to Factorio process. Factorio forced to exit.")

		return nil
	}

	err := f.Cmd.Process.Signal(os.Kill)
	if err != nil {
		if err.Error() == "os: process already finished" {
			f.Running = false
			return err
		}
		log.Printf("Error sending SIGKILL to Factorio process: %s", err)
		return err
	}
	f.Running = false
	log.Printf("Sent SIGKILL to Factorio process. Factorio forced to exit.")

	err = f.Rcon.Close()
	if err != nil {
		log.Printf("Error close rcon connection: %s", err)
	}

	return nil
}
<commit_msg>fixed hardcoded SettingsPath<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"regexp"

	"github.com/majormjr/rcon"
)

type FactorioServer struct {
	Cmd            *exec.Cmd              `json:"-"`
	Savefile       string                 `json:"savefile"`
	Latency        int                    `json:"latency"`
	BindIP         string                 `json:"bindip"`
	Port           int                    `json:"port"`
	Running        bool                   `json:"running"`
	Version        Version                `json:"fac_version"`
	BaseModVersion string                 `json:"base_mod_version"`
	StdOut         io.ReadCloser          `json:"-"`
	StdErr         io.ReadCloser          `json:"-"`
	StdIn          io.WriteCloser         `json:"-"`
	Settings       map[string]interface{} `json:"-"`
	Rcon           *rcon.RemoteConsole    `json:"-"`
	LogChan        chan []string          `json:"-"`
}

func randomPort() int {
	// Returns random port to use for rcon connection
	return rand.Intn(45000-40000) + 40000
}

func initFactorio() (f *FactorioServer, err error) {
	f = new(FactorioServer)
	f.Settings = make(map[string]interface{})

	if err = os.MkdirAll(config.FactorioConfigDir, 0755); err != nil {
		return nil, fmt.Errorf("failed to create config directory: %v", err)
	}

	settingsPath := filepath.Join(config.FactorioConfigDir, config.SettingsFile)
	var settings *os.File

	if _, err := os.Stat(settingsPath); os.IsNotExist(err) {
		// copy example settings to supplied settings file, if not exists
		log.Printf("Server settings at %s not found, copying example server settings.\n", settingsPath)

		examplePath := filepath.Join(config.FactorioDir, "data", "server-settings.example.json")

		example, err := os.Open(examplePath)
		if err != nil {
			return nil, fmt.Errorf("failed to open example server settings: %v", err)
		}
		defer example.Close()

		settings, err = os.Create(settingsPath)
		if err != nil {
			return nil, fmt.Errorf("failed to create server settings file: %v", err)
		}
		defer settings.Close()

		_, err = io.Copy(settings, example)
		if err != nil {
			return nil, fmt.Errorf("failed to copy example server settings: %v", err)
		}
	} else {
		// otherwise, open file normally
		settings, err = os.Open(settingsPath)
		if err != nil {
			return nil, fmt.Errorf("failed to open server settings file: %v", err)
		}
		defer settings.Close()
	}

	// before reading reset offset
	if _, err = settings.Seek(0, 0); err != nil {
		return nil, fmt.Errorf("error while seeking in settings file: %v", err)
	}

	if err = json.NewDecoder(settings).Decode(&f.Settings); err != nil {
		return nil, fmt.Errorf("error reading %s: %v", settingsPath, err)
	}

	log.Printf("Loaded Factorio settings from %s\n", settingsPath)

	//Load factorio version
	out, err := exec.Command(config.FactorioBinary, "--version").Output()
	if err != nil {
		log.Printf("error on loading factorio version: %s", err)
		return
	}

	reg := regexp.MustCompile("Version.*?((\\d+\\.)?(\\d+\\.)?(\\*|\\d+)+)")
	found := reg.FindStringSubmatch(string(out))
	err = f.Version.UnmarshalText([]byte(found[1]))
	if err != nil {
		log.Printf("could not parse version: %v", err)
		return
	}

	//Load baseMod version
	baseModInfoFile := filepath.Join(config.FactorioDir, "data", "base", "info.json")
	bmifBa, err := ioutil.ReadFile(baseModInfoFile)
	if err != nil {
		log.Printf("couldn't open baseMods info.json: %s", err)
		return
	}
	var modInfo ModInfo
	err = json.Unmarshal(bmifBa, &modInfo)
	if err != nil {
		log.Printf("error unmarshalling baseMods info.json to a modInfo: %s", err)
		return
	}

	f.BaseModVersion = modInfo.Version

	return
}

func (f *FactorioServer) Run() error {
	var err error

	data, err := json.MarshalIndent(f.Settings, "", "  ")
	if err != nil {
		log.Println("Failed to marshal FactorioServerSettings: ", err)
	} else {
		ioutil.WriteFile(filepath.Join(config.FactorioConfigDir, config.SettingsFile), data, 0644)
	}

	args := []string{
		"--bind", (f.BindIP),
		"--port", strconv.Itoa(f.Port),
		"--server-settings", filepath.Join(config.FactorioConfigDir, config.SettingsFile),
		"--rcon-port", strconv.Itoa(config.FactorioRconPort),
		"--rcon-password", config.FactorioRconPass,
	}

	if f.Savefile == "Load Latest" {
		args = append(args, "--start-server-load-latest")
	} else {
		args = append(args, "--start-server", filepath.Join(config.FactorioSavesDir, f.Savefile))
	}

	log.Println("Starting server with command: ", config.FactorioBinary, args)

	f.Cmd = exec.Command(config.FactorioBinary, args...)

	f.StdOut, err = f.Cmd.StdoutPipe()
	if err != nil {
		log.Printf("Error opening stdout pipe: %s", err)
		return err
	}

	f.StdIn, err = f.Cmd.StdinPipe()
	if err != nil {
		log.Printf("Error opening stdin pipe: %s", err)
		return err
	}

	f.StdErr, err = f.Cmd.StderrPipe()
	if err != nil {
		log.Printf("Error opening stderr pipe: %s", err)
		return err
	}

	go f.parseRunningCommand(f.StdOut)
	go f.parseRunningCommand(f.StdErr)

	err = f.Cmd.Start()
	if err != nil {
		log.Printf("Factorio process failed to start: %s", err)
		return err
	}
	f.Running = true

	err = f.Cmd.Wait()
	if err != nil {
		log.Printf("Factorio process exited with error: %s", err)
		f.Running = false
		return err
	}

	return nil
}

func (f *FactorioServer) parseRunningCommand(std io.ReadCloser) (err error) {
	stdScanner := bufio.NewScanner(std)
	for stdScanner.Scan() {
		log.Printf("Factorio Server: %s", stdScanner.Text())
		if err := f.writeLog(stdScanner.Text()); err != nil {
			log.Printf("Error: %s", err)
		}

		line := strings.Fields(stdScanner.Text())
		// Ensure logline slice is in bounds
		if len(line) > 1 {
			// Check if Factorio Server reports any errors if so handle it
			if line[1] == "Error" {
				err := f.checkLogError(line)
				if err != nil {
					log.Printf("Error checking Factorio Server Error: %s", err)
				}
			}
			// If rcon port opens indicated in log connect to rcon
			rconLog := "Starting RCON interface at port " + strconv.Itoa(config.FactorioRconPort)
			// check if slice index is greater than 2 to prevent panic
			if len(line) > 2 {
				// log line for opened rcon connection
				if strings.Join(line[3:], " ") == rconLog {
					log.Printf("Rcon running on Factorio Server")
					err = connectRC()
					if err != nil {
						log.Printf("Error: %s", err)
					}
				}
			}
		}
	}
	if err := stdScanner.Err(); err != nil {
		log.Printf("Error reading std buffer: %s", err)
		return err
	}
	return nil
}

func (f *FactorioServer) writeLog(logline string) error {
	logfileName := config.FactorioDir + "factorio-server-console.log"
	file, err := os.OpenFile(logfileName, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		log.Printf("Cannot open logfile for appending Factorio Server output: %s", err)
		return err
	}
	defer file.Close()

	logline = logline + "\n"

	if _, err = file.WriteString(logline); err != nil {
		log.Printf("Error appending to factorio-server-console.log: %s", err)
		return err
	}

	return nil
}

func (f *FactorioServer) checkLogError(logline []string) error {
	// TODO Handle errors generated by running Factorio Server
	log.Println(logline)

	return nil
}

func (f *FactorioServer) Stop() error {
	if runtime.GOOS == "windows" {

		// Disable our own handling of CTRL+C, so we don't close when we send it to the console.
		setCtrlHandlingIsDisabledForThisProcess(true)

		// Send CTRL+C to all processes attached to the console (ourself, and the factorio server instance)
		sendCtrlCToPid(0)
		log.Println("Sent SIGINT to Factorio process. Factorio shutting down...")

		// Somehow, the Factorio devs managed to code the game to react appropriately to CTRL+C, including
		// saving the game, but not actually exit. So, we still have to manually kill the process, and
		// for extra fun, there's no way to know when the server save has actually completed (unless we want
		// to inject filesystem logic into what should be a process-level Stop() routine), so our best option
		// is to just wait an arbitrary amount of time and hope that the save is successful in that time.
		time.Sleep(2 * time.Second)
		f.Cmd.Process.Signal(os.Kill)

		// Re-enable handling of CTRL+C after we're sure that the factrio server is shut down.
		setCtrlHandlingIsDisabledForThisProcess(false)

		f.Running = false
		return nil
	}

	err := f.Cmd.Process.Signal(os.Interrupt)
	if err != nil {
		if err.Error() == "os: process already finished" {
			f.Running = false
			return err
		}
		log.Printf("Error sending SIGINT to Factorio process: %s", err)
		return err
	}
	f.Running = false
	log.Printf("Sent SIGINT to Factorio process. Factorio shutting down...")

	err = f.Rcon.Close()
	if err != nil {
		log.Printf("Error close rcon connection: %s", err)
	}

	return nil
}

func (f *FactorioServer) Kill() error {
	if runtime.GOOS == "windows" {

		err := f.Cmd.Process.Signal(os.Kill)
		if err != nil {
			if err.Error() == "os: process already finished" {
				f.Running = false
				return err
			}
			log.Printf("Error sending SIGKILL to Factorio process: %s", err)
			return err
		}
		f.Running = false
		log.Println("Sent SIGKILL to Factorio process. Factorio forced to exit.")

		return nil
	}

	err := f.Cmd.Process.Signal(os.Kill)
	if err != nil {
		if err.Error() == "os: process already finished" {
			f.Running = false
			return err
		}
		log.Printf("Error sending SIGKILL to Factorio process: %s", err)
		return err
	}
	f.Running = false
	log.Printf("Sent SIGKILL to Factorio process. Factorio forced to exit.")

	err = f.Rcon.Close()
	if err != nil {
		log.Printf("Error close rcon connection: %s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

type gatekeeper struct {
	objects map[string]struct {
		value       string
		owner       string
		permissions map[string]bool
	}

	owners map[string]bool
}

func (g *gatekeeper) Get(item, key string) (value string, err error) {
	err = errors.New("No Such Item or Permission Denied")

	o, ok := g.objects[item]
	if !ok {
		return
	}

	ok = o.permissions[key]
	if !ok {
		return
	}

	return o.value, nil
}

func (g *gatekeeper) Set(item, value, key string) (err error) {
	err = errors.New("Permission Denied")

	ok := g.owners[key]
	if !ok {
		return
	}

	v, found := g.objects[item]

	if found && v.owner != key {
		return
	}

	if !found {
		v.owner = key
		v.permissions[key] = true
	}

	v.value = value
	g.objects[item] = v
	return nil
}

func (g *gatekeeper) AddAccess(item, key, newkey string) (err error) {
	err = errors.New("Permission Denied")

	ok := g.owners[key]
	if !ok {
		return
	}

	v, found := g.objects[item]

	if found && v.owner != key {
		return
	}

	v.permissions[newkey] = true
	g.objects[item] = v
	return nil
}

func (g *gatekeeper) RemoveAccess(item, key, newkey string) (err error) {
	err = errors.New("Permission Denied")

	ok := g.owners[key]
	if !ok {
		return
	}

	v, found := g.objects[item]

	if found && v.owner != key {
		return
	}
	v.permissions[newkey] = false
	g.objects[item] = v
	return nil
}

func (g *gatekeeper) object(w http.ResponseWriter, r *http.Request) {
	//Extract information from query
	key := r.FormValue("key")
	item := r.URL.Path
	s := strings.Split(item, "/")
	item = s[len(s)]

	var err error
	var v string

	switch r.Method {

	case "GET":
		v, err = g.Get(item, key)

	case "PUT":
	case "POST":
		value, err := ioutil.ReadAll(r.Body)
		if err == nil {
			err = g.Set(item, string(value), key)
		}
	}

	w.Header().Set("Content-Type", "text/plain; chaset=utf-8")
	if err != nil {
		w.WriteHeader(400)
		io.WriteString(w, err.Error())
		return
	}

	w.WriteHeader(200)
	io.WriteString(w, v)
}

func main() {

	g := new(gatekeeper)

	http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
		io.WriteString(w, "gatekeeper v0")
	})

	http.HandleFunc("/object/", g.object)
	http.HandleFunc("/permissions/", g.object)

	log.Fatal(http.ListenAndServe(":800", nil))
}
<commit_msg>gatekeeper: Actually route permissions<commit_after>package main

import (
	"errors"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

type gatekeeper struct {
	objects map[string]struct {
		value       string
		owner       string
		permissions map[string]bool
	}

	owners map[string]bool
}

func (g *gatekeeper) Get(item, key string) (value string, err error) {
	err = errors.New("No Such Item or Permission Denied")

	o, ok := g.objects[item]
	if !ok {
		return
	}

	ok = o.permissions[key]
	if !ok {
		return
	}

	return o.value, nil
}

func (g *gatekeeper) New(item, value, key string) (err error) {
	err = errors.New("Permission Denied")

	ok := g.owners[key]
	if !ok {
		return
	}

	v, found := g.objects[item]

	if found {
		return
	}

    v.owner = key
    v.permissions[key] = true
	v.value = value
	g.objects[item] = v
	return nil
}

func (g *gatekeeper) Set(item, value, key string) (err error) {
	err = errors.New("Permission Denied")

	ok := g.owners[key]
	if !ok {
		return
	}

	v, found := g.objects[item]

	if found && v.owner != key {
		return
	}

	if !found {
        return
	}

	v.value = value
	g.objects[item] = v
	return nil
}

func (g *gatekeeper) AddAccess(item, key, newkey string) (err error) {
	err = errors.New("Permission Denied")

	ok := g.owners[key]
	if !ok {
		return
	}

	v, found := g.objects[item]

	if found && v.owner != key {
		return
	}

	v.permissions[newkey] = true
	g.objects[item] = v
	return nil
}

func (g *gatekeeper) RemoveAccess(item, key, newkey string) (err error) {
	err = errors.New("Permission Denied")

	ok := g.owners[key]
	if !ok {
		return
	}

	v, found := g.objects[item]

	if found && v.owner != key {
		return
	}
	v.permissions[newkey] = false
	g.objects[item] = v
	return nil
}

func (g *gatekeeper) object(w http.ResponseWriter, r *http.Request) {
	key := r.FormValue("key")
	item := r.URL.Path
	s := strings.Split(item, "/")
	item = s[len(s)]

	var err error
	var v string

    value, err := ioutil.ReadAll(http.MaxBytesReader(w, r.Body, 1000000))

	switch r.Method {

	case "GET":
		v, err = g.Get(item, key)

	case "PUT":
        if err == nil {
            err = g.New(item, string(value), key)
        }

	case "POST":
		if err == nil {
			err = g.Set(item, string(value), key)
		}
	}

	w.Header().Set("Content-Type", "text/plain; chaset=utf-8")
	if err != nil {
		w.WriteHeader(400)
		io.WriteString(w, err.Error())
		return
	}

	w.WriteHeader(200)
	io.WriteString(w, v)
}

func (g *gatekeeper) permission(w http.ResponseWriter, r *http.Request) {
	key := r.FormValue("key")
	item := r.URL.Path
	s := strings.Split(item, "/")
	item = s[len(s)]

	var err error
	var v string

    value, err := ioutil.ReadAll(http.MaxBytesReader(w, r.Body, 1000000))

	switch r.Method {

	case "POST":
		if err == nil {
			err = g.AddAccess(item, key, string(value))
		}

    case "DELETE":
        if err == nil {
            err = g.RemoveAccess(item, key, string(value))
        }
    }

	w.Header().Set("Content-Type", "text/plain; chaset=utf-8")
	if err != nil {
		w.WriteHeader(400)
		io.WriteString(w, err.Error())
		return
	}

	w.WriteHeader(200)
	io.WriteString(w, v)
}

func main() {

	g := new(gatekeeper)

	http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
		io.WriteString(w, "gatekeeper v0")
	})

	http.HandleFunc("/object/", g.object)
	http.HandleFunc("/permissions/", g.object)

	log.Fatal(http.ListenAndServe(":800", nil))
}
<|endoftext|>


<commit_before>//
// config  - application specification and CLI parsing
//
// @author darryl.west <darryl.west@raincitysoftware.com>
// @created 2017-06-26 17:56:46

package geozipdb

import (
	"flag"
	"os"
	"path"
)

// Config the config structure
type Config struct {
	Port         int
	PrimaryRoute string
}

// NewDefaultConfig default settings
func NewDefaultConfig() *Config {
	cfg := new(Config)

	cfg.Port = 5000
	cfg.PrimaryRoute = "/v1/zipdb"

	return cfg
}

// ParseArgs parse the command line args
func ParseArgs() *Config {
	dflt := NewDefaultConfig()

	vers := flag.Bool("version", false, "show the version and exit")

	port := flag.Int("port", dflt.Port, "set the server's port number (e.g., 29444)...")
	route := flag.String("route", dflt.PrimaryRoute, "set the server's primary route (e.g., /v1/zipdb)...")
	level := flag.Int("loglevel", 2, "set the server's log level 0..5 for trace..error, default info=2")

	flag.Parse()

	log.Info("%s Version: %s\n", path.Base(os.Args[0]), Version())

	if *vers == true {
		os.Exit(0)
	}

	cfg := new(Config)

	cfg.Port = *port
	cfg.PrimaryRoute = *route

	log.SetLevel(*level)

	return cfg
}

// IsProduction return true if the current env is production
func IsProduction(env string) bool {
	return true
}
<commit_msg>fixed show version<commit_after>//
// config  - application specification and CLI parsing
//
// @author darryl.west <darryl.west@raincitysoftware.com>
// @created 2017-06-26 17:56:46

package geozipdb

import (
	"flag"
	"fmt"
	"os"
	"path"
)

// Config the config structure
type Config struct {
	Port         int
	PrimaryRoute string
}

// NewDefaultConfig default settings
func NewDefaultConfig() *Config {
	cfg := new(Config)

	cfg.Port = 5000
	cfg.PrimaryRoute = "/v1/zipdb"

	return cfg
}

// ParseArgs parse the command line args
func ParseArgs() *Config {
	dflt := NewDefaultConfig()

	vers := flag.Bool("version", false, "show the version and exit")

	port := flag.Int("port", dflt.Port, "set the server's port number (e.g., 29444)...")
	route := flag.String("route", dflt.PrimaryRoute, "set the server's primary route (e.g., /v1/zipdb)...")
	level := flag.Int("loglevel", 2, "set the server's log level 0..5 for trace..error, default info=2")

	flag.Parse()

	log.Info("%s Version: %s\n", path.Base(os.Args[0]), Version())

	if *vers == true {
		fmt.Printf("Version %s\n", Version())
		os.Exit(0)
	}

	cfg := new(Config)

	cfg.Port = *port
	cfg.PrimaryRoute = *route

	log.SetLevel(*level)

	return cfg
}

// IsProduction return true if the current env is production
func IsProduction(env string) bool {
	return true
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// Build this command explicitly: go build gotype.go

/*
The gotype command, like the front-end of a Go compiler, parses and
type-checks a single Go package. Errors are reported if the analysis
fails; otherwise gotype is quiet (unless -v is set).

Without a list of paths, gotype reads from standard input, which
must provide a single Go source file defining a complete package.

With a single directory argument, gotype checks the Go files in
that directory, comprising a single package. Use -t to include the
(in-package) _test.go files. Use -x to type check only external
test files.

Otherwise, each path must be the filename of a Go file belonging
to the same package.

Imports are processed by importing directly from the source of
imported packages (default), or by importing from compiled and
installed packages (by setting -c to the respective compiler).

The -c flag must be set to a compiler ("gc", "gccgo") when type-
checking packages containing imports with relative import paths
(import "./mypkg") because the source importer cannot know which
files to include for such packages.

Usage:
	gotype [flags] [path...]

The flags are:
	-t
		include local test files in a directory (ignored if -x is provided)
	-x
		consider only external test files in a directory
	-e
		report all errors (not just the first 10)
	-v
		verbose mode
	-c
		compiler used for installed packages (gc, gccgo, or source); default: source

Flags controlling additional output:
	-ast
		print AST (forces -seq)
	-trace
		print parse trace (forces -seq)
	-comments
		parse comments (ignored unless -ast or -trace is provided)

Examples:

To check the files a.go, b.go, and c.go:

	gotype a.go b.go c.go

To check an entire package including (in-package) tests in the directory dir and print the processed files:

	gotype -t -v dir

To check the external test package (if any) in the current directory, based on installed packages compiled with
cmd/compile:

	gotype -c=gc -x .

To verify the output of a pipe:

	echo "package foo" | gotype

*/
package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/importer"
	"go/parser"
	"go/scanner"
	"go/token"
	"go/types"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
	"time"
)

var (
	// main operation modes
	testFiles  = flag.Bool("t", false, "include in-package test files in a directory")
	xtestFiles = flag.Bool("x", false, "consider only external test files in a directory")
	allErrors  = flag.Bool("e", false, "report all errors, not just the first 10")
	verbose    = flag.Bool("v", false, "verbose mode")
	compiler   = flag.String("c", "source", "compiler used for installed packages (gc, gccgo, or source)")

	// additional output control
	printAST      = flag.Bool("ast", false, "print AST (forces -seq)")
	printTrace    = flag.Bool("trace", false, "print parse trace (forces -seq)")
	parseComments = flag.Bool("comments", false, "parse comments (ignored unless -ast or -trace is provided)")
)

var (
	fset       = token.NewFileSet()
	errorCount = 0
	sequential = false
	parserMode parser.Mode
)

func initParserMode() {
	if *allErrors {
		parserMode |= parser.AllErrors
	}
	if *printAST {
		sequential = true
	}
	if *printTrace {
		parserMode |= parser.Trace
		sequential = true
	}
	if *parseComments && (*printAST || *printTrace) {
		parserMode |= parser.ParseComments
	}
}

const usageString = `usage: gotype [flags] [path ...]

The gotype command, like the front-end of a Go compiler, parses and
type-checks a single Go package. Errors are reported if the analysis
fails; otherwise gotype is quiet (unless -v is set).

Without a list of paths, gotype reads from standard input, which
must provide a single Go source file defining a complete package.

With a single directory argument, gotype checks the Go files in
that directory, comprising a single package. Use -t to include the
(in-package) _test.go files. Use -x to type check only external
test files.

Otherwise, each path must be the filename of a Go file belonging
to the same package.

Imports are processed by importing directly from the source of
imported packages (default), or by importing from compiled and
installed packages (by setting -c to the respective compiler).

The -c flag must be set to a compiler ("gc", "gccgo") when type-
checking packages containing imports with relative import paths
(import "./mypkg") because the source importer cannot know which
files to include for such packages.
`

func usage() {
	fmt.Fprintln(os.Stderr, usageString)
	flag.PrintDefaults()
	os.Exit(2)
}

func report(err error) {
	scanner.PrintError(os.Stderr, err)
	if list, ok := err.(scanner.ErrorList); ok {
		errorCount += len(list)
		return
	}
	errorCount++
}

// parse may be called concurrently
func parse(filename string, src interface{}) (*ast.File, error) {
	if *verbose {
		fmt.Println(filename)
	}
	file, err := parser.ParseFile(fset, filename, src, parserMode) // ok to access fset concurrently
	if *printAST {
		ast.Print(fset, file)
	}
	return file, err
}

func parseStdin() (*ast.File, error) {
	src, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		return nil, err
	}
	return parse("<standard input>", src)
}

func parseFiles(dir string, filenames []string) ([]*ast.File, error) {
	files := make([]*ast.File, len(filenames))
	errors := make([]error, len(filenames))

	var wg sync.WaitGroup
	for i, filename := range filenames {
		wg.Add(1)
		go func(i int, filepath string) {
			defer wg.Done()
			files[i], errors[i] = parse(filepath, nil)
		}(i, filepath.Join(dir, filename))
		if sequential {
			wg.Wait()
		}
	}
	wg.Wait()

	// If there are errors, return the first one for deterministic results.
	var first error
	for _, err := range errors {
		if err != nil {
			first = err
			// If we have an error, some files may be nil.
			// Remove them. (The go/parser always returns
			// a possibly partial AST even in the presence
			// of errors, except if the file doesn't exist
			// in the first place, in which case it cannot
			// matter.)
			i := 0
			for _, f := range files {
				if f != nil {
					files[i] = f
					i++
				}
			}
			files = files[:i]
			break
		}
	}

	return files, first
}

func parseDir(dir string) ([]*ast.File, error) {
	ctxt := build.Default
	pkginfo, err := ctxt.ImportDir(dir, 0)
	if _, nogo := err.(*build.NoGoError); err != nil && !nogo {
		return nil, err
	}

	if *xtestFiles {
		return parseFiles(dir, pkginfo.XTestGoFiles)
	}

	filenames := append(pkginfo.GoFiles, pkginfo.CgoFiles...)
	if *testFiles {
		filenames = append(filenames, pkginfo.TestGoFiles...)
	}
	return parseFiles(dir, filenames)
}

func getPkgFiles(args []string) ([]*ast.File, error) {
	if len(args) == 0 {
		// stdin
		file, err := parseStdin()
		if err != nil {
			return nil, err
		}
		return []*ast.File{file}, nil
	}

	if len(args) == 1 {
		// possibly a directory
		path := args[0]
		info, err := os.Stat(path)
		if err != nil {
			return nil, err
		}
		if info.IsDir() {
			return parseDir(path)
		}
	}

	// list of files
	return parseFiles("", args)
}

func checkPkgFiles(files []*ast.File) {
	type bailout struct{}

	// if checkPkgFiles is called multiple times, set up conf only once
	conf := types.Config{
		FakeImportC: true,
		Error: func(err error) {
			if !*allErrors && errorCount >= 10 {
				panic(bailout{})
			}
			report(err)
		},
		Importer: importer.For(*compiler, nil),
		Sizes:    types.SizesFor(build.Default.Compiler, build.Default.GOARCH),
	}

	defer func() {
		switch p := recover().(type) {
		case nil, bailout:
			// normal return or early exit
		default:
			// re-panic
			panic(p)
		}
	}()

	const path = "pkg" // any non-empty string will do for now
	conf.Check(path, fset, files, nil)
}

func printStats(d time.Duration) {
	fileCount := 0
	lineCount := 0
	fset.Iterate(func(f *token.File) bool {
		fileCount++
		lineCount += f.LineCount()
		return true
	})

	fmt.Printf(
		"%s (%d files, %d lines, %d lines/s)\n",
		d, fileCount, lineCount, int64(float64(lineCount)/d.Seconds()),
	)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	initParserMode()

	start := time.Now()

	files, err := getPkgFiles(flag.Args())
	if err != nil {
		report(err)
		// ok to continue (files may be empty, but not nil)
	}

	checkPkgFiles(files)
	if errorCount > 0 {
		os.Exit(2)
	}

	if *verbose {
		printStats(time.Since(start))
	}
}
<commit_msg>go/types: add -panic flag to gotype command for debugging<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// Build this command explicitly: go build gotype.go

/*
The gotype command, like the front-end of a Go compiler, parses and
type-checks a single Go package. Errors are reported if the analysis
fails; otherwise gotype is quiet (unless -v is set).

Without a list of paths, gotype reads from standard input, which
must provide a single Go source file defining a complete package.

With a single directory argument, gotype checks the Go files in
that directory, comprising a single package. Use -t to include the
(in-package) _test.go files. Use -x to type check only external
test files.

Otherwise, each path must be the filename of a Go file belonging
to the same package.

Imports are processed by importing directly from the source of
imported packages (default), or by importing from compiled and
installed packages (by setting -c to the respective compiler).

The -c flag must be set to a compiler ("gc", "gccgo") when type-
checking packages containing imports with relative import paths
(import "./mypkg") because the source importer cannot know which
files to include for such packages.

Usage:
	gotype [flags] [path...]

The flags are:
	-t
		include local test files in a directory (ignored if -x is provided)
	-x
		consider only external test files in a directory
	-e
		report all errors (not just the first 10)
	-v
		verbose mode
	-c
		compiler used for installed packages (gc, gccgo, or source); default: source

Flags controlling additional output:
	-ast
		print AST (forces -seq)
	-trace
		print parse trace (forces -seq)
	-comments
		parse comments (ignored unless -ast or -trace is provided)
	-panic
		panic on first error

Examples:

To check the files a.go, b.go, and c.go:

	gotype a.go b.go c.go

To check an entire package including (in-package) tests in the directory dir and print the processed files:

	gotype -t -v dir

To check the external test package (if any) in the current directory, based on installed packages compiled with
cmd/compile:

	gotype -c=gc -x .

To verify the output of a pipe:

	echo "package foo" | gotype

*/
package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/importer"
	"go/parser"
	"go/scanner"
	"go/token"
	"go/types"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
	"time"
)

var (
	// main operation modes
	testFiles  = flag.Bool("t", false, "include in-package test files in a directory")
	xtestFiles = flag.Bool("x", false, "consider only external test files in a directory")
	allErrors  = flag.Bool("e", false, "report all errors, not just the first 10")
	verbose    = flag.Bool("v", false, "verbose mode")
	compiler   = flag.String("c", "source", "compiler used for installed packages (gc, gccgo, or source)")

	// additional output control
	printAST      = flag.Bool("ast", false, "print AST (forces -seq)")
	printTrace    = flag.Bool("trace", false, "print parse trace (forces -seq)")
	parseComments = flag.Bool("comments", false, "parse comments (ignored unless -ast or -trace is provided)")
	panicOnError  = flag.Bool("panic", false, "panic on first error")
)

var (
	fset       = token.NewFileSet()
	errorCount = 0
	sequential = false
	parserMode parser.Mode
)

func initParserMode() {
	if *allErrors {
		parserMode |= parser.AllErrors
	}
	if *printAST {
		sequential = true
	}
	if *printTrace {
		parserMode |= parser.Trace
		sequential = true
	}
	if *parseComments && (*printAST || *printTrace) {
		parserMode |= parser.ParseComments
	}
}

const usageString = `usage: gotype [flags] [path ...]

The gotype command, like the front-end of a Go compiler, parses and
type-checks a single Go package. Errors are reported if the analysis
fails; otherwise gotype is quiet (unless -v is set).

Without a list of paths, gotype reads from standard input, which
must provide a single Go source file defining a complete package.

With a single directory argument, gotype checks the Go files in
that directory, comprising a single package. Use -t to include the
(in-package) _test.go files. Use -x to type check only external
test files.

Otherwise, each path must be the filename of a Go file belonging
to the same package.

Imports are processed by importing directly from the source of
imported packages (default), or by importing from compiled and
installed packages (by setting -c to the respective compiler).

The -c flag must be set to a compiler ("gc", "gccgo") when type-
checking packages containing imports with relative import paths
(import "./mypkg") because the source importer cannot know which
files to include for such packages.
`

func usage() {
	fmt.Fprintln(os.Stderr, usageString)
	flag.PrintDefaults()
	os.Exit(2)
}

func report(err error) {
	if *panicOnError {
		panic(err)
	}
	scanner.PrintError(os.Stderr, err)
	if list, ok := err.(scanner.ErrorList); ok {
		errorCount += len(list)
		return
	}
	errorCount++
}

// parse may be called concurrently
func parse(filename string, src interface{}) (*ast.File, error) {
	if *verbose {
		fmt.Println(filename)
	}
	file, err := parser.ParseFile(fset, filename, src, parserMode) // ok to access fset concurrently
	if *printAST {
		ast.Print(fset, file)
	}
	return file, err
}

func parseStdin() (*ast.File, error) {
	src, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		return nil, err
	}
	return parse("<standard input>", src)
}

func parseFiles(dir string, filenames []string) ([]*ast.File, error) {
	files := make([]*ast.File, len(filenames))
	errors := make([]error, len(filenames))

	var wg sync.WaitGroup
	for i, filename := range filenames {
		wg.Add(1)
		go func(i int, filepath string) {
			defer wg.Done()
			files[i], errors[i] = parse(filepath, nil)
		}(i, filepath.Join(dir, filename))
		if sequential {
			wg.Wait()
		}
	}
	wg.Wait()

	// If there are errors, return the first one for deterministic results.
	var first error
	for _, err := range errors {
		if err != nil {
			first = err
			// If we have an error, some files may be nil.
			// Remove them. (The go/parser always returns
			// a possibly partial AST even in the presence
			// of errors, except if the file doesn't exist
			// in the first place, in which case it cannot
			// matter.)
			i := 0
			for _, f := range files {
				if f != nil {
					files[i] = f
					i++
				}
			}
			files = files[:i]
			break
		}
	}

	return files, first
}

func parseDir(dir string) ([]*ast.File, error) {
	ctxt := build.Default
	pkginfo, err := ctxt.ImportDir(dir, 0)
	if _, nogo := err.(*build.NoGoError); err != nil && !nogo {
		return nil, err
	}

	if *xtestFiles {
		return parseFiles(dir, pkginfo.XTestGoFiles)
	}

	filenames := append(pkginfo.GoFiles, pkginfo.CgoFiles...)
	if *testFiles {
		filenames = append(filenames, pkginfo.TestGoFiles...)
	}
	return parseFiles(dir, filenames)
}

func getPkgFiles(args []string) ([]*ast.File, error) {
	if len(args) == 0 {
		// stdin
		file, err := parseStdin()
		if err != nil {
			return nil, err
		}
		return []*ast.File{file}, nil
	}

	if len(args) == 1 {
		// possibly a directory
		path := args[0]
		info, err := os.Stat(path)
		if err != nil {
			return nil, err
		}
		if info.IsDir() {
			return parseDir(path)
		}
	}

	// list of files
	return parseFiles("", args)
}

func checkPkgFiles(files []*ast.File) {
	type bailout struct{}

	// if checkPkgFiles is called multiple times, set up conf only once
	conf := types.Config{
		FakeImportC: true,
		Error: func(err error) {
			if !*allErrors && errorCount >= 10 {
				panic(bailout{})
			}
			report(err)
		},
		Importer: importer.For(*compiler, nil),
		Sizes:    types.SizesFor(build.Default.Compiler, build.Default.GOARCH),
	}

	defer func() {
		switch p := recover().(type) {
		case nil, bailout:
			// normal return or early exit
		default:
			// re-panic
			panic(p)
		}
	}()

	const path = "pkg" // any non-empty string will do for now
	conf.Check(path, fset, files, nil)
}

func printStats(d time.Duration) {
	fileCount := 0
	lineCount := 0
	fset.Iterate(func(f *token.File) bool {
		fileCount++
		lineCount += f.LineCount()
		return true
	})

	fmt.Printf(
		"%s (%d files, %d lines, %d lines/s)\n",
		d, fileCount, lineCount, int64(float64(lineCount)/d.Seconds()),
	)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	initParserMode()

	start := time.Now()

	files, err := getPkgFiles(flag.Args())
	if err != nil {
		report(err)
		// ok to continue (files may be empty, but not nil)
	}

	checkPkgFiles(files)
	if errorCount > 0 {
		os.Exit(2)
	}

	if *verbose {
		printStats(time.Since(start))
	}
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package tests_test

import (
	"flag"
	"fmt"
	"time"

	"github.com/google/goexpect"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	kubev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"

	"kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/pkg/log"
	"kubevirt.io/kubevirt/tests"
)

var _ = Describe("Configurations", func() {

	flag.Parse()

	virtClient, err := kubecli.GetKubevirtClient()
	tests.PanicOnError(err)

	BeforeEach(func() {
		tests.BeforeTestCleanup()
	})

	Describe("VirtualMachineInstance definition", func() {
		Context("with 3 CPU cores", func() {
			var vmi *v1.VirtualMachineInstance

			BeforeEach(func() {
				vmi = tests.NewRandomVMIWithEphemeralDisk(tests.RegistryDiskFor(tests.RegistryDiskAlpine))
			})
			It("should report 3 cpu cores under guest OS", func() {
				vmi.Spec.Domain.CPU = &v1.CPU{
					Cores: 3,
				}
				vmi.Spec.Domain.Resources = v1.ResourceRequirements{
					Requests: kubev1.ResourceList{
						kubev1.ResourceMemory: resource.MustParse("64M"),
					},
				}

				By("Starting a VirtualMachineInstance")
				vmi, err = virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
				Expect(err).ToNot(HaveOccurred())
				tests.WaitForSuccessfulVMIStart(vmi)

				By("Expecting the VirtualMachineInstance console")
				expecter, _, err := tests.NewConsoleExpecter(virtClient, vmi, 10*time.Second)
				Expect(err).ToNot(HaveOccurred())
				defer expecter.Close()

				By("Checking the number of CPU cores under guest OS")
				_, err = expecter.ExpectBatch([]expect.Batcher{
					&expect.BSnd{S: "\n"},
					&expect.BExp{R: "Welcome to Alpine"},
					&expect.BSnd{S: "\n"},
					&expect.BExp{R: "login"},
					&expect.BSnd{S: "root\n"},
					&expect.BExp{R: "#"},
					&expect.BSnd{S: "grep -c ^processor /proc/cpuinfo\n"},
					&expect.BExp{R: "3"},
				}, 250*time.Second)

				By("Checking the requested amount of memory allocated for a guest")
				Expect(vmi.Spec.Domain.Resources.Requests.Memory().String()).To(Equal("64M"))

				readyPod := tests.GetRunningPodByLabel(vmi.Name, v1.DomainLabel, tests.NamespaceTestDefault)
				var computeContainer *kubev1.Container
				for _, container := range readyPod.Spec.Containers {
					println(container.Name)
					if container.Name == "compute" {
						computeContainer = &container
					}
				}
				if computeContainer == nil {
					tests.PanicOnError(fmt.Errorf("could not find the compute container"))
				}
				Expect(computeContainer.Resources.Requests.Memory().ToDec().ScaledValue(resource.Mega)).To(Equal(int64(179)))

				Expect(err).ToNot(HaveOccurred())
			}, 300)
		})

		FContext("with hugepages", func() {
			var hugepagesVm *v1.VirtualMachine

			BeforeEach(func() {
				hugepagesVm = tests.NewRandomVMWithEphemeralDiskAndUserdata(tests.RegistryDiskFor(tests.RegistryDiskCirros), "#!/bin/bash\necho 'hello'\n")
				hugepagesVm.Spec.Domain.Resources.Requests[kubev1.ResourceMemory] = resource.MustParse("64Mi")
			})

			table.DescribeTable("should consume hugepages ", func(resourceName kubev1.ResourceName, value string) {
				nodeWithHugepages := tests.GetNodeWithHugepages(virtClient, resourceName)
				if nodeWithHugepages == nil {
					Skip(fmt.Sprintf("No node with hugepages %s capacity", resourceName))
				}
				// initialHugepages := nodeWithHugepages.Status.Capacity[resourceName]
				hugepagesVm.Spec.Affinity = &v1.Affinity{
					NodeAffinity: &kubev1.NodeAffinity{
						RequiredDuringSchedulingIgnoredDuringExecution: &kubev1.NodeSelector{
							NodeSelectorTerms: []kubev1.NodeSelectorTerm{
								{
									MatchExpressions: []kubev1.NodeSelectorRequirement{
										{Key: "kubernetes.io/hostname", Operator: kubev1.NodeSelectorOpIn, Values: []string{nodeWithHugepages.Name}},
									},
								},
							},
						},
					},
				}
				hugepagesVm.Spec.Domain.Hugepages = &v1.Hugepages{}
				hugepagesVm.Spec.Domain.Hugepages.Size = value

				By("Starting a VM")
				_, err = virtClient.VM(tests.NamespaceTestDefault).Create(hugepagesVm)
				Expect(err).ToNot(HaveOccurred())
				tests.WaitForSuccessfulVMStart(hugepagesVm)

				time.Sleep(2 * time.Minute)

				By("Checking decrease of the node hugepage capacity")
				pods, err := virtClient.Core().Pods(tests.NamespaceTestDefault).List(tests.UnfinishedVMPodSelector(hugepagesVm))
				Expect(err).ToNot(HaveOccurred())
				Expect(len(pods.Items)).To(Equal(1))

				output, err := tests.ExecuteCommandOnPod(
					virtClient,
					&pods.Items[0],
					pods.Items[0].Spec.Containers[0].Name,
					[]string{"cat", "/proc/meminfo", "|", "grep", "HugePages_Free:"},
				)
				Expect(err).ToNot(HaveOccurred())

				fmt.Println(output)
			},
				table.Entry("hugepages-2Mi", v1.Hugepage2MiResource, "2Mi"),
				table.Entry("hugepages-1Gi", v1.Hugepage1GiResource, "1Gi"),
			)
		})
	})

	Context("New VirtualMachineInstance with all supported drives", func() {

		var vmi *v1.VirtualMachineInstance

		BeforeEach(func() {
			// ordering:
			// use a small disk for the other ones
			containerImage := tests.RegistryDiskFor(tests.RegistryDiskCirros)
			// virtio - added by NewRandomVMIWithEphemeralDisk
			vmi = tests.NewRandomVMIWithEphemeralDiskAndUserdata(containerImage, "echo hi!\n")
			// sata
			tests.AddEphemeralDisk(vmi, "disk2", "sata", containerImage)
			// ide
			tests.AddEphemeralDisk(vmi, "disk3", "ide", containerImage)
			// floppy
			tests.AddEphemeralFloppy(vmi, "disk4", containerImage)
			// NOTE: we have one disk per bus, so we expect vda, sda, hda, fda

			// We need ide support for the test, q35 does not support ide
			vmi.Spec.Domain.Machine.Type = "pc"
		})

		// FIXME ide and floppy is not recognized by the used image right now
		It("should have all the device nodes", func() {
			vmi, err = virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
			Expect(err).ToNot(HaveOccurred())
			tests.WaitForSuccessfulVMIStart(vmi)

			expecter, err := tests.LoggedInCirrosExpecter(vmi)
			Expect(err).ToNot(HaveOccurred())
			defer expecter.Close()
			res, err := expecter.ExpectBatch([]expect.Batcher{
				// keep the ordering!
				&expect.BSnd{S: "ls /dev/sda  /dev/vda  /dev/vdb\n"},
				&expect.BExp{R: "/dev/sda  /dev/vda  /dev/vdb"},
			}, 10*time.Second)
			log.DefaultLogger().Object(vmi).Infof("%v", res)

			Expect(err).ToNot(HaveOccurred())
		})
	})

})
<commit_msg>Continue with functional tests<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package tests_test

import (
	"flag"
	"fmt"
	"regexp"
	"strconv"
	"time"

	"github.com/google/goexpect"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	kubev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"

	"kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/pkg/log"
	"kubevirt.io/kubevirt/tests"
)

var _ = Describe("Configurations", func() {

	flag.Parse()

	virtClient, err := kubecli.GetKubevirtClient()
	tests.PanicOnError(err)

	BeforeEach(func() {
		tests.BeforeTestCleanup()
	})

	Describe("VirtualMachineInstance definition", func() {
		Context("with 3 CPU cores", func() {
			var vmi *v1.VirtualMachineInstance

			BeforeEach(func() {
				vmi = tests.NewRandomVMIWithEphemeralDisk(tests.RegistryDiskFor(tests.RegistryDiskAlpine))
			})
			It("should report 3 cpu cores under guest OS", func() {
				vmi.Spec.Domain.CPU = &v1.CPU{
					Cores: 3,
				}
				vmi.Spec.Domain.Resources = v1.ResourceRequirements{
					Requests: kubev1.ResourceList{
						kubev1.ResourceMemory: resource.MustParse("64M"),
					},
				}

				By("Starting a VirtualMachineInstance")
				vmi, err = virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
				Expect(err).ToNot(HaveOccurred())
				tests.WaitForSuccessfulVMIStart(vmi)

				By("Expecting the VirtualMachineInstance console")
				expecter, _, err := tests.NewConsoleExpecter(virtClient, vmi, 10*time.Second)
				Expect(err).ToNot(HaveOccurred())
				defer expecter.Close()

				By("Checking the number of CPU cores under guest OS")
				_, err = expecter.ExpectBatch([]expect.Batcher{
					&expect.BSnd{S: "\n"},
					&expect.BExp{R: "Welcome to Alpine"},
					&expect.BSnd{S: "\n"},
					&expect.BExp{R: "login"},
					&expect.BSnd{S: "root\n"},
					&expect.BExp{R: "#"},
					&expect.BSnd{S: "grep -c ^processor /proc/cpuinfo\n"},
					&expect.BExp{R: "3"},
				}, 250*time.Second)

				By("Checking the requested amount of memory allocated for a guest")
				Expect(vmi.Spec.Domain.Resources.Requests.Memory().String()).To(Equal("64M"))

				readyPod := tests.GetRunningPodByLabel(vmi.Name, v1.DomainLabel, tests.NamespaceTestDefault)
				var computeContainer *kubev1.Container
				for _, container := range readyPod.Spec.Containers {
					println(container.Name)
					if container.Name == "compute" {
						computeContainer = &container
					}
				}
				if computeContainer == nil {
					tests.PanicOnError(fmt.Errorf("could not find the compute container"))
				}
				Expect(computeContainer.Resources.Requests.Memory().ToDec().ScaledValue(resource.Mega)).To(Equal(int64(179)))

				Expect(err).ToNot(HaveOccurred())
			}, 300)
		})

		Context("with hugepages", func() {
			var hugepagesVm *v1.VirtualMachine

			verifyHugepagesConsumption := func() {
				// TODO: we need to check hugepages state via node allocated resources, but currently it has the issue
				// https://github.com/kubernetes/kubernetes/issues/64691
				pods, err := virtClient.Core().Pods(tests.NamespaceTestDefault).List(tests.UnfinishedVMPodSelector(hugepagesVm))
				Expect(err).ToNot(HaveOccurred())
				Expect(len(pods.Items)).To(Equal(1))

				// Get a hugepages statistics from virt-launcher pod
				output, err := tests.ExecuteCommandOnPod(
					virtClient,
					&pods.Items[0],
					pods.Items[0].Spec.Containers[0].Name,
					[]string{"cat", "/proc/meminfo"},
				)
				Expect(err).ToNot(HaveOccurred())

				hostHugepages := regexp.MustCompile(`HugePages_Total:\s*([0-9]+)\nHugePages_Free:\s*([0-9]+)`)

				totalHugepages, err := strconv.Atoi(hostHugepages.FindStringSubmatch(output)[1])
				Expect(err).ToNot(HaveOccurred())

				freeHugepages, err := strconv.Atoi(hostHugepages.FindStringSubmatch(output)[2])
				Expect(err).ToNot(HaveOccurred())

				// Verify that the VM memory equals to a number of consumed hugepages
				hugepagesSize := resource.MustParse(hugepagesVm.Spec.Domain.Hugepages.Size)
				vmHugepagesConsumption := int64(totalHugepages-freeHugepages) * hugepagesSize.Value()
				vmMemory := hugepagesVm.Spec.Domain.Resources.Requests[kubev1.ResourceMemory]

				Expect(vmHugepagesConsumption).To(Equal(vmMemory.Value()))
			}

			BeforeEach(func() {
				hugepagesVm = tests.NewRandomVMWithEphemeralDiskAndUserdata(tests.RegistryDiskFor(tests.RegistryDiskCirros), "#!/bin/bash\necho 'hello'\n")
				hugepagesVm.Spec.Domain.Resources.Requests[kubev1.ResourceMemory] = resource.MustParse("64Mi")
			})

			table.DescribeTable("should consume hugepages ", func(resourceName kubev1.ResourceName, value string) {
				nodeWithHugepages := tests.GetNodeWithHugepages(virtClient, resourceName)
				if nodeWithHugepages == nil {
					Skip(fmt.Sprintf("No node with hugepages %s capacity", resourceName))
				}
				// initialHugepages := nodeWithHugepages.Status.Capacity[resourceName]
				hugepagesVm.Spec.Affinity = &v1.Affinity{
					NodeAffinity: &kubev1.NodeAffinity{
						RequiredDuringSchedulingIgnoredDuringExecution: &kubev1.NodeSelector{
							NodeSelectorTerms: []kubev1.NodeSelectorTerm{
								{
									MatchExpressions: []kubev1.NodeSelectorRequirement{
										{Key: "kubernetes.io/hostname", Operator: kubev1.NodeSelectorOpIn, Values: []string{nodeWithHugepages.Name}},
									},
								},
							},
						},
					},
				}
				hugepagesVm.Spec.Domain.Hugepages = &v1.Hugepages{}
				hugepagesVm.Spec.Domain.Hugepages.Size = value

				By("Starting a VM")
				_, err = virtClient.VM(tests.NamespaceTestDefault).Create(hugepagesVm)
				Expect(err).ToNot(HaveOccurred())
				tests.WaitForSuccessfulVMStart(hugepagesVm)

				By("Checking that the VM memory equals to a number of consumed hugepages")
				// TODO: we need to check hugepages state via node allocated resources, but currently it has the issue
				// https://github.com/kubernetes/kubernetes/issues/64691
				verifyHugepagesConsumption()
			},
				table.Entry("hugepages-2Mi", v1.Hugepage2MiResource, "2Mi"),
				table.Entry("hugepages-1Gi", v1.Hugepage1GiResource, "1Gi"),
			)
		})
	})

	Context("New VirtualMachineInstance with all supported drives", func() {

		var vmi *v1.VirtualMachineInstance

		BeforeEach(func() {
			// ordering:
			// use a small disk for the other ones
			containerImage := tests.RegistryDiskFor(tests.RegistryDiskCirros)
			// virtio - added by NewRandomVMIWithEphemeralDisk
			vmi = tests.NewRandomVMIWithEphemeralDiskAndUserdata(containerImage, "echo hi!\n")
			// sata
			tests.AddEphemeralDisk(vmi, "disk2", "sata", containerImage)
			// ide
			tests.AddEphemeralDisk(vmi, "disk3", "ide", containerImage)
			// floppy
			tests.AddEphemeralFloppy(vmi, "disk4", containerImage)
			// NOTE: we have one disk per bus, so we expect vda, sda, hda, fda

			// We need ide support for the test, q35 does not support ide
			vmi.Spec.Domain.Machine.Type = "pc"
		})

		// FIXME ide and floppy is not recognized by the used image right now
		It("should have all the device nodes", func() {
			vmi, err = virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
			Expect(err).ToNot(HaveOccurred())
			tests.WaitForSuccessfulVMIStart(vmi)

			expecter, err := tests.LoggedInCirrosExpecter(vmi)
			Expect(err).ToNot(HaveOccurred())
			defer expecter.Close()
			res, err := expecter.ExpectBatch([]expect.Batcher{
				// keep the ordering!
				&expect.BSnd{S: "ls /dev/sda  /dev/vda  /dev/vdb\n"},
				&expect.BExp{R: "/dev/sda  /dev/vda  /dev/vdb"},
			}, 10*time.Second)
			log.DefaultLogger().Object(vmi).Infof("%v", res)

			Expect(err).ToNot(HaveOccurred())
		})
	})

})
<|endoftext|>