<commit_before>//
// Copyright 2017, Sander van Harmelen
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"net/url"
)

// BranchesService handles communication with the branch related methods
// of the GitLab API.
//
// GitLab API docs: https://docs.gitlab.com/ce/api/branches.html
type BranchesService struct {
	client *Client
}

// Branch represents a GitLab branch.
//
// GitLab API docs: https://docs.gitlab.com/ce/api/branches.html
type Branch struct {
	Commit             *Commit `json:"commit"`
	Name               string  `json:"name"`
	Protected          bool    `json:"protected"`
	Merged             bool    `json:"merged"`
	DevelopersCanPush  bool    `json:"developers_can_push"`
	DevelopersCanMerge bool    `json:"developers_can_merge"`
}

func (b Branch) String() string {
	return Stringify(b)
}

// ListBranchesOptions represents the available ListBranches() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#list-repository-branches
type ListBranchesOptions ListOptions

// ListBranches gets a list of repository branches from a project, sorted by
// name alphabetically.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#list-repository-branches
func (s *BranchesService) ListBranches(pid interface{}, opts *ListBranchesOptions, options ...OptionFunc) ([]*Branch, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/repository/branches", pathEscape(project))

	req, err := s.client.NewRequest("GET", u, opts, options)
	if err != nil {
		return nil, nil, err
	}

	var b []*Branch
	resp, err := s.client.Do(req, &b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, err
}

// GetBranch gets a single project repository branch.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#get-single-repository-branch
func (s *BranchesService) GetBranch(pid interface{}, branch string, options ...OptionFunc) (*Branch, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/repository/branches/%s", pathEscape(project), url.PathEscape(branch))

	req, err := s.client.NewRequest("GET", u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	b := new(Branch)
	resp, err := s.client.Do(req, b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, err
}

// ProtectBranchOptions represents the available ProtectBranch() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#protect-repository-branch
type ProtectBranchOptions struct {
	DevelopersCanPush  *bool `url:"developers_can_push,omitempty" json:"developers_can_push,omitempty"`
	DevelopersCanMerge *bool `url:"developers_can_merge,omitempty" json:"developers_can_merge,omitempty"`
}

// ProtectBranch protects a single project repository branch. This is an
// idempotent function, protecting an already protected repository branch
// still returns a 200 OK status code.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#protect-repository-branch
func (s *BranchesService) ProtectBranch(pid interface{}, branch string, opts *ProtectBranchOptions, options ...OptionFunc) (*Branch, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/repository/branches/%s/protect", pathEscape(project), url.PathEscape(branch))

	req, err := s.client.NewRequest("PUT", u, opts, options)
	if err != nil {
		return nil, nil, err
	}

	b := new(Branch)
	resp, err := s.client.Do(req, b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, err
}

// UnprotectBranch unprotects a single project repository branch. This is an
// idempotent function, unprotecting an already unprotected repository branch
// still returns a 200 OK status code.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#unprotect-repository-branch
func (s *BranchesService) UnprotectBranch(pid interface{}, branch string, options ...OptionFunc) (*Branch, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/repository/branches/%s/unprotect", pathEscape(project), url.PathEscape(branch))

	req, err := s.client.NewRequest("PUT", u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	b := new(Branch)
	resp, err := s.client.Do(req, b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, err
}

// CreateBranchOptions represents the available CreateBranch() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#create-repository-branch
type CreateBranchOptions struct {
	Branch *string `url:"branch,omitempty" json:"branch,omitempty"`
	Ref    *string `url:"ref,omitempty" json:"ref,omitempty"`
}

// CreateBranch creates branch from commit SHA or existing branch.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#create-repository-branch
func (s *BranchesService) CreateBranch(pid interface{}, opt *CreateBranchOptions, options ...OptionFunc) (*Branch, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/repository/branches", pathEscape(project))

	req, err := s.client.NewRequest("POST", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	b := new(Branch)
	resp, err := s.client.Do(req, b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, err
}

// DeleteBranch deletes an existing branch.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#delete-repository-branch
func (s *BranchesService) DeleteBranch(pid interface{}, branch string, options ...OptionFunc) (*Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, err
	}
	u := fmt.Sprintf("projects/%s/repository/branches/%s", pathEscape(project), url.PathEscape(branch))

	req, err := s.client.NewRequest("DELETE", u, nil, options)
	if err != nil {
		return nil, err
	}

	return s.client.Do(req, nil)
}

// DeleteMergedBranches deletes all branches that are merged into the project's default branch.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#delete-merged-branches
func (s *BranchesService) DeleteMergedBranches(pid interface{}, options ...OptionFunc) (*Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, err
	}
	u := fmt.Sprintf("projects/%s/repository/merged_branches", pathEscape(project))

	req, err := s.client.NewRequest("DELETE", u, nil, options)
	if err != nil {
		return nil, err
	}

	return s.client.Do(req, nil)
}
<commit_msg>Added field which shows if the branch is default<commit_after>//
// Copyright 2017, Sander van Harmelen
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"net/url"
)

// BranchesService handles communication with the branch related methods
// of the GitLab API.
//
// GitLab API docs: https://docs.gitlab.com/ce/api/branches.html
type BranchesService struct {
	client *Client
}

// Branch represents a GitLab branch.
//
// GitLab API docs: https://docs.gitlab.com/ce/api/branches.html
type Branch struct {
	Commit             *Commit `json:"commit"`
	Name               string  `json:"name"`
	Protected          bool    `json:"protected"`
	Merged             bool    `json:"merged"`
	Default            bool    `json:"default"`
	DevelopersCanPush  bool    `json:"developers_can_push"`
	DevelopersCanMerge bool    `json:"developers_can_merge"`
}

func (b Branch) String() string {
	return Stringify(b)
}

// ListBranchesOptions represents the available ListBranches() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#list-repository-branches
type ListBranchesOptions ListOptions

// ListBranches gets a list of repository branches from a project, sorted by
// name alphabetically.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#list-repository-branches
func (s *BranchesService) ListBranches(pid interface{}, opts *ListBranchesOptions, options ...OptionFunc) ([]*Branch, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/repository/branches", pathEscape(project))

	req, err := s.client.NewRequest("GET", u, opts, options)
	if err != nil {
		return nil, nil, err
	}

	var b []*Branch
	resp, err := s.client.Do(req, &b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, err
}

// GetBranch gets a single project repository branch.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#get-single-repository-branch
func (s *BranchesService) GetBranch(pid interface{}, branch string, options ...OptionFunc) (*Branch, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/repository/branches/%s", pathEscape(project), url.PathEscape(branch))

	req, err := s.client.NewRequest("GET", u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	b := new(Branch)
	resp, err := s.client.Do(req, b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, err
}

// ProtectBranchOptions represents the available ProtectBranch() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#protect-repository-branch
type ProtectBranchOptions struct {
	DevelopersCanPush  *bool `url:"developers_can_push,omitempty" json:"developers_can_push,omitempty"`
	DevelopersCanMerge *bool `url:"developers_can_merge,omitempty" json:"developers_can_merge,omitempty"`
}

// ProtectBranch protects a single project repository branch. This is an
// idempotent function, protecting an already protected repository branch
// still returns a 200 OK status code.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#protect-repository-branch
func (s *BranchesService) ProtectBranch(pid interface{}, branch string, opts *ProtectBranchOptions, options ...OptionFunc) (*Branch, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/repository/branches/%s/protect", pathEscape(project), url.PathEscape(branch))

	req, err := s.client.NewRequest("PUT", u, opts, options)
	if err != nil {
		return nil, nil, err
	}

	b := new(Branch)
	resp, err := s.client.Do(req, b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, err
}

// UnprotectBranch unprotects a single project repository branch. This is an
// idempotent function, unprotecting an already unprotected repository branch
// still returns a 200 OK status code.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#unprotect-repository-branch
func (s *BranchesService) UnprotectBranch(pid interface{}, branch string, options ...OptionFunc) (*Branch, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/repository/branches/%s/unprotect", pathEscape(project), url.PathEscape(branch))

	req, err := s.client.NewRequest("PUT", u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	b := new(Branch)
	resp, err := s.client.Do(req, b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, err
}

// CreateBranchOptions represents the available CreateBranch() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#create-repository-branch
type CreateBranchOptions struct {
	Branch *string `url:"branch,omitempty" json:"branch,omitempty"`
	Ref    *string `url:"ref,omitempty" json:"ref,omitempty"`
}

// CreateBranch creates branch from commit SHA or existing branch.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#create-repository-branch
func (s *BranchesService) CreateBranch(pid interface{}, opt *CreateBranchOptions, options ...OptionFunc) (*Branch, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/repository/branches", pathEscape(project))

	req, err := s.client.NewRequest("POST", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	b := new(Branch)
	resp, err := s.client.Do(req, b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, err
}

// DeleteBranch deletes an existing branch.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#delete-repository-branch
func (s *BranchesService) DeleteBranch(pid interface{}, branch string, options ...OptionFunc) (*Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, err
	}
	u := fmt.Sprintf("projects/%s/repository/branches/%s", pathEscape(project), url.PathEscape(branch))

	req, err := s.client.NewRequest("DELETE", u, nil, options)
	if err != nil {
		return nil, err
	}

	return s.client.Do(req, nil)
}

// DeleteMergedBranches deletes all branches that are merged into the project's default branch.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/branches.html#delete-merged-branches
func (s *BranchesService) DeleteMergedBranches(pid interface{}, options ...OptionFunc) (*Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, err
	}
	u := fmt.Sprintf("projects/%s/repository/merged_branches", pathEscape(project))

	req, err := s.client.NewRequest("DELETE", u, nil, options)
	if err != nil {
		return nil, err
	}

	return s.client.Do(req, nil)
}
<|endoftext|>


<commit_before>package comm

import (
	"net"

	"github.com/kenix/gomad/util"
)

type Client struct {
	conn  net.Conn
	bytes uint64
}

func NewClient(conn net.Conn) *Client {
	return &Client{conn, 0}
}

func (cln *Client) Do(dc DualChan, notify chan<- *Client) {
	for dat := range dc.Out {
		n, err := snd(cln.conn, dat)
		if err != nil {
			notify <- cln
			util.Li.Printf("notify close client from %s\n", cln.conn.RemoteAddr())
			return
		}
		cln.bytes += uint64(n)
	}
	util.Li.Printf("done on client from %s\n", cln.conn.RemoteAddr())
}

func (cln *Client) Close() {
	util.Li.Printf("close client from %s\n", cln.conn.RemoteAddr())
	cln.conn.Close()
}

func (cln *Client) BytesTransferred() uint64 {
	return cln.bytes
}

func snd(conn net.Conn, dat []byte) (int, error) {
	sent := 0
	for sent < len(dat) {
		n, err := conn.Write(dat[sent:])
		if err != nil {
			util.Le.Printf("failed sending data %s\n", err)
			return sent, err
		}
		sent += n
	}
	util.Lt.Printf("sent %d byte(s) to %s\n", sent, conn.RemoteAddr())
	return sent, nil
}
<commit_msg>reduce client logs<commit_after>package comm

import (
	"net"

	"github.com/kenix/gomad/util"
)

type Client struct {
	conn  net.Conn
	bytes uint64
}

func NewClient(conn net.Conn) *Client {
	return &Client{conn, 0}
}

func (cln *Client) Do(dc DualChan, notify chan<- *Client) {
	for dat := range dc.Out {
		n, err := snd(cln.conn, dat)
		if err != nil {
			notify <- cln
			util.Li.Printf("notify close client from %s\n", cln.conn.RemoteAddr())
			return
		}
		cln.bytes += uint64(n)
	}
	util.Li.Printf("done on client from %s\n", cln.conn.RemoteAddr())
}

func (cln *Client) Close() {
	util.Li.Printf("close client from %s\n", cln.conn.RemoteAddr())
	cln.conn.Close()
}

func (cln *Client) BytesTransferred() uint64 {
	return cln.bytes
}

func snd(conn net.Conn, dat []byte) (int, error) {
	sent := 0
	for sent < len(dat) {
		n, err := conn.Write(dat[sent:])
		if err != nil {
			util.Le.Printf("failed sending data %s\n", err)
			return sent, err
		}
		sent += n
	}
	// util.Lt.Printf("sent %d byte(s) to %s\n", sent, conn.RemoteAddr())
	return sent, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
)

const (
	issueTracker = "https://github.com/Matt3o12/dict-cc/issues"

	// DictBaseURL The base url of dict.cc
	DictBaseURL = "http://dict.cc/"

	// AllLangaugesGet URL where all available langauge pairs can be found.
	AllLangaugesGet = "http://browse.dict.cc/"

	allAvaiableLangsCSSPath = "#maincontent form[name='langbarchooser'] " +
		"table td a"
)

func handleErr(err error) {
	fmt.Println(err)
	fmt.Printf("If you believe that is a bug, please open a ticket at %v.\n",
		issueTracker)
}

func updateLanguages() {
	fmt.Println("This is your first usage.")
	fmt.Println("Available languages are being updated.")
	fmt.Println("This may take a while ...")
	fmt.Println()

	response, err := http.Get(AllLangaugesGet)
	if err != nil {
		handleErr(err)
		return
	}

	langs, err := GetLanguagesFromRemote(response)
	if err != nil {
		handleErr(err)
		return
	}

	fmt.Printf("Total langs: %v\n", len(langs))
}

func main() {
	updateLanguages()
}
<commit_msg>Implemented save languages feature.<commit_after>package main

import (
	"errors"
	"fmt"
	"net/http"
	"os"
	"path"

	"github.com/codegangsta/cli"
	"github.com/mitchellh/go-homedir"
)

const (
	issueTracker = "https://github.com/Matt3o12/dict-cc/issues"

	// DictBaseURL The base url of dict.cc
	DictBaseURL = "http://dict.cc/"

	// AllLangaugesGet URL where all available langauge pairs can be found.
	AllLangaugesGet = "http://browse.dict.cc/"

	allAvaiableLangsCSSPath = "#maincontent form[name='langbarchooser'] " +
		"table td a"

	// Version current version
	Version = "0.0.0"

	// Author of the app
	Author = "Matteo Kloiber"

	// Email my email
	Email = "info@matt3o12.de"
)

func handleErr(err error) {
	if err != nil {
		fmt.Println(err)
		fmt.Printf("If you believe that is a bug, please open a ticket at %v.\n",
			issueTracker)
		os.Exit(1)

	}
}

func getLangSaveFile() (string, error) {
	home, err := homedir.Dir()
	if err != nil {
		return "", errors.New("Could not find home direcotry.")
	}

	langsFile := path.Join(home, ".dict_cc", "languages.json")
	err = os.Mkdir(path.Dir(langsFile), 0755)
	if err != nil && !os.IsExist(err) {
		return "", err
	}

	return langsFile, nil
}

func updateLanguages() {
	fmt.Println("Available languages are being updated.")
	fmt.Println("This may take a while ...")
	fmt.Println()

	response, err := http.Get(AllLangaugesGet)
	handleErr(err)

	langs, err := GetLanguagesFromRemote(response)
	handleErr(err)

	fmt.Printf("Total langs: %v\n", len(langs))

	fileName, err := getLangSaveFile()
	handleErr(err)

	file, err := os.Create(fileName)
	handleErr(err)
	defer file.Close()

	SaveLanguagesToDisk(langs, file)
}

func updateLanguagesCommand(c *cli.Context) {
	if len(c.Args()) > 0 {
		fmt.Println("You may not set additional commands.")
		fmt.Println()

		cli.ShowSubcommandHelp(c)
		os.Exit(1)
	}

	updateLanguages()
}

func lookupCommand(c *cli.Context) {
	langFile, err := getLangSaveFile()
	handleErr(err)

	if stats, err := os.Stat(langFile); err != nil || !stats.Mode().IsRegular() {
		fmt.Println("This is your first usage.")
		updateLanguages()
	}

	// Code for looking up words goes here.
}

func main() {
	app := cli.NewApp()
	app.Name = "Dict.cc client"

	// Usage probably means description
	app.Usage = "Look up any word in many languages!"
	app.Author = Author
	app.Version = Version
	app.Email = Email
	app.Action = lookupCommand
	app.Commands = []cli.Command{
		{
			Name:   "update-langs",
			Usage:  "Updates all languages",
			Action: updateLanguagesCommand,
			Flags:  make([]cli.Flag, 0),
		},
	}

	app.Run(os.Args)
	os.Exit(0)
}
<|endoftext|>


<commit_before>// Copyright 2011 Julian Phillips.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package py

// #include "utils.h"
import "C"

import (
	"runtime"
)

type GilState struct {
	state C.PyGILState_STATE
	set   bool
}

func GilState_Ensure() *GilState {
	ret := C.PyGILState_Ensure()
	return &GilState{ret, true}
}

func (g *GilState) Release() {
	if g.set {
		C.PyGILState_Release(g.state)
		g.set = false
	}
}

func AcquireGil() {
	C.PyEval_AcquireLock()
}

func ReleaseGil() {
	C.PyEval_ReleaseLock()
}

// InitAndLock is a convience function.  It initializes Python, enables thread
// support, and returns a locked Lock instance.
func InitAndLock() *Lock {
	C.Py_Initialize()
	C.PyEval_InitThreads()
	C.PyEval_SaveThread()
	return NewLock()
}

// Lock is a high-level representation of the Python Global Interpreter Lock
// (GIL) and thread state.  When calling from Go into Python the GIL needs to be
// held and the appropriate thread state loaded.  Lock also calls
// runtime.LockOSThread() to make sure that the calling goroutine doesn't move
// thread whilst calling Python code which would invalidate the per-thread
// state.
//
// Basic usage is:
//
//   lock = py.NewLock()
//
//   // Call Python code ...
//
//   lock.Unlock()
//
// If it appropriate to let other Python threads run (e.g. during a long
// computation, or blocking operation), then there are two options.  Either
// unlock:
//
//   lock = py.NewLock()
//
//   // Call Python code ...
//
//   lock.Unlock()
//
//   // Slow or blocking Go operation
//
//   lock.Lock()
//
//   // Call Python code ...
//
//   lock.Unlock()
//
// or unblock threads, which will not call runtme.UnlockOSThread() but it less
// expensive, as we do not free and then recreate a thread state variable:
//
//   lock = py.NewLock()
//
//   // Call Python code ...
//
//   lock.UnblockThreads()
//
//   // Slow or blocking Go operation
//
//   lock.BlockThreads()
//
//   // Call Python code ...
//
//   lock.Unlock()
type Lock struct {
	gilState *GilState
	thState  *C.PyThreadState
}

// NewLock returns a new locked Lock
func NewLock() (lock *Lock) {
	lock = new(Lock)
	lock.Lock()
	return
}

// Lock locks the lock.  When it returns everything is setup for calling into
// Python.  No other Python threads will run until either Unlock() or
// UnblockThreads() are called.
//
// If the lock is already locked when this function is called, then nothing
// happens, and the function will return immediately.
func (lock *Lock) Lock() {
	if lock.gilState != nil {
		return
	}

	if lock.thState != nil {
		panic("Lock() called with threads unblocked!")
	}

	runtime.LockOSThread()
	lock.gilState = GilState_Ensure()
}

// Unlock unlocks the lock.  When it returns no calls into Python made be made.
//
// If the lock is not locked when this function is called, then nothing happens,
// and the function returns immediately.  Also, it is not necessay to call
// BlockThreads() before calling Unlock(), even if UnblockThreads() has been
// called.
func (lock *Lock) Unlock() {
	if lock.gilState == nil {
		return
	}

	if lock.thState != nil {
		C.PyEval_RestoreThread(lock.thState)
		lock.thState = nil
	}

	lock.gilState.Release()
	runtime.UnlockOSThread()
	lock.gilState = nil
}

// UnblockThreads() releases the GIL so that other Python threads may run.  It
// does not free the per-thread state created by Lock, nor does it call
// runtime.UnlockOSThread().  This function is intended to allow other Python
// threads to run whilst the calling code is either performing a slow/long
// running operation or is going to block.
//
// Nothing happens if this function is called more than once, all calls but the
// first will be ignored.
func (lock *Lock) UnblockThreads() {
	if lock.gilState == nil {
		panic("UnblockThreads() called on Unlocked Lock")
	}

	if lock.thState == nil {
		lock.thState = C.PyEval_SaveThread()
	}
}

// BlockThreads() reclaims the GIL (and restores per-thread state), after is has
// been released by UnblockThreads().
//
// If this function is called without UnblockThreads() having been called, then
// nothing happens and the function returns immediately.
func (lock *Lock) BlockThreads() {
	if lock.gilState == nil {
		panic("BlockThreads() called on Unlocked Lock")
	}

	if lock.thState != nil {
		C.PyEval_RestoreThread(lock.thState)
		lock.thState = nil
	}
}
<commit_msg>Improve InitAndLock<commit_after>// Copyright 2011 Julian Phillips.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package py

// #include "utils.h"
import "C"

import (
	"runtime"
)

type GilState struct {
	state C.PyGILState_STATE
	set   bool
}

func GilState_Ensure() *GilState {
	ret := C.PyGILState_Ensure()
	return &GilState{ret, true}
}

func (g *GilState) Release() {
	if g.set {
		C.PyGILState_Release(g.state)
		g.set = false
	}
}

func AcquireGil() {
	C.PyEval_AcquireLock()
}

func ReleaseGil() {
	C.PyEval_ReleaseLock()
}

// InitAndLock is a convience function.  It initializes Python, enables thread
// support, and returns a locked Lock instance.
func InitAndLock() *Lock {
	// Lock the current goroutine to the current OS thread, until we have
	// released the GIL (as CPython uses per-thread state)
	runtime.LockOSThread()

	// Initialize the default Python interpreter
	C.Py_Initialize()

	// Enable Python thread support, and then immediately release the GIL (and
	// thus "deativate" and per-thread state associated with the current thread
	C.PyEval_InitThreads()
	C.PyEval_SaveThread()

	// We can now unlock the current goroutine from the current OS thread, as
	// there is no active per-thread state
	runtime.UnlockOSThread()

	// Now that Python is setup, we can return a locked Lock, ready for the
	// calling code to use
	return NewLock()
}

// Lock is a high-level representation of the Python Global Interpreter Lock
// (GIL) and thread state.  When calling from Go into Python the GIL needs to be
// held and the appropriate thread state loaded.  Lock also calls
// runtime.LockOSThread() to make sure that the calling goroutine doesn't move
// thread whilst calling Python code which would invalidate the per-thread
// state.
//
// Basic usage is:
//
//   lock = py.NewLock()
//
//   // Call Python code ...
//
//   lock.Unlock()
//
// If it appropriate to let other Python threads run (e.g. during a long
// computation, or blocking operation), then there are two options.  Either
// unlock:
//
//   lock = py.NewLock()
//
//   // Call Python code ...
//
//   lock.Unlock()
//
//   // Slow or blocking Go operation
//
//   lock.Lock()
//
//   // Call Python code ...
//
//   lock.Unlock()
//
// or unblock threads, which will not call runtme.UnlockOSThread() but it less
// expensive, as we do not free and then recreate a thread state variable:
//
//   lock = py.NewLock()
//
//   // Call Python code ...
//
//   lock.UnblockThreads()
//
//   // Slow or blocking Go operation
//
//   lock.BlockThreads()
//
//   // Call Python code ...
//
//   lock.Unlock()
type Lock struct {
	gilState *GilState
	thState  *C.PyThreadState
}

// NewLock returns a new locked Lock
func NewLock() (lock *Lock) {
	lock = new(Lock)
	lock.Lock()
	return
}

// Lock locks the lock.  When it returns everything is setup for calling into
// Python.  No other Python threads will run until either Unlock() or
// UnblockThreads() are called.
//
// If the lock is already locked when this function is called, then nothing
// happens, and the function will return immediately.
func (lock *Lock) Lock() {
	if lock.gilState != nil {
		return
	}

	if lock.thState != nil {
		panic("Lock() called with threads unblocked!")
	}

	runtime.LockOSThread()
	lock.gilState = GilState_Ensure()
}

// Unlock unlocks the lock.  When it returns no calls into Python made be made.
//
// If the lock is not locked when this function is called, then nothing happens,
// and the function returns immediately.  Also, it is not necessay to call
// BlockThreads() before calling Unlock(), even if UnblockThreads() has been
// called.
func (lock *Lock) Unlock() {
	if lock.gilState == nil {
		return
	}

	if lock.thState != nil {
		C.PyEval_RestoreThread(lock.thState)
		lock.thState = nil
	}

	lock.gilState.Release()
	runtime.UnlockOSThread()
	lock.gilState = nil
}

// UnblockThreads() releases the GIL so that other Python threads may run.  It
// does not free the per-thread state created by Lock, nor does it call
// runtime.UnlockOSThread().  This function is intended to allow other Python
// threads to run whilst the calling code is either performing a slow/long
// running operation or is going to block.
//
// Nothing happens if this function is called more than once, all calls but the
// first will be ignored.
func (lock *Lock) UnblockThreads() {
	if lock.gilState == nil {
		panic("UnblockThreads() called on Unlocked Lock")
	}

	if lock.thState == nil {
		lock.thState = C.PyEval_SaveThread()
	}
}

// BlockThreads() reclaims the GIL (and restores per-thread state), after is has
// been released by UnblockThreads().
//
// If this function is called without UnblockThreads() having been called, then
// nothing happens and the function returns immediately.
func (lock *Lock) BlockThreads() {
	if lock.gilState == nil {
		panic("BlockThreads() called on Unlocked Lock")
	}

	if lock.thState != nil {
		C.PyEval_RestoreThread(lock.thState)
		lock.thState = nil
	}
}
<|endoftext|>


<commit_before>package build

import (
	"fmt"
	"time"

	"github.com/asdine/storm"
	uuid "github.com/satori/go.uuid"
	"github.com/velocity-ci/velocity/backend/pkg/domain"
	"github.com/velocity-ci/velocity/backend/pkg/velocity"
	"go.uber.org/zap"
)

const (
	EventStreamLineCreate = "streamLine:new"
)

type StreamManager struct {
	db *streamStormDB

	brokers []domain.Broker
}

func NewStreamManager(
	db *storm.DB,
) *StreamManager {
	m := &StreamManager{
		db:      newStreamStormDB(db),
		brokers: []domain.Broker{},
	}
	return m
}

func (m *StreamManager) AddBroker(b domain.Broker) {
	m.brokers = append(m.brokers, b)
}

func (m *StreamManager) create(
	s *Step,
	name string,
) *Stream {
	stream := &Stream{
		ID:   uuid.NewV3(uuid.NewV1(), s.ID).String(),
		Step: s,
		Name: name,
	}

	m.db.save(stream)
	return stream
}

func (m *StreamManager) update(s *Stream) error {
	return m.db.save(s)
}

func (m *StreamManager) GetByID(id string) (*Stream, error) {
	return GetStreamByID(m.db.DB, id)
}

func (m *StreamManager) CreateStreamLine(
	stream *Stream,
	lineNumber int,
	timestamp time.Time,
	output string,
) *StreamLine {
	sL := &StreamLine{
		StreamID:   stream.ID,
		LineNumber: lineNumber,
		Timestamp:  timestamp,
		Output:     fmt.Sprintf("%s", output),
	}
	m.db.saveStreamLine(sL)
	for _, b := range m.brokers {
		b.EmitAll(&domain.Emit{
			Topic:   fmt.Sprintf("stream:%s", stream.ID),
			Event:   EventStreamLineCreate,
			Payload: sL,
		})
	}
	return sL
}

func (m *StreamManager) GetStreamsForStep(s *Step) []*Stream {
	return getStreamsByStepID(m.db.DB, s.ID)
}

func (m *StreamManager) GetStreamLines(s *Stream, q *domain.PagingQuery) ([]*StreamLine, int) {
	return m.db.getLinesByStream(s, q)
}

func (m *StreamManager) Update(stream *Stream) error {
	if err := m.db.save(stream); err != nil {
		return err
	}
	// for _, b := range m.brokers {
	// 	b.EmitAll(&domain.Emit{
	// 		Event:   ,
	// 		Payload: s,
	// 	})
	// }

	return nil
}

func GetStreamByID(db *storm.DB, id string) (*Stream, error) {
	var sS StormStream
	if err := db.One("ID", id, &sS); err != nil {
		velocity.GetLogger().Error("error", zap.Error(err))
		return nil, err
	}
	return sS.toStream(db), nil
}
<commit_msg>Not saving carriage returns to DB so ensuring that every line has a new-line<commit_after>package build

import (
	"fmt"
	"strings"
	"time"

	"github.com/asdine/storm"
	uuid "github.com/satori/go.uuid"
	"github.com/velocity-ci/velocity/backend/pkg/domain"
	"github.com/velocity-ci/velocity/backend/pkg/velocity"
	"go.uber.org/zap"
)

const (
	EventStreamLineCreate = "streamLine:new"
)

type StreamManager struct {
	db *streamStormDB

	brokers []domain.Broker
}

func NewStreamManager(
	db *storm.DB,
) *StreamManager {
	m := &StreamManager{
		db:      newStreamStormDB(db),
		brokers: []domain.Broker{},
	}
	return m
}

func (m *StreamManager) AddBroker(b domain.Broker) {
	m.brokers = append(m.brokers, b)
}

func (m *StreamManager) create(
	s *Step,
	name string,
) *Stream {
	stream := &Stream{
		ID:   uuid.NewV3(uuid.NewV1(), s.ID).String(),
		Step: s,
		Name: name,
	}

	m.db.save(stream)
	return stream
}

func (m *StreamManager) update(s *Stream) error {
	return m.db.save(s)
}

func (m *StreamManager) GetByID(id string) (*Stream, error) {
	return GetStreamByID(m.db.DB, id)
}

func (m *StreamManager) CreateStreamLine(
	stream *Stream,
	lineNumber int,
	timestamp time.Time,
	output string,
) *StreamLine {
	sL := &StreamLine{
		StreamID:   stream.ID,
		LineNumber: lineNumber,
		Timestamp:  timestamp,
		Output:     fmt.Sprintf("%s", output),
	}
	if !strings.HasSuffix(sL.Output, "\r") {
		sL.Output = fmt.Sprintf("%s\n", strings.TrimSpace(sL.Output))
		m.db.saveStreamLine(sL)
	}
	for _, b := range m.brokers {
		b.EmitAll(&domain.Emit{
			Topic:   fmt.Sprintf("stream:%s", stream.ID),
			Event:   EventStreamLineCreate,
			Payload: sL,
		})
	}
	return sL
}

func (m *StreamManager) GetStreamsForStep(s *Step) []*Stream {
	return getStreamsByStepID(m.db.DB, s.ID)
}

func (m *StreamManager) GetStreamLines(s *Stream, q *domain.PagingQuery) ([]*StreamLine, int) {
	return m.db.getLinesByStream(s, q)
}

func (m *StreamManager) Update(stream *Stream) error {
	if err := m.db.save(stream); err != nil {
		return err
	}
	// for _, b := range m.brokers {
	// 	b.EmitAll(&domain.Emit{
	// 		Event:   ,
	// 		Payload: s,
	// 	})
	// }

	return nil
}

func GetStreamByID(db *storm.DB, id string) (*Stream, error) {
	var sS StormStream
	if err := db.One("ID", id, &sS); err != nil {
		velocity.GetLogger().Error("error", zap.Error(err))
		return nil, err
	}
	return sS.toStream(db), nil
}
<|endoftext|>


<commit_before>//  Description = ipquail website
//  Author = Theodore Baschak
//  Version = 1.0

package main

import (
  "fmt"
  "net"
  "github.com/pilu/traffic"
)

func indexHandler(w traffic.ResponseWriter, r *traffic.Request) {
  w.Render("ipquail")
}

func ipHandler(w traffic.ResponseWriter, r *traffic.Request) {
  traffic.Logger().Print( r.Header.Get("X-Forwarded-For") )
  w.Header().Add("Content-type", "text/plain")
  w.WriteText( r.Header.Get("X-Forwarded-For") )
}

func ptrHandler(w traffic.ResponseWriter, r *traffic.Request) {
  addr, err := net.LookupAddr( r.Header.Get("X-Forwarded-For") )
  fmt.Println(addr, err)
  traffic.Logger().Print( r.Header.Get("X-Forwarded-For") )
  w.Header().Add("Content-type", "text/plain")
  if len(addr) > 0 {
    w.WriteText( addr[0] )
  } else {
    w.WriteText( "none" )
  }
}

func ipapiHandler(w traffic.ResponseWriter, r *traffic.Request) {
  traffic.Logger().Print( r.Header.Get("X-Forwarded-For") )
  w.Header().Add("Access-Control-Allow-Origin", "*")
  w.Header().Add("Access-Control-Allow-Methods", "GET")
  w.Header().Add("Access-Control-Allow-Headers", "X-Requested-With,Accept,Content-Type,Origin")
  w.Header().Add("Content-type", "application/json")
  w.WriteText( "{ \"ip\": \"" )
  w.WriteText( r.Header.Get("X-Forwarded-For") )
  w.WriteText( "\" }" )
}

func ptrapiHandler(w traffic.ResponseWriter, r *traffic.Request) {
  addr, err := net.LookupAddr( r.Header.Get("X-Forwarded-For") )
  fmt.Println(addr, err)
  traffic.Logger().Print( r.Header.Get("X-Forwarded-For") )
  w.Header().Add("Access-Control-Allow-Origin", "*")
  w.Header().Add("Access-Control-Allow-Methods", "GET")
  w.Header().Add("Access-Control-Allow-Headers", "X-Requested-With,Accept,Content-Type,Origin")
  w.Header().Add("Content-type", "application/json")
  if len(addr) > 0 {
    w.WriteText( "{ \"ptr\": \"" )
    w.WriteText( addr[0] )
    w.WriteText( "\" }" )
  } else {
    w.WriteText( "{ \"ptr\": \"none\" }" )
  }
}

func main() {
  router := traffic.New()

  // add a route for each page you add to the site
  // make sure you create a route handler for it

  router.Get("/", indexHandler)
  router.Get("/ip", ipapiHandler)
  router.Get("/ptr", ptrapiHandler)
  router.Get("/api/ip", ipapiHandler)
  router.Get("/api/ptr", ptrapiHandler)
  router.Run()
}
<commit_msg>fix shell endpoints routing<commit_after>//  Description = ipquail website
//  Author = Theodore Baschak
//  Version = 1.0

package main

import (
  "fmt"
  "net"
  "github.com/pilu/traffic"
)

func indexHandler(w traffic.ResponseWriter, r *traffic.Request) {
  w.Render("ipquail")
}

func ipHandler(w traffic.ResponseWriter, r *traffic.Request) {
  traffic.Logger().Print( r.Header.Get("X-Forwarded-For") )
  w.Header().Add("Content-type", "text/plain")
  w.WriteText( r.Header.Get("X-Forwarded-For") )
}

func ptrHandler(w traffic.ResponseWriter, r *traffic.Request) {
  addr, err := net.LookupAddr( r.Header.Get("X-Forwarded-For") )
  fmt.Println(addr, err)
  traffic.Logger().Print( r.Header.Get("X-Forwarded-For") )
  w.Header().Add("Content-type", "text/plain")
  if len(addr) > 0 {
    w.WriteText( addr[0] )
  } else {
    w.WriteText( "none" )
  }
}

func ipapiHandler(w traffic.ResponseWriter, r *traffic.Request) {
  traffic.Logger().Print( r.Header.Get("X-Forwarded-For") )
  w.Header().Add("Access-Control-Allow-Origin", "*")
  w.Header().Add("Access-Control-Allow-Methods", "GET")
  w.Header().Add("Access-Control-Allow-Headers", "X-Requested-With,Accept,Content-Type,Origin")
  w.Header().Add("Content-type", "application/json")
  w.WriteText( "{ \"ip\": \"" )
  w.WriteText( r.Header.Get("X-Forwarded-For") )
  w.WriteText( "\" }" )
}

func ptrapiHandler(w traffic.ResponseWriter, r *traffic.Request) {
  addr, err := net.LookupAddr( r.Header.Get("X-Forwarded-For") )
  fmt.Println(addr, err)
  traffic.Logger().Print( r.Header.Get("X-Forwarded-For") )
  w.Header().Add("Access-Control-Allow-Origin", "*")
  w.Header().Add("Access-Control-Allow-Methods", "GET")
  w.Header().Add("Access-Control-Allow-Headers", "X-Requested-With,Accept,Content-Type,Origin")
  w.Header().Add("Content-type", "application/json")
  if len(addr) > 0 {
    w.WriteText( "{ \"ptr\": \"" )
    w.WriteText( addr[0] )
    w.WriteText( "\" }" )
  } else {
    w.WriteText( "{ \"ptr\": \"none\" }" )
  }
}

func main() {
  router := traffic.New()

  // add a route for each page you add to the site
  // make sure you create a route handler for it

  router.Get("/", indexHandler)
  router.Get("/ip", ipHandler)
  router.Get("/ptr", ptrHandler)
  router.Get("/api/ip", ipapiHandler)
  router.Get("/api/ptr", ptrapiHandler)
  router.Run()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

const jaegerTemplateExtension = ".jgrt"
const jaegerDescription = "JaegerH - Jaeger Helper program\n\nJaeger is a JSON encoded GPG encrypted key value store. It is useful for separating development with operations and keeping configuration files secure."
const jaegerQuote = "\"Stacker Pentecost: Haven't you heard Mr. Beckett? The world is coming to an end. So where would you rather die? Here? Or in a Jaeger!\" - Pacific Rim"
const jaegerRecommendedUsage = "RECOMMENDED:\n    jaegerh -i file.txt.jgrt"

var debug debugging = false

type debugging bool

func (d debugging) Printf(format string, args ...interface{}) {
	// From: https://groups.google.com/forum/#!msg/golang-nuts/gU7oQGoCkmg/BNIl-TqB-4wJ
	if d {
		log.Printf(format, args...)
	}
}

func main() {
	// Define flags
	var (
		debugFlag     = flag.Bool("d", false, "Enable Debug")
		inputTemplate = flag.String("i", "", "Input Template file. eg. file.txt.jgrt")
	)

	flag.Usage = func() {
		fmt.Printf("%s\n%s\n\n%s\n\n", jaegerDescription, jaegerQuote, jaegerRecommendedUsage)
		fmt.Fprintf(os.Stderr, "OPTIONS:\n")
		flag.PrintDefaults()
	}

	flag.Parse()

	if *debugFlag {
		debug = true
	}

	if *inputTemplate == "" {
		assumedTemplate, err := checkExistsJaegerT()
		if err != nil {
			flag.Usage()
			log.Fatalf("\n\nError: %s", err)
		}
		*inputTemplate = assumedTemplate
	}

	processInputFile(inputTemplate)
}

func checkExistsJaegerT() (string, error) {
	// Check that one template file is in the current directory and use that file
	files, err := filepath.Glob("*.jgrt")
	if err != nil {
		return "", err
	}
	if len(files) == 1 {
		return files[0], nil
	}
	return "", fmt.Errorf("No input template file specified")
}

func processInputFile(inputFile *string) {
	file, err := os.Open(*inputFile)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		parseString := scanner.Text()
		matched, _ := regexp.MatchString("^[[:space:]]*#", parseString)
		if matched {
			if debug {
				fmt.Println("Ignoring: %s", parseString)
			}
			continue
		}
		matched, _ = regexp.MatchString(".*=.*", parseString)
		if matched {
			parseLine(parseString)
		}
	}

	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}
}

func parseLine(line string) {
	s := regexp.MustCompile("=").Split(line, 2)
	key := camelKey(strings.TrimSpace(s[0]))
	value := strings.TrimSpace(s[1])
	if value != "" {
		fmt.Printf("jaegerdb -a %s -v '%s'\n", key, value)
	}
}

func camelKey(src string) string {
	// From: https://github.com/etgryphon/stringUp

	var camelingRegex = regexp.MustCompile("[0-9A-Za-z]+")

	byteSrc := []byte(src)
	chunks := camelingRegex.FindAll(byteSrc, -1)
	for idx, val := range chunks {
		chunks[idx] = bytes.Title(val)
	}
	return string(bytes.Join(chunks, nil))
}
<commit_msg>Add output lines for templates<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

const jaegerTemplateExtension = ".jgrt"
const jaegerDescription = "JaegerH - Jaeger Helper program\n\nJaeger is a JSON encoded GPG encrypted key value store. It is useful for separating development with operations and keeping configuration files secure."
const jaegerQuote = "\"Stacker Pentecost: Haven't you heard Mr. Beckett? The world is coming to an end. So where would you rather die? Here? Or in a Jaeger!\" - Pacific Rim"
const jaegerRecommendedUsage = "RECOMMENDED:\n    jaegerh -i file.txt.jgrt"

var debug debugging = false

type debugging bool

func (d debugging) Printf(format string, args ...interface{}) {
	// From: https://groups.google.com/forum/#!msg/golang-nuts/gU7oQGoCkmg/BNIl-TqB-4wJ
	if d {
		log.Printf(format, args...)
	}
}

func main() {
	// Define flags
	var (
		debugFlag     = flag.Bool("d", false, "Enable Debug")
		inputTemplate = flag.String("i", "", "Input Template file. eg. file.txt.jgrt")
	)

	flag.Usage = func() {
		fmt.Printf("%s\n%s\n\n%s\n\n", jaegerDescription, jaegerQuote, jaegerRecommendedUsage)
		fmt.Fprintf(os.Stderr, "OPTIONS:\n")
		flag.PrintDefaults()
	}

	flag.Parse()

	if *debugFlag {
		debug = true
	}

	if *inputTemplate == "" {
		assumedTemplate, err := checkExistsJaegerT()
		if err != nil {
			flag.Usage()
			log.Fatalf("\n\nError: %s", err)
		}
		*inputTemplate = assumedTemplate
	}

	processInputFile(inputTemplate)
}

func checkExistsJaegerT() (string, error) {
	// Check that one template file is in the current directory and use that file
	files, err := filepath.Glob("*.jgrt")
	if err != nil {
		return "", err
	}
	if len(files) == 1 {
		return files[0], nil
	}
	return "", fmt.Errorf("No input template file specified")
}

func processInputFile(inputFile *string) {
	file, err := os.Open(*inputFile)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		parseString := scanner.Text()
		matched, _ := regexp.MatchString("^[[:space:]]*#", parseString)
		if matched {
			if debug {
				fmt.Println("Ignoring: %s", parseString)
			}
			continue
		}
		matched, _ = regexp.MatchString(".*=.*", parseString)
		if matched {
			parseLine(parseString)
		}
	}

	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}
}

func parseLine(line string) {
	s := regexp.MustCompile("=").Split(line, 2)
	rawKey := strings.TrimSpace(s[0])
	key := camelKey(rawKey)
	value := strings.TrimSpace(s[1])
	if value != "" {
		fmt.Printf("jaegerdb -a %s -v '%s' # %s = {{{.%s}}}\n", key, value, rawKey, key)
	}
}

func camelKey(src string) string {
	// From: https://github.com/etgryphon/stringUp

	var camelingRegex = regexp.MustCompile("[0-9A-Za-z]+")

	byteSrc := []byte(src)
	chunks := camelingRegex.FindAll(byteSrc, -1)
	for idx, val := range chunks {
		chunks[idx] = bytes.Title(val)
	}
	return string(bytes.Join(chunks, nil))
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build go1.8

package proxy

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"mime/multipart"
	"net/http"
	"reflect"
	"strings"

	"github.com/google/martian/martianlog"
)

// ForReplaying returns a Proxy configured to replay.
func ForReplaying(filename string, port int) (*Proxy, error) {
	p, err := newProxy(filename)
	if err != nil {
		return nil, err
	}
	calls, initial, err := readLog(filename)
	if err != nil {
		return nil, err
	}
	p.mproxy.SetRoundTripper(replayRoundTripper{
		calls:         calls,
		ignoreHeaders: p.ignoreHeaders,
	})
	p.Initial = initial

	// Debug logging.
	// TODO(jba): factor out from here and ForRecording.
	logger := martianlog.NewLogger()
	logger.SetDecode(true)
	p.mproxy.SetRequestModifier(logger)
	p.mproxy.SetResponseModifier(logger)

	if err := p.start(port); err != nil {
		return nil, err
	}
	return p, nil
}

// A call is an HTTP request and its matching response.
type call struct {
	req     *Request
	reqBody *requestBody // parsed request body
	res     *Response
}

func readLog(filename string) ([]*call, []byte, error) {
	bytes, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, nil, err
	}
	var lg Log
	if err := json.Unmarshal(bytes, &lg); err != nil {
		return nil, nil, fmt.Errorf("%s: %v", filename, err)
	}
	if lg.Version != LogVersion {
		return nil, nil, fmt.Errorf("httpreplay proxy: read log version %s but current version is %s",
			lg.Version, LogVersion)
	}
	ignoreIDs := map[string]bool{} // IDs of requests to ignore
	callsByID := map[string]*call{}
	var calls []*call
	for _, e := range lg.Entries {
		if ignoreIDs[e.ID] {
			continue
		}
		c, ok := callsByID[e.ID]
		switch {
		case !ok:
			if e.Request == nil {
				return nil, nil, fmt.Errorf("first entry for ID %s does not have a request", e.ID)
			}
			if e.Request.Method == "CONNECT" {
				// Ignore CONNECT methods.
				ignoreIDs[e.ID] = true
			} else {
				reqBody, err := newRequestBodyFromLog(e.Request)
				if err != nil {
					return nil, nil, err
				}
				c := &call{e.Request, reqBody, e.Response}
				calls = append(calls, c)
				callsByID[e.ID] = c
			}
		case e.Request != nil:
			if e.Response != nil {
				return nil, nil, errors.New("HAR entry has both request and response")
			}
			c.req = e.Request
		case e.Response != nil:
			c.res = e.Response
		default:
			return nil, nil, errors.New("HAR entry has neither request nor response")
		}
	}
	for _, c := range calls {
		if c.req == nil || c.res == nil {
			return nil, nil, fmt.Errorf("missing request or response: %+v", c)
		}
	}
	return calls, lg.Initial, nil
}

type replayRoundTripper struct {
	calls         []*call
	ignoreHeaders map[string]bool
}

func (r replayRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	reqBody, err := newRequestBodyFromHTTP(req)
	if err != nil {
		return nil, err
	}
	for i, call := range r.calls {
		if call == nil {
			continue
		}
		if requestsMatch(req, reqBody, call.req, call.reqBody, r.ignoreHeaders) {
			r.calls[i] = nil // nil out this call so we don't reuse it
			return toHTTPResponse(call.res, req), nil
		}
	}
	return nil, fmt.Errorf("no matching request for %+v", req)
}

// Headers that shouldn't be compared, because they may differ on different executions
// of the same code, or may not be present during record or replay.
var ignoreHeaders = map[string]bool{}

func init() {
	// Sensitive headers are redacted in the log, so they won't be equal to incoming values.
	for h := range sensitiveHeaders {
		ignoreHeaders[h] = true
	}
	for _, h := range []string{
		"Content-Type", // handled by requestBody
		"Connection",
		"Date",
		"Host",
		"Transfer-Encoding",
		"Via",
		"X-Forwarded-For",
		"X-Forwarded-Host",
		"X-Forwarded-Proto",
		"X-Forwarded-Url",
		"X-Cloud-Trace-Context", // OpenCensus traces have a random ID
		"X-Goog-Api-Client",     // can differ for, e.g., different Go versions
	} {
		ignoreHeaders[h] = true
	}
}

// Report whether the incoming request in matches the candidate request cand.
func requestsMatch(in *http.Request, inBody *requestBody, cand *Request, candBody *requestBody, ignoreHeaders map[string]bool) bool {
	if in.Method != cand.Method {
		return false
	}
	if in.URL.String() != cand.URL {
		return false
	}
	if !inBody.equal(candBody) {
		return false
	}
	// Check headers last. See DebugHeaders.
	return headersMatch(in.Header, cand.Header, ignoreHeaders)
}

// A requestBody represents the body of a request. If the content type is multipart, the
// body is split into parts.
//
// The replaying proxy needs to understand multipart bodies because the boundaries are
// generated randomly, so we can't just compare the entire bodies for equality.
type requestBody struct {
	mediaType string   // the media type part of the Content-Type header
	parts     [][]byte // the parts of the body, or just a single []byte if not multipart
}

func newRequestBodyFromHTTP(req *http.Request) (*requestBody, error) {
	defer req.Body.Close()
	return newRequestBody(req.Header.Get("Content-Type"), req.Body)
}

func newRequestBodyFromLog(req *Request) (*requestBody, error) {
	if req.Body == nil {
		return nil, nil
	}
	return newRequestBody(req.Header.Get("Content-Type"), bytes.NewReader(req.Body))
}

// newRequestBody parses the Content-Type header, reads the body, and splits it into
// parts if necessary.
func newRequestBody(contentType string, body io.Reader) (*requestBody, error) {
	if contentType == "" {
		// No content-type header. There should not be a body.
		if _, err := body.Read(make([]byte, 1)); err != io.EOF {
			return nil, errors.New("no Content-Type, but body")
		}
		return nil, nil
	}
	mediaType, params, err := mime.ParseMediaType(contentType)
	if err != nil {
		return nil, err
	}
	rb := &requestBody{mediaType: mediaType}
	if strings.HasPrefix(mediaType, "multipart/") {
		mr := multipart.NewReader(body, params["boundary"])
		for {
			p, err := mr.NextPart()
			if err == io.EOF {
				break
			}
			if err != nil {
				return nil, err
			}
			part, err := ioutil.ReadAll(p)
			if err != nil {
				return nil, err
			}
			// TODO(jba): care about part headers?
			rb.parts = append(rb.parts, part)
		}
	} else {
		bytes, err := ioutil.ReadAll(body)
		if err != nil {
			return nil, err
		}
		rb.parts = [][]byte{bytes}
	}
	return rb, nil
}

func (r1 *requestBody) equal(r2 *requestBody) bool {
	if r1 == nil || r2 == nil {
		return r1 == r2
	}
	if r1.mediaType != r2.mediaType {
		return false
	}
	if len(r1.parts) != len(r2.parts) {
		return false
	}
	for i, p1 := range r1.parts {
		if !bytes.Equal(p1, r2.parts[i]) {
			return false
		}
	}
	return true
}

// DebugHeaders helps to determine whether a header should be ignored.
// When true, if requests have the same method, URL and body but differ
// in a header, the first mismatched header is logged.
var DebugHeaders = false

func headersMatch(in, cand http.Header, ignores map[string]bool) bool {
	for k1, v1 := range in {
		if ignores[k1] {
			continue
		}
		v2 := cand[k1]
		if v2 == nil {
			if DebugHeaders {
				log.Printf("header %s: present in incoming request but not candidate", k1)
			}
			return false
		}
		if !reflect.DeepEqual(v1, v2) {
			if DebugHeaders {
				log.Printf("header %s: incoming %v, candidate %v", k1, v1, v2)
			}
			return false
		}
	}
	for k2 := range cand {
		if ignores[k2] {
			continue
		}
		if in[k2] == nil {
			if DebugHeaders {
				log.Printf("header %s: not in incoming request but present in candidate", k2)
			}
			return false
		}
	}
	return true
}
<commit_msg>httpreplay: fix race<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build go1.8

package proxy

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"mime/multipart"
	"net/http"
	"reflect"
	"strings"
	"sync"

	"github.com/google/martian/martianlog"
)

// ForReplaying returns a Proxy configured to replay.
func ForReplaying(filename string, port int) (*Proxy, error) {
	p, err := newProxy(filename)
	if err != nil {
		return nil, err
	}
	calls, initial, err := readLog(filename)
	if err != nil {
		return nil, err
	}
	p.mproxy.SetRoundTripper(&replayRoundTripper{
		calls:         calls,
		ignoreHeaders: p.ignoreHeaders,
	})
	p.Initial = initial

	// Debug logging.
	// TODO(jba): factor out from here and ForRecording.
	logger := martianlog.NewLogger()
	logger.SetDecode(true)
	p.mproxy.SetRequestModifier(logger)
	p.mproxy.SetResponseModifier(logger)

	if err := p.start(port); err != nil {
		return nil, err
	}
	return p, nil
}

// A call is an HTTP request and its matching response.
type call struct {
	req     *Request
	reqBody *requestBody // parsed request body
	res     *Response
}

func readLog(filename string) ([]*call, []byte, error) {
	bytes, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, nil, err
	}
	var lg Log
	if err := json.Unmarshal(bytes, &lg); err != nil {
		return nil, nil, fmt.Errorf("%s: %v", filename, err)
	}
	if lg.Version != LogVersion {
		return nil, nil, fmt.Errorf("httpreplay proxy: read log version %s but current version is %s",
			lg.Version, LogVersion)
	}
	ignoreIDs := map[string]bool{} // IDs of requests to ignore
	callsByID := map[string]*call{}
	var calls []*call
	for _, e := range lg.Entries {
		if ignoreIDs[e.ID] {
			continue
		}
		c, ok := callsByID[e.ID]
		switch {
		case !ok:
			if e.Request == nil {
				return nil, nil, fmt.Errorf("first entry for ID %s does not have a request", e.ID)
			}
			if e.Request.Method == "CONNECT" {
				// Ignore CONNECT methods.
				ignoreIDs[e.ID] = true
			} else {
				reqBody, err := newRequestBodyFromLog(e.Request)
				if err != nil {
					return nil, nil, err
				}
				c := &call{e.Request, reqBody, e.Response}
				calls = append(calls, c)
				callsByID[e.ID] = c
			}
		case e.Request != nil:
			if e.Response != nil {
				return nil, nil, errors.New("HAR entry has both request and response")
			}
			c.req = e.Request
		case e.Response != nil:
			c.res = e.Response
		default:
			return nil, nil, errors.New("HAR entry has neither request nor response")
		}
	}
	for _, c := range calls {
		if c.req == nil || c.res == nil {
			return nil, nil, fmt.Errorf("missing request or response: %+v", c)
		}
	}
	return calls, lg.Initial, nil
}

type replayRoundTripper struct {
	mu            sync.Mutex
	calls         []*call
	ignoreHeaders map[string]bool
}

func (r *replayRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	reqBody, err := newRequestBodyFromHTTP(req)
	if err != nil {
		return nil, err
	}
	r.mu.Lock()
	defer r.mu.Unlock()
	for i, call := range r.calls {
		if call == nil {
			continue
		}
		if requestsMatch(req, reqBody, call.req, call.reqBody, r.ignoreHeaders) {
			r.calls[i] = nil // nil out this call so we don't reuse it
			return toHTTPResponse(call.res, req), nil
		}
	}
	return nil, fmt.Errorf("no matching request for %+v", req)
}

// Headers that shouldn't be compared, because they may differ on different executions
// of the same code, or may not be present during record or replay.
var ignoreHeaders = map[string]bool{}

func init() {
	// Sensitive headers are redacted in the log, so they won't be equal to incoming values.
	for h := range sensitiveHeaders {
		ignoreHeaders[h] = true
	}
	for _, h := range []string{
		"Content-Type", // handled by requestBody
		"Connection",
		"Date",
		"Host",
		"Transfer-Encoding",
		"Via",
		"X-Forwarded-For",
		"X-Forwarded-Host",
		"X-Forwarded-Proto",
		"X-Forwarded-Url",
		"X-Cloud-Trace-Context", // OpenCensus traces have a random ID
		"X-Goog-Api-Client",     // can differ for, e.g., different Go versions
	} {
		ignoreHeaders[h] = true
	}
}

// Report whether the incoming request in matches the candidate request cand.
func requestsMatch(in *http.Request, inBody *requestBody, cand *Request, candBody *requestBody, ignoreHeaders map[string]bool) bool {
	if in.Method != cand.Method {
		return false
	}
	if in.URL.String() != cand.URL {
		return false
	}
	if !inBody.equal(candBody) {
		return false
	}
	// Check headers last. See DebugHeaders.
	return headersMatch(in.Header, cand.Header, ignoreHeaders)
}

// A requestBody represents the body of a request. If the content type is multipart, the
// body is split into parts.
//
// The replaying proxy needs to understand multipart bodies because the boundaries are
// generated randomly, so we can't just compare the entire bodies for equality.
type requestBody struct {
	mediaType string   // the media type part of the Content-Type header
	parts     [][]byte // the parts of the body, or just a single []byte if not multipart
}

func newRequestBodyFromHTTP(req *http.Request) (*requestBody, error) {
	defer req.Body.Close()
	return newRequestBody(req.Header.Get("Content-Type"), req.Body)
}

func newRequestBodyFromLog(req *Request) (*requestBody, error) {
	if req.Body == nil {
		return nil, nil
	}
	return newRequestBody(req.Header.Get("Content-Type"), bytes.NewReader(req.Body))
}

// newRequestBody parses the Content-Type header, reads the body, and splits it into
// parts if necessary.
func newRequestBody(contentType string, body io.Reader) (*requestBody, error) {
	if contentType == "" {
		// No content-type header. There should not be a body.
		if _, err := body.Read(make([]byte, 1)); err != io.EOF {
			return nil, errors.New("no Content-Type, but body")
		}
		return nil, nil
	}
	mediaType, params, err := mime.ParseMediaType(contentType)
	if err != nil {
		return nil, err
	}
	rb := &requestBody{mediaType: mediaType}
	if strings.HasPrefix(mediaType, "multipart/") {
		mr := multipart.NewReader(body, params["boundary"])
		for {
			p, err := mr.NextPart()
			if err == io.EOF {
				break
			}
			if err != nil {
				return nil, err
			}
			part, err := ioutil.ReadAll(p)
			if err != nil {
				return nil, err
			}
			// TODO(jba): care about part headers?
			rb.parts = append(rb.parts, part)
		}
	} else {
		bytes, err := ioutil.ReadAll(body)
		if err != nil {
			return nil, err
		}
		rb.parts = [][]byte{bytes}
	}
	return rb, nil
}

func (r1 *requestBody) equal(r2 *requestBody) bool {
	if r1 == nil || r2 == nil {
		return r1 == r2
	}
	if r1.mediaType != r2.mediaType {
		return false
	}
	if len(r1.parts) != len(r2.parts) {
		return false
	}
	for i, p1 := range r1.parts {
		if !bytes.Equal(p1, r2.parts[i]) {
			return false
		}
	}
	return true
}

// DebugHeaders helps to determine whether a header should be ignored.
// When true, if requests have the same method, URL and body but differ
// in a header, the first mismatched header is logged.
var DebugHeaders = false

func headersMatch(in, cand http.Header, ignores map[string]bool) bool {
	for k1, v1 := range in {
		if ignores[k1] {
			continue
		}
		v2 := cand[k1]
		if v2 == nil {
			if DebugHeaders {
				log.Printf("header %s: present in incoming request but not candidate", k1)
			}
			return false
		}
		if !reflect.DeepEqual(v1, v2) {
			if DebugHeaders {
				log.Printf("header %s: incoming %v, candidate %v", k1, v1, v2)
			}
			return false
		}
	}
	for k2 := range cand {
		if ignores[k2] {
			continue
		}
		if in[k2] == nil {
			if DebugHeaders {
				log.Printf("header %s: not in incoming request but present in candidate", k2)
			}
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>// Copyright 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package terminalapi

// color_mode.go defines the terminal color modes.

// ColorMode represents
type ColorMode int

// String implements fmt.Stringer()
func (cm ColorMode) String() string {
	if n, ok := colorModeNames[cm]; ok {
		return n
	}
	return "ColorModeUnknown"
}

// colorModeNames maps ColorMode values to human readable names.
var colorModeNames = map[ColorMode]string{
	ColorModeNormal:    "ColorModeNormal",
	ColorMode256:       "ColorMode256",
	ColorMode216:       "ColorMode216",
	ColorModeGrayscale: "ColorModeGrayscale",
}

// Supported color modes.
const (
	// ColorModeNormal supports 8 "system" colors.
	// These are defined as constants in the cell package.
	ColorModeNormal ColorMode = iota

	// ColorMode256 enables using any of the 256 terminal colors.
	//     0-7: the 8 "system" colors accessible in ColorModeNormal.
	//    8-15: the 8 "bright system" colors.
	//  16-231: the 216 different terminal colors.
	// 232-255: the 24 different shades of grey.
	ColorMode256

	// ColorMode216 supports only the third range of the ColorMode256, i.e the
	// 216 different terminal colors. However in this mode the colors are zero
	// based, so the called doesn't need to provide an offset.
	ColorMode216

	// ColorModeGrayscale supports only the fourth range of the ColorMode256,
	// i.e the 24 different shades of grey. However in this mode the colors are
	// zero based, so the called doesn't need to provide an offset.
	ColorModeGrayscale
)
<commit_msg>Two typos s/called/caller<commit_after>// Copyright 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package terminalapi

// color_mode.go defines the terminal color modes.

// ColorMode represents
type ColorMode int

// String implements fmt.Stringer()
func (cm ColorMode) String() string {
	if n, ok := colorModeNames[cm]; ok {
		return n
	}
	return "ColorModeUnknown"
}

// colorModeNames maps ColorMode values to human readable names.
var colorModeNames = map[ColorMode]string{
	ColorModeNormal:    "ColorModeNormal",
	ColorMode256:       "ColorMode256",
	ColorMode216:       "ColorMode216",
	ColorModeGrayscale: "ColorModeGrayscale",
}

// Supported color modes.
const (
	// ColorModeNormal supports 8 "system" colors.
	// These are defined as constants in the cell package.
	ColorModeNormal ColorMode = iota

	// ColorMode256 enables using any of the 256 terminal colors.
	//     0-7: the 8 "system" colors accessible in ColorModeNormal.
	//    8-15: the 8 "bright system" colors.
	//  16-231: the 216 different terminal colors.
	// 232-255: the 24 different shades of grey.
	ColorMode256

	// ColorMode216 supports only the third range of the ColorMode256, i.e the
	// 216 different terminal colors. However in this mode the colors are zero
	// based, so the caller doesn't need to provide an offset.
	ColorMode216

	// ColorModeGrayscale supports only the fourth range of the ColorMode256,
	// i.e the 24 different shades of grey. However in this mode the colors are
	// zero based, so the caller doesn't need to provide an offset.
	ColorModeGrayscale
)
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"
	"log"
)

// EvalRefresh is an EvalNode implementation that does a refresh for
// a resource.
type EvalRefresh struct {
	Provider *ResourceProvider
	State    **InstanceState
	Info     *InstanceInfo
	Output   **InstanceState
}

// TODO: test
func (n *EvalRefresh) Eval(ctx EvalContext) (interface{}, error) {
	provider := *n.Provider
	state := *n.State

	// If we have no state, we don't do any refreshing
	if state == nil {
		log.Printf("[DEBUG] refresh: %s: no state, not refreshing", n.Info.Id)
		return nil, nil
	}

	// Call pre-refresh hook
	err := ctx.Hook(func(h Hook) (HookAction, error) {
		return h.PreRefresh(n.Info, state)
	})
	if err != nil {
		return nil, err
	}

	// Refresh!
	state, err = provider.Refresh(n.Info, state)
	if err != nil {
		return nil, fmt.Errorf("%s: %s", n.Info.Id, err.Error())
	}

	// Call post-refresh hook
	err = ctx.Hook(func(h Hook) (HookAction, error) {
		return h.PostRefresh(n.Info, state)
	})
	if err != nil {
		return nil, err
	}

	log.Printf("STATE: %#v", state)
	if n.Output != nil {
		*n.Output = state
	}

	return nil, nil
}
<commit_msg>providers/aws: security group import imports rules<commit_after>package terraform

import (
	"fmt"
	"log"
)

// EvalRefresh is an EvalNode implementation that does a refresh for
// a resource.
type EvalRefresh struct {
	Provider *ResourceProvider
	State    **InstanceState
	Info     *InstanceInfo
	Output   **InstanceState
}

// TODO: test
func (n *EvalRefresh) Eval(ctx EvalContext) (interface{}, error) {
	provider := *n.Provider
	state := *n.State

	// If we have no state, we don't do any refreshing
	if state == nil {
		log.Printf("[DEBUG] refresh: %s: no state, not refreshing", n.Info.Id)
		return nil, nil
	}

	// Call pre-refresh hook
	err := ctx.Hook(func(h Hook) (HookAction, error) {
		return h.PreRefresh(n.Info, state)
	})
	if err != nil {
		return nil, err
	}

	// Refresh!
	state, err = provider.Refresh(n.Info, state)
	if err != nil {
		return nil, fmt.Errorf("%s: %s", n.Info.Id, err.Error())
	}

	// Call post-refresh hook
	err = ctx.Hook(func(h Hook) (HookAction, error) {
		return h.PostRefresh(n.Info, state)
	})
	if err != nil {
		return nil, err
	}

	if n.Output != nil {
		*n.Output = state
	}

	return nil, nil
}
<|endoftext|>


<commit_before>package gorums

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"testing"
)

const (
	gorumsBaseImport  = "github.com/relab/gorums"
	devImport         = gorumsBaseImport + "/" + devFolder
	testdataDevImport = gorumsBaseImport + "/" + testdataFolder + "/" + devFolder

	devFolder       = "dev"
	testdataFolder  = "testdata"
	regGoldenFolder = "register_golden"

	regProtoFile = "register.proto"
	regPbGoFile  = "register.pb.go"

	devRegProtoRelPath = devFolder + "/" + regProtoFile
	tdRegProtoRelPath  = testdataFolder + "/" + regGoldenFolder + "/" + regProtoFile
	tdRegPbGoRelPath   = testdataFolder + "/" + regGoldenFolder + "/" + regPbGoFile

	protoc        = "protoc"
	protocIFlag   = "-I=../../../:."
	protocOutFlag = "--gorums_out=plugins=grpc+gorums:"
)

func run(t *testing.T, name string, args ...string) {
	cmd := exec.Command(name, args...)
	cmd.Env = append(cmd.Env, os.Environ()...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func runAndCaptureOutput(command string, args ...string) ([]byte, error) {
	cmd := exec.Command(command, args...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("%v\n%s", err, out)
	}
	return bytes.TrimSuffix(out, []byte{'\n'}), nil
}

const (
	protocVersionPrefix  = "libprotoc "
	currentProtocVersion = "3.2.0"
)

func checkProtocVersion(t *testing.T) {
	out, err := runAndCaptureOutput("protoc", "--version")
	if err != nil {
		t.Skipf("skipping test due to protoc error: %v", err)
	}
	gotVersion := string(out)
	gotVersion = strings.TrimPrefix(gotVersion, protocVersionPrefix)
	if gotVersion != currentProtocVersion {
		t.Skipf("skipping test due to old protoc version, got %q, required is %q", gotVersion, currentProtocVersion)
	}
}
<commit_msg>gorums: update currentProtocVersion to 3.3.1<commit_after>package gorums

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"testing"
)

const (
	gorumsBaseImport  = "github.com/relab/gorums"
	devImport         = gorumsBaseImport + "/" + devFolder
	testdataDevImport = gorumsBaseImport + "/" + testdataFolder + "/" + devFolder

	devFolder       = "dev"
	testdataFolder  = "testdata"
	regGoldenFolder = "register_golden"

	regProtoFile = "register.proto"
	regPbGoFile  = "register.pb.go"

	devRegProtoRelPath = devFolder + "/" + regProtoFile
	tdRegProtoRelPath  = testdataFolder + "/" + regGoldenFolder + "/" + regProtoFile
	tdRegPbGoRelPath   = testdataFolder + "/" + regGoldenFolder + "/" + regPbGoFile

	protoc        = "protoc"
	protocIFlag   = "-I=../../../:."
	protocOutFlag = "--gorums_out=plugins=grpc+gorums:"
)

func run(t *testing.T, name string, args ...string) {
	cmd := exec.Command(name, args...)
	cmd.Env = append(cmd.Env, os.Environ()...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func runAndCaptureOutput(command string, args ...string) ([]byte, error) {
	cmd := exec.Command(command, args...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("%v\n%s", err, out)
	}
	return bytes.TrimSuffix(out, []byte{'\n'}), nil
}

const (
	protocVersionPrefix  = "libprotoc "
	currentProtocVersion = "3.3.1"
)

func checkProtocVersion(t *testing.T) {
	out, err := runAndCaptureOutput("protoc", "--version")
	if err != nil {
		t.Skipf("skipping test due to protoc error: %v", err)
	}
	gotVersion := string(out)
	gotVersion = strings.TrimPrefix(gotVersion, protocVersionPrefix)
	if gotVersion != currentProtocVersion {
		t.Skipf("skipping test due to old protoc version, got %q, required is %q", gotVersion, currentProtocVersion)
	}
}
<|endoftext|>


<commit_before>package control

import (
	"github.com/emicklei/melrose/core"
	"github.com/emicklei/melrose/notify"
)

type Knob struct {
	vars     core.VariableStorage
	deviceID int
	channel  int
	number   int
	// changes
	currentValue int
}

func NewKnob(ctx core.Context, deviceID, channel, number int) *Knob {
	k := &Knob{vars: ctx.Variables(), deviceID: deviceID, channel: channel, number: number}
	ctx.Device().Listen(deviceID, k, true)
	return k
}

func (k *Knob) NoteOn(n core.Note) {
	if core.IsDebug() {
		notify.Debugf("knob.NoteOn %v", n)
	}
}
func (k *Knob) NoteOff(n core.Note) {
	if core.IsDebug() {
		notify.Debugf("knob.NoteOff %v", n)
	}
}
func (k *Knob) ControlChange(channel, number, value int) {
	if core.IsDebug() {
		notify.Debugf("knob.ControlChange ch=%d,nr=%d,val=%d", channel, number, value)
	}
	// TODO check channel
	if number != number {
		return
	}
	k.currentValue = value
}

func (k *Knob) Value() interface{} {
	return k.currentValue
}
<commit_msg>simplify knob<commit_after>package control

import (
	"fmt"

	"github.com/emicklei/melrose/core"
	"github.com/emicklei/melrose/notify"
)

type Knob struct {
	deviceID int
	channel  int
	number   int
	// changes
	currentValue int
}

func NewKnob(deviceID, channel, number int) *Knob {
	return &Knob{deviceID: deviceID, channel: channel, number: number}
}

// Inspect is part of Inspectable
func (k *Knob) Inspect(i core.Inspection) {
	i.Properties["device"] = k.deviceID
	i.Properties["channel"] = k.channel
	i.Properties["number"] = k.number
	i.Properties["currentValue"] = k.currentValue
}

// Storex is part of core.Storable
func (k *Knob) Storex() string {
	return fmt.Sprintf("knob(%d,%d)", k.deviceID, k.number)
}

func (k *Knob) NoteOn(n core.Note) {
	if core.IsDebug() {
		notify.Debugf("knob.NoteOn %v", n)
	}
}
func (k *Knob) NoteOff(n core.Note) {
	if core.IsDebug() {
		notify.Debugf("knob.NoteOff %v", n)
	}
}
func (k *Knob) ControlChange(channel, number, value int) {
	if core.IsDebug() {
		notify.Debugf("knob.ControlChange ch=%d,nr=%d,val=%d", channel, number, value)
	}
	// TODO check channel
	if number != number {
		return
	}
	k.currentValue = value
}

func (k *Knob) Value() interface{} {
	return k.currentValue
}
<|endoftext|>


<commit_before>package queue

import (
	"bytes"
	"github.com/FireEater64/gamq/message"
	"github.com/onsi/gomega"
	"testing"
	"time"
)

func TestQueue_CanSendAndReceiveBasicMessages(t *testing.T) {
	underTest := NewQueue("TestQueue")

	testMessagePayload := []byte("TestMessage")
	underTest.InputChannel <- (message.NewHeaderlessMessage(&testMessagePayload))

	receivedMessage := <-underTest.OutputChannel

	if !bytes.Equal(*receivedMessage.Body, testMessagePayload) {
		t.Fail()
	}
}

func TestQueue_ReceiveBeforeSend_ReturnsExpectedResult(t *testing.T) {
	gomega.RegisterTestingT(t)

	underTest := NewQueue("TestQueue")

	var receivedMessage *message.Message
	go func() {
		receivedMessage = <-underTest.OutputChannel
	}()

	time.Sleep(time.Millisecond * 10)

	testMessagePayload := []byte("TestMessage")
	underTest.InputChannel <- (message.NewHeaderlessMessage(&testMessagePayload))

	gomega.Eventually(func() *message.Message {
		return receivedMessage
	}).Should(gomega.Not(gomega.BeNil()))

	if !bytes.Equal(*receivedMessage.Body, testMessagePayload) {
		t.Fail()
	}
}

func TestQueue_EvenNumberOfPushesAndPops_GivesZeroFinalLength(t *testing.T) {
	underTest := NewQueue("Test")
	numberOfRounds := 200

	for i := 0; i < numberOfRounds; i++ {
		dummyMessagePayLoad := []byte{byte(i)}
		dummyMessage := message.NewHeaderlessMessage(&dummyMessagePayLoad)
		underTest.InputChannel <- dummyMessage
	}

	gomega.Eventually(func() int {
		return underTest.length
	}).Should(gomega.Equal(numberOfRounds))

	for i := 0; i < numberOfRounds; i++ {
		message := <-underTest.OutputChannel
		if int((*message.Body)[0]) != i {
			t.Logf("Expected %d, got %d", i, int((*message.Body)[0]))
			t.FailNow()
		}
	}

	gomega.Eventually(func() int {
		return underTest.length
	}).Should(gomega.Equal(0))
}

func BenchmarkQueueSendRecv(b *testing.B) {
	dummyMessagePayLoad := []byte("Test")
	dummyMessage := message.NewHeaderlessMessage(&dummyMessagePayLoad)

	underTest := NewQueue("Test")

	for i := 0; i < b.N; i++ {
		underTest.InputChannel <- dummyMessage
	}

	for i := 0; i < b.N; i++ {
		_ = <-underTest.OutputChannel
	}
}
<commit_msg>More unit tests<commit_after>package queue

import (
	"bytes"
	"testing"
	"time"

	"github.com/FireEater64/gamq/message"
	"github.com/onsi/gomega"
)

func TestQueue_CanSendAndReceiveBasicMessages(t *testing.T) {
	underTest := NewQueue("TestQueue")

	testMessagePayload := []byte("TestMessage")
	underTest.InputChannel <- (message.NewHeaderlessMessage(&testMessagePayload))

	receivedMessage := <-underTest.OutputChannel

	if !bytes.Equal(*receivedMessage.Body, testMessagePayload) {
		t.Fail()
	}
}

func TestQueue_ReceiveBeforeSend_ReturnsExpectedResult(t *testing.T) {
	gomega.RegisterTestingT(t)

	underTest := NewQueue("TestQueue")

	var receivedMessage *message.Message
	go func() {
		receivedMessage = <-underTest.OutputChannel
	}()

	time.Sleep(time.Millisecond * 10)

	testMessagePayload := []byte("TestMessage")
	underTest.InputChannel <- (message.NewHeaderlessMessage(&testMessagePayload))

	gomega.Eventually(func() *message.Message {
		return receivedMessage
	}).Should(gomega.Not(gomega.BeNil()))

	if !bytes.Equal(*receivedMessage.Body, testMessagePayload) {
		t.Fail()
	}
}

func TestQueue_CloseQueueImmediately_ThrowsNoErrors(t *testing.T) {
	gomega.RegisterTestingT(t)
	underTest := NewQueue("Test")

	close(underTest.InputChannel)

	gomega.Eventually(func() bool {
		_, open := <-underTest.OutputChannel
		return open
	}).Should(gomega.BeFalse())
}

func TestQueue_CloseChannelBeforeConsuming_GivesZeroFinalLength(t *testing.T) {
	underTest := NewQueue("Test")
	numberOfRounds := 10

	for i := 0; i < numberOfRounds; i++ {
		dummyMessagePayLoad := []byte{byte(i)}
		dummyMessage := message.NewHeaderlessMessage(&dummyMessagePayLoad)
		underTest.InputChannel <- dummyMessage
	}

	gomega.Eventually(func() int {
		return underTest.length
	}).Should(gomega.Equal(numberOfRounds))

	close(underTest.InputChannel)

	for i := 0; i < numberOfRounds; i++ {
		message := <-underTest.OutputChannel
		if int((*message.Body)[0]) != i {
			t.Logf("Expected %d, got %d", i, int((*message.Body)[0]))
			t.FailNow()
		}
	}

	gomega.Eventually(func() int {
		return underTest.PendingMessages()
	}).Should(gomega.Equal(0))
}

func TestQueue_EvenNumberOfPushesAndPops_GivesZeroFinalLength(t *testing.T) {
	underTest := NewQueue("Test")
	numberOfRounds := 200

	for i := 0; i < numberOfRounds; i++ {
		dummyMessagePayLoad := []byte{byte(i)}
		dummyMessage := message.NewHeaderlessMessage(&dummyMessagePayLoad)
		underTest.InputChannel <- dummyMessage
	}

	gomega.Eventually(func() int {
		return underTest.length
	}).Should(gomega.Equal(numberOfRounds))

	for i := 0; i < numberOfRounds; i++ {
		message := <-underTest.OutputChannel
		if int((*message.Body)[0]) != i {
			t.Logf("Expected %d, got %d", i, int((*message.Body)[0]))
			t.FailNow()
		}
	}

	gomega.Eventually(func() int {
		return underTest.length
	}).Should(gomega.Equal(0))
}

func BenchmarkQueueSendRecv(b *testing.B) {
	dummyMessagePayLoad := []byte("Test")
	dummyMessage := message.NewHeaderlessMessage(&dummyMessagePayLoad)

	underTest := NewQueue("Test")

	for i := 0; i < b.N; i++ {
		underTest.InputChannel <- dummyMessage
	}

	for i := 0; i < b.N; i++ {
		_ = <-underTest.OutputChannel
	}
}
<|endoftext|>


<commit_before>package conversion

import (
	"testing"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/equality"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"

	"sigs.k8s.io/multi-tenancy/incubator/virtualcluster/pkg/apis/tenancy/v1alpha1"
	"sigs.k8s.io/multi-tenancy/incubator/virtualcluster/pkg/syncer/constants"
)

func Test_mutateDownwardAPIField(t *testing.T) {
	aPod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "name",
			Namespace: "ns",
			UID:       types.UID("5033b5b7-104f-11ea-b309-525400c042d5"),
		},
	}

	for _, tt := range []struct {
		name        string
		pod         *v1.Pod
		env         *v1.EnvVar
		expectedEnv *v1.EnvVar
	}{
		{
			name: "env without fieldRef",
			pod:  aPod,
			env: &v1.EnvVar{
				Name:      "env_name",
				Value:     "env_value",
				ValueFrom: nil,
			},
			expectedEnv: &v1.EnvVar{
				Name:      "env_name",
				Value:     "env_value",
				ValueFrom: nil,
			},
		},
		{
			name: "env with other fieldRef",
			pod:  aPod,
			env: &v1.EnvVar{
				Name: "env_name",
				ValueFrom: &v1.EnvVarSource{
					FieldRef: &v1.ObjectFieldSelector{
						APIVersion: "v1",
						FieldPath:  "spec.nodeName",
					},
				},
			},
			expectedEnv: &v1.EnvVar{
				Name: "env_name",
				ValueFrom: &v1.EnvVarSource{
					FieldRef: &v1.ObjectFieldSelector{
						APIVersion: "v1",
						FieldPath:  "spec.nodeName",
					},
				},
			},
		},
		{
			name: "env with metadata.name",
			pod:  aPod,
			env: &v1.EnvVar{
				Name: "env_name",
				ValueFrom: &v1.EnvVarSource{
					FieldRef: &v1.ObjectFieldSelector{
						APIVersion: "v1",
						FieldPath:  "metadata.name",
					},
				},
			},
			expectedEnv: &v1.EnvVar{
				Name:  "env_name",
				Value: aPod.Name,
			},
		},
		{
			name: "env with metadata.namespace",
			pod:  aPod,
			env: &v1.EnvVar{
				Name: "env_name",
				ValueFrom: &v1.EnvVarSource{
					FieldRef: &v1.ObjectFieldSelector{
						APIVersion: "v1",
						FieldPath:  "metadata.namespace",
					},
				},
			},
			expectedEnv: &v1.EnvVar{
				Name:  "env_name",
				Value: aPod.Namespace,
			},
		},
		{
			name: "env with metadata.uid",
			pod:  aPod,
			env: &v1.EnvVar{
				Name: "env_name",
				ValueFrom: &v1.EnvVarSource{
					FieldRef: &v1.ObjectFieldSelector{
						APIVersion: "v1",
						FieldPath:  "metadata.uid",
					},
				},
			},
			expectedEnv: &v1.EnvVar{
				Name:  "env_name",
				Value: string(aPod.UID),
			},
		},
	} {
		t.Run(tt.name, func(tc *testing.T) {
			mutateDownwardAPIField(tt.env, tt.pod)
			if !equality.Semantic.DeepEqual(tt.env, tt.expectedEnv) {
				tc.Errorf("expected env %+v, got %+v", tt.expectedEnv, tt.env)
			}
		})
	}
}

func Test_mutateContainerSecret(t *testing.T) {
	saSecret := &v1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name: "service-token-secret",
			Annotations: map[string]string{
				constants.LabelSecretName: "service-token-secret-tenant",
			},
		},
		Type: v1.SecretTypeOpaque,
	}
	for _, tt := range []struct {
		name              string
		container         *v1.Container
		saSecret          *v1.Secret
		expectedContainer *v1.Container
	}{
		{
			name: "normal case",
			container: &v1.Container{
				VolumeMounts: []v1.VolumeMount{
					{
						Name:      "some-mount",
						MountPath: "/path/to/mount",
						ReadOnly:  true,
					},
					{
						Name:      "service-token-secret-tenant",
						MountPath: "/var/run/secrets/kubernetes.io/serviceaccount",
						ReadOnly:  true,
					},
				},
			},
			saSecret: saSecret,
			expectedContainer: &v1.Container{
				VolumeMounts: []v1.VolumeMount{
					{
						Name:      "some-mount",
						MountPath: "/path/to/mount",
						ReadOnly:  true,
					},
					{
						Name:      "service-token-secret",
						MountPath: "/var/run/secrets/kubernetes.io/serviceaccount",
						ReadOnly:  true,
					},
				},
			},
		},
	} {
		t.Run(tt.name, func(tc *testing.T) {
			mutateContainerSecret(tt.container, tt.saSecret)
			if !equality.Semantic.DeepEqual(tt.container, tt.expectedContainer) {
				tc.Errorf("expected container %+v, got %+v", tt.expectedContainer, tt.container)
			}
		})
	}
}

func TestToClusterKey(t *testing.T) {
	for _, tt := range []struct {
		name        string
		vc          *v1alpha1.VirtualCluster
		expectedKey string
	}{
		{
			name: "normal vc",
			vc: &v1alpha1.VirtualCluster{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "name",
					Namespace: "ns",
					UID:       "d64ea0c0-91f8-46f5-8643-c0cab32ab0cd",
				},
			},
			expectedKey: "ns-fd1b34-name",
		},
	} {
		t.Run(tt.name, func(tc *testing.T) {
			key := ToClusterKey(tt.vc)
			if key != tt.expectedKey {
				tc.Errorf("expected key %s, got %s", tt.expectedKey, key)
			}
		})
	}
}
<commit_msg>fix mutate func ut<commit_after>package conversion

import (
	"testing"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/equality"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"

	"sigs.k8s.io/multi-tenancy/incubator/virtualcluster/pkg/apis/tenancy/v1alpha1"
)

func Test_mutateDownwardAPIField(t *testing.T) {
	aPod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "name",
			Namespace: "ns",
			UID:       types.UID("5033b5b7-104f-11ea-b309-525400c042d5"),
		},
	}

	for _, tt := range []struct {
		name        string
		pod         *v1.Pod
		env         *v1.EnvVar
		expectedEnv *v1.EnvVar
	}{
		{
			name: "env without fieldRef",
			pod:  aPod,
			env: &v1.EnvVar{
				Name:      "env_name",
				Value:     "env_value",
				ValueFrom: nil,
			},
			expectedEnv: &v1.EnvVar{
				Name:      "env_name",
				Value:     "env_value",
				ValueFrom: nil,
			},
		},
		{
			name: "env with other fieldRef",
			pod:  aPod,
			env: &v1.EnvVar{
				Name: "env_name",
				ValueFrom: &v1.EnvVarSource{
					FieldRef: &v1.ObjectFieldSelector{
						APIVersion: "v1",
						FieldPath:  "spec.nodeName",
					},
				},
			},
			expectedEnv: &v1.EnvVar{
				Name: "env_name",
				ValueFrom: &v1.EnvVarSource{
					FieldRef: &v1.ObjectFieldSelector{
						APIVersion: "v1",
						FieldPath:  "spec.nodeName",
					},
				},
			},
		},
		{
			name: "env with metadata.name",
			pod:  aPod,
			env: &v1.EnvVar{
				Name: "env_name",
				ValueFrom: &v1.EnvVarSource{
					FieldRef: &v1.ObjectFieldSelector{
						APIVersion: "v1",
						FieldPath:  "metadata.name",
					},
				},
			},
			expectedEnv: &v1.EnvVar{
				Name:  "env_name",
				Value: aPod.Name,
			},
		},
		{
			name: "env with metadata.namespace",
			pod:  aPod,
			env: &v1.EnvVar{
				Name: "env_name",
				ValueFrom: &v1.EnvVarSource{
					FieldRef: &v1.ObjectFieldSelector{
						APIVersion: "v1",
						FieldPath:  "metadata.namespace",
					},
				},
			},
			expectedEnv: &v1.EnvVar{
				Name:  "env_name",
				Value: aPod.Namespace,
			},
		},
		{
			name: "env with metadata.uid",
			pod:  aPod,
			env: &v1.EnvVar{
				Name: "env_name",
				ValueFrom: &v1.EnvVarSource{
					FieldRef: &v1.ObjectFieldSelector{
						APIVersion: "v1",
						FieldPath:  "metadata.uid",
					},
				},
			},
			expectedEnv: &v1.EnvVar{
				Name:  "env_name",
				Value: string(aPod.UID),
			},
		},
	} {
		t.Run(tt.name, func(tc *testing.T) {
			mutateDownwardAPIField(tt.env, tt.pod)
			if !equality.Semantic.DeepEqual(tt.env, tt.expectedEnv) {
				tc.Errorf("expected env %+v, got %+v", tt.expectedEnv, tt.env)
			}
		})
	}
}

func Test_mutateContainerSecret(t *testing.T) {
	saSecretMap := map[string]string{
		"service-token-secret-tenant": "service-token-secret",
	}
	for _, tt := range []struct {
		name              string
		container         *v1.Container
		saSecretMap       map[string]string
		expectedContainer *v1.Container
	}{
		{
			name: "normal case",
			container: &v1.Container{
				VolumeMounts: []v1.VolumeMount{
					{
						Name:      "some-mount",
						MountPath: "/path/to/mount",
						ReadOnly:  true,
					},
					{
						Name:      "service-token-secret-tenant",
						MountPath: "/var/run/secrets/kubernetes.io/serviceaccount",
						ReadOnly:  true,
					},
				},
			},
			saSecretMap: saSecretMap,
			expectedContainer: &v1.Container{
				VolumeMounts: []v1.VolumeMount{
					{
						Name:      "some-mount",
						MountPath: "/path/to/mount",
						ReadOnly:  true,
					},
					{
						Name:      "service-token-secret",
						MountPath: "/var/run/secrets/kubernetes.io/serviceaccount",
						ReadOnly:  true,
					},
				},
			},
		},
	} {
		t.Run(tt.name, func(tc *testing.T) {
			mutateContainerSecret(tt.container, tt.saSecretMap)
			if !equality.Semantic.DeepEqual(tt.container, tt.expectedContainer) {
				tc.Errorf("expected container %+v, got %+v", tt.expectedContainer, tt.container)
			}
		})
	}
}

func TestToClusterKey(t *testing.T) {
	for _, tt := range []struct {
		name        string
		vc          *v1alpha1.VirtualCluster
		expectedKey string
	}{
		{
			name: "normal vc",
			vc: &v1alpha1.VirtualCluster{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "name",
					Namespace: "ns",
					UID:       "d64ea0c0-91f8-46f5-8643-c0cab32ab0cd",
				},
			},
			expectedKey: "ns-fd1b34-name",
		},
	} {
		t.Run(tt.name, func(tc *testing.T) {
			key := ToClusterKey(tt.vc)
			if key != tt.expectedKey {
				tc.Errorf("expected key %s, got %s", tt.expectedKey, key)
			}
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"io/ioutil"
	"os"
	"reflect"
	"strconv"
	"flag"
	"strings"
	"time"

)

type RadarPoint struct {
	Code string
	Lat  float64
	Lon  float64
}

var (
	dburl string = "54.223.146.200:27110"
)

//db.radar.createIndex({"loc":"2dsphere"})
func buildPointDoc(point RadarPoint) bson.M {

	m := bson.M{"loc": bson.M{"type": "Point", "coordinates": []float64{point.Lon, point.Lat}}, "code": point.Code}

	return m
}
func main() {

	rootdir:=flag.String("rootdir","/Users/tiger/radar","rootdir")
	worker:=flag.Int("worker",2,"worker")
	_dburl:=flag.String("db",dburl,"db")
	flag.Parse();
	dburl=*_dburl
	ch := readPoint(*rootdir)
	saveCounter := *worker
	chs := make([]<-chan int, 0, saveCounter)
	for i := 0; i < saveCounter; i++ {
		chs = append(chs, writeToDb(ch))
	}

	cases := make([]reflect.SelectCase, len(chs))
	for i, tch := range chs {
		cases[i] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(tch)}
	}
	cmCount := 0
	for {
		_, _, ok := reflect.Select(cases)
		if ok == false {
			cmCount++
		}
		if cmCount == saveCounter {
			break
		}
	}
}

func writeToDb(ch <-chan RadarPoint) <-chan int {
	rch := make(chan int)
	go func() {
		session, err := mgo.Dial(dburl)
		if err != nil {
			panic(err)
		}
		defer session.Close()
		session.SetMode(mgo.Monotonic, true)
		c := session.DB("radar").C("radar")
		sum:=0

		for {
			point ,ok:= <-ch
			if(ok==true){
				m:=bson.M{"code":point.Code,"loc":bson.M{"type":"Point","coordinates":[]float64{point.Lon,point.Lat}}}
				sum++
				c.Insert(m)
				if(sum%500==0){
					fmt.Println("save "+strconv.Itoa(sum)+" success")
				}
			}else{

				break
			}
		}
		fmt.Println("save complete "+strconv.Itoa(sum)+" success")
		time.Sleep(time.Duration(time.Second*2))
		rch <- 1
		close(rch)
	}()

	return rch
}
func readPoint(dir string) <-chan RadarPoint {
	ch := make(chan RadarPoint, 10)
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		fmt.Printf("%v\n", err)
		close(ch)
		return ch
	}
	go func() {
		for _, file := range files {
			if file.IsDir() == true {
				continue
			}
			path := dir + string(os.PathSeparator) + file.Name()
			readFile(file.Name(), path, ch)
		}
		close(ch)
	}()

	return ch
}

func readFile(code, filePath string, ch chan RadarPoint) {
	file, err := os.Open(filePath)
	if err != nil {
		println("file open error")
		fmt.Printf("%v\n", err)
		return
	}
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		s := scanner.Text()
		if s == "" || len(s) == 0 {
			continue
		}

		ss := strings.Split(s, ",")
		lon, er_lon := strconv.ParseFloat(ss[1], 64)
		lat, er_lat := strconv.ParseFloat(ss[0], 64)
		if er_lon != nil || er_lat != nil {
			fmt.Printf("bad data"+"  "+s )
			continue
		}
		p := RadarPoint{Lon: lon, Lat: lat, Code: code}
		ch <- p
	}

}
<commit_msg>close file<commit_after>package main

import (
	"bufio"
	"fmt"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"io/ioutil"
	"os"
	"reflect"
	"strconv"
	"flag"
	"strings"
	"time"

)

type RadarPoint struct {
	Code string
	Lat  float64
	Lon  float64
}

var (
	dburl string = "54.223.146.200:27110"
)

//db.radar.createIndex({"loc":"2dsphere"})
func buildPointDoc(point RadarPoint) bson.M {

	m := bson.M{"loc": bson.M{"type": "Point", "coordinates": []float64{point.Lon, point.Lat}}, "code": point.Code}

	return m
}
func main() {

	rootdir:=flag.String("rootdir","/Users/tiger/radar","rootdir")
	worker:=flag.Int("worker",2,"worker")
	_dburl:=flag.String("db",dburl,"db")
	flag.Parse();
	dburl=*_dburl
	ch := readPoint(*rootdir)
	saveCounter := *worker
	chs := make([]<-chan int, 0, saveCounter)
	for i := 0; i < saveCounter; i++ {
		chs = append(chs, writeToDb(ch))
	}

	cases := make([]reflect.SelectCase, len(chs))
	for i, tch := range chs {
		cases[i] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(tch)}
	}
	cmCount := 0
	for {
		_, _, ok := reflect.Select(cases)
		if ok == false {
			cmCount++
		}
		if cmCount == saveCounter {
			break
		}
	}
}

func writeToDb(ch <-chan RadarPoint) <-chan int {
	rch := make(chan int)
	go func() {
		session, err := mgo.Dial(dburl)
		if err != nil {
			panic(err)
		}
		defer session.Close()
		session.SetMode(mgo.Monotonic, true)
		c := session.DB("radar").C("radar")
		sum:=0

		for {
			point ,ok:= <-ch
			if(ok==true){
				m:=bson.M{"code":point.Code,"loc":bson.M{"type":"Point","coordinates":[]float64{point.Lon,point.Lat}}}
				sum++
				c.Insert(m)
				if(sum%500==0){
					fmt.Println("save "+strconv.Itoa(sum)+" success")
				}
			}else{

				break
			}
		}
		fmt.Println("save complete "+strconv.Itoa(sum)+" success")
		time.Sleep(time.Duration(time.Second*2))
		rch <- 1
		close(rch)
	}()

	return rch
}
func readPoint(dir string) <-chan RadarPoint {
	ch := make(chan RadarPoint, 10)
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		fmt.Printf("%v\n", err)
		close(ch)
		return ch
	}
	go func() {
		for _, file := range files {
			if file.IsDir() == true {
				continue
			}
			path := dir + string(os.PathSeparator) + file.Name()
			readFile(file.Name(), path, ch)
		}
		close(ch)
	}()

	return ch
}

func readFile(code, filePath string, ch chan RadarPoint) {
	file, err := os.Open(filePath)
	if err != nil {
		println("file open error")
		fmt.Printf("%v\n", err)
		return
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		s := scanner.Text()
		if s == "" || len(s) == 0 {
			continue
		}

		ss := strings.Split(s, ",")
		lon, er_lon := strconv.ParseFloat(ss[1], 64)
		lat, er_lat := strconv.ParseFloat(ss[0], 64)
		if er_lon != nil || er_lat != nil {
			fmt.Printf("bad data"+"  "+s )
			continue
		}
		p := RadarPoint{Lon: lon, Lat: lat, Code: code}
		ch <- p
	}

}
<|endoftext|>