<commit_before>/*
Package psimgo is a go implementation of the library PSim for python.
PSim is used to simulate parallel processing on a single machine.

 */
package psimgo

import (
	"math"
	"sync"
	"fmt"
)

// ====== Helper Functions ======

// Go booleans cannot be treated as integers using
// the ^ xor operation. Created a simple function
// to be used in following code
func xor(i, j, v int) bool {
	return (i == v || j == v) && i != j
}

func divmod(i, j int) (d, r int) {
	result := int(math.Floor(float64(i/j)))
	remainder := i - (result*j)
	return result, remainder
}

// ====== Topology Communication functions ======
// boolean functions to test if node i can
// communicate directly with node j. Some
// functions first require input of p (number of nodes)
// and will then return test function

// Bus topology, all nodes can communicate
// with all others. Always true
func BUS(i, j int) bool {
	return true
}

// Switch topology, all nodes can communicate
// with all others. Always true
func SWITCH(i,j int) bool {
	return true
}

// 1-D Mesh topology. Returns function to test.
// if i and j are neighbors, communication is available
func MESH1(p int) func(i, j int) bool {
	return func(i, j int) bool {
		return int(math.Abs(float64(i - j))) == 1
	}
}

// 1-D Torus topology. Like 1-D mesh, but circularly
// linked. Function must account for first and last
// nodes connected
func TORUS1(p int) func(i, j int) bool {
	return func(i, j int) bool {
		return (i - j + p) % p == 1 || (j - i + p) % p == 1
	}
}

//2-D Mesh.
func MESH2(p int) func(i, j int) bool {
	q := int(math.Floor(math.Sqrt(float64(p)) + 0.1))
	return func(i, j int) bool {
		return xor(int(math.Abs(float64(i%q-j%q))), int(math.Abs(float64(i/q-j/q))), 1)
	}
}

func TORUS2(p int) func(i, j int) bool {
	q := int(math.Floor(math.Sqrt(float64(p)) + 0.1))
	return func(i, j int) bool {
		return xor(((i%q - j%q + q) %q), ((i/q - j/q + q) %q), 1) || xor(((j%q - i%q + q) %q), ((j/q - i/q + q) %q), 1)
	}
}

func TREE(i, j int) bool {
	return i == int(math.Ceil(float64(j-1) /2.0)) || j == int(math.Ceil(float64(i-1) /2.0))
}

// ====== PSim type and Functions ======

type PSim struct {
	P int
	Topology func(i, j int) (bool)
	initialized bool
	Pipes [][]chan interface {}
}

func (psim PSim) Run(f func(rank int, comm PSim)) {
	var wg sync.WaitGroup

	// init arrays
	psim.Pipes = make([][]chan interface {}, psim.P)

	// default p num procs
	if psim.P == 0 {
		psim.P = 1
	}

	// default topology
	if psim.Topology == nil {
		psim.Topology = SWITCH
	}

	// initialize channels
	for i := 0; i < psim.P; i++ {
		psim.Pipes[i] = make([]chan interface {}, psim.P)
		for j := 0; j < psim.P; j++ {
			psim.Pipes[i][j] = make(chan interface {})
		}
	}

	wg.Add(psim.P)

	// start go threads
	for r := 0; r < psim.P; r++ {
		go func (r int) {
			defer wg.Done()
			f(r, psim)
		}(r)
	}

	// psim object has been initialized
	psim.initialized = true

	wg.Wait()
}

// ======= Reflective Decorators =======

// Casts --
// 	using go's type assertions to make
//  casting functions as to keep the channels
//  and hence all of the message passing generic
//  conversions used on Recv

func ToInt(a interface {}) int{
	if v, ok := a.(int); ok {
		return v
	}

	// TODO Error
	fmt.Printf("Cannot cast to int")
	return 0
}

func ToFloat64(a interface {}) float64 {
	if v, ok := a.(float64); ok {
		return v
	}

	// TODO Error
	fmt.Printf("Cannot cast to int")
	return 0
}

func ToFloat(a interface {}) float64 {
	return ToFloat64(a)
}

func ToFloat32(a interface {}) float32 {
	if v, ok := a.(float32); ok {
		return v
	}

	// TODO Error
	fmt.Printf("Cannot cast to int")
	return 0
}

func ToIntArray(a []interface {}) []int {
	l := len(a)
	res := make([]int, l, l)

	for i := range a {
		res[i] = ToInt(a[i])
	}

	return res
}

func ToFloat64Array(a []interface {}) []float64 {
	l := len(a)
	res := make([]float64, l, l)

	for i := range a {
		res[i] = ToFloat64(a[i])
	}

	return res
}

func ToFloatArray(a []interface {}) []float64 {
	return ToFloat64Array(a)
}

func ToFloat32Array(a []interface {}) []float32 {
	l := len(a)
	res := make([]float32, l, l)

	for i := range a {
		res[i] = ToFloat32(a[i])
	}

	return res
}

// Reduce Operations --
// 	for the reduce operations, an associative
//  binary function is necessary. Unfortunately
//  type checking must be made so here are some basic
//  operations for users

func IntSum(a, b interface {}) int {
	if x, ok := a.(int); ok {
		if y, ok := b.(int); ok {
			return x + y
		}
	}

	fmt.Printf("Illegal add")
	return 0
}

func IntProd(a, b interface {}) int {
	if x, ok := a.(int); ok {
		if y, ok := b.(int); ok {
			return x * y
		}
	}

	fmt.Printf("Illegal prod")
	return 0
}

func FloatSum(a, b interface {}) float64 {
	if x, ok := a.(float64); ok {
		if y, ok := b.(float64); ok {
			return x + y
		}
	}

	fmt.Printf("Illegal add")
	return 0
}

func FloatProd(a, b interface {}) float64 {
	if x, ok := a.(float64); ok {
		if y, ok := b.(float64); ok {
			return x * y
		}
	}

	fmt.Printf("Illegal prod")
	return 0
}

func PMax(a, b interface {}) float64 {
	if x, ok := a.(float64); ok {
		if y, ok := b.(float64); ok {
			return math.Max(x, y)
		}
	}

	fmt.Printf("Illegal prod")
	return 0
}

func PMin(a, b interface {}) float64 {
	if x, ok := a.(float64); ok {
		if y, ok := b.(float64); ok {
			return math.Min(x, y)
		}
	}

	fmt.Printf("Illegal prod")
	return 0
}

// Decorators --
// 	using the functions created above, we make
//  decorator functions for Recv to return the
//  expected type directly rather than cast

// Recv
func (psim PSim) RecvInt(rank, source int) int {
	return ToInt(psim.Recv(rank, source))
}

func (psim PSim) RecvFloat64(rank, source int) float64 {
	return ToFloat64(psim.Recv(rank, source))
}

func (psim PSim) RecvFloat(rank, source int) float64 {
	return ToFloat64(psim.Recv(rank, source))
}

func (psim PSim) RecvFloat32(rank, source int) float32 {
	return ToFloat32(psim.Recv(rank, source))
}

// a2aBcast
func (psim PSim) All2all_broadcastInt(rank int, data int) []int {
	return ToIntArray(psim.All2all_broadcast(rank, data))
}

func (psim PSim) All2all_broadcastFloat(rank int, data float64) []float64 {
	return ToFloat64Array(psim.All2all_broadcast(rank, data))
}

// scatter
func (psim PSim) One2all_scatterInt(rank, source int, data []interface {}) []int {
	return ToIntArray(psim.One2all_scatter(rank, source, data))
}

func (psim PSim) One2all_scatterFloat(rank, source int, data []interface {}) []float64 {
	return ToFloatArray(psim.One2all_scatter(rank, source, data))
}

// collect
func (psim PSim) All2one_collectInt(rank, dest int, data int) []int {
	return ToIntArray(psim.All2one_collect(rank, dest, data))
}

func (psim PSim) All2one_collectFloat(rank, dest int, data float64) []float64 {
	return ToFloatArray(psim.All2one_collect(rank, dest, data))
}

// Message Passing functions

func (psim PSim) Send(source, dest int, data interface {}) {
	// if i or j less than 0 or greater then nprocs error
	// if i -> j communication unavailable, error
	if source < 0 || source > psim.P-1 || dest < 0 || dest > psim.P-1 || !psim.Topology(source, dest) {
		fmt.Printf("Send ERR:\nOut of range, i: %d; j: %d\n", source, dest);
	} else {
		// send data
		psim.Pipes[source][dest] <-data
	}
}

func (psim PSim) Recv(rank, source int) interface {} {
	// if i or j less than 0 or greater then nprocs error
	// if i -> j communication unavailable, error
	if rank < 0 || rank > psim.P-1 || source < 0 || source > psim.P-1 || !psim.Topology(rank, source) {
		fmt.Printf("Recv ERR:\nOut of range, i: %d; j: %d\n", rank, source);
	} else {
		// recv data
		return <-psim.Pipes[source][rank]
	}
	return nil
}

func (psim PSim) One2all_broadcast(rank, source int, data interface {}) interface {} {
	if rank == source {
		for i := 0; i < psim.P; i++ {
			if i != source {
				psim.Send(source, i, data)
			}
		}
		return data
	} else {
		return psim.Recv(rank, source)
	}
}

func (psim PSim) All2all_broadcast(rank int, data interface {}) []interface {} {
	vector := psim.All2one_collect(rank, 0, data)
	v := psim.One2all_broadcast(rank, 0, vector)
	switch t:= v.(type){
	case []interface {}:
		return t
	case interface {}:
		return []interface {} {t}
	default:
		// TODO error
	}
	return nil
}

func (psim PSim) One2all_scatter(rank, source int, data []interface {}) []interface {} {
	if rank == source {
		h, reminder := divmod(len(data), psim.P)

		if reminder > 0 {
			h += 1
		}

		for i := 1; i < psim.P; i++ {
			psim.Send(rank, i, data[i*h:i*h+h])
		}
		return data[0:h]
	} else {
		v := psim.Recv(rank, source)
		if vector, ok := v.([]interface {}); ok {
			return vector
		} else {
			return nil
		}
	}
}

func (psim PSim) All2one_collect(rank, dest int, data interface {}) []interface {} {
	var result []interface {}

	if rank == dest {
		for i := 0; i < psim.P; i++ {
			if i == rank {
				result = append(result, data)
			} else {
				result = append(result, psim.Recv(rank, i))
			}
		}
	} else {
		psim.Send(rank, dest, data)
	}

	return result
}

func (psim PSim) All2one_reduce(
	rank, dest int,
	data interface {},
	op func(a, b interface {}) interface {}) interface {} {
	if rank == dest {
		result := data
		for i := 1; i< psim.P; i++ {
			if i != rank {
				result = op(result, psim.Recv(rank, i))
			}
		}
		return result
	} else {
		psim.Send(rank, dest, data)
		return 0
	}
}

func (psim PSim) All2all_reduce(rank int, data interface {}, op func(a, b interface {}) interface {}) interface {} {
	result := psim.All2one_reduce(rank, 0, data, op)
	result = psim.One2all_broadcast(rank, 0, result)
	return result
}

func (psim PSim) Barrier(rank int) {
	psim.All2all_broadcast(rank, 0)
}
<commit_msg>Added InterfaceToSlice function to handle type conversion of generic interface to array of interface, added array decorators, required reflect package<commit_after>/*
Package psimgo is a go implementation of the library PSim for python.
PSim is used to simulate parallel processing on a single machine.

 */
package psimgo

import (
	"math"
	"sync"
	"fmt"
	"reflect"
)

// ====== Helper Functions ======

// Go booleans cannot be treated as integers using
// the ^ xor operation. Created a simple function
// to be used in following code
func xor(i, j, v int) bool {
	return (i == v || j == v) && i != j
}

func divmod(i, j int) (d, r int) {
	result := int(math.Floor(float64(i/j)))
	remainder := i - (result*j)
	return result, remainder
}

// ====== Topology Communication functions ======
// boolean functions to test if node i can
// communicate directly with node j. Some
// functions first require input of p (number of nodes)
// and will then return test function

// Bus topology, all nodes can communicate
// with all others. Always true
func BUS(i, j int) bool {
	return true
}

// Switch topology, all nodes can communicate
// with all others. Always true
func SWITCH(i,j int) bool {
	return true
}

// 1-D Mesh topology. Returns function to test.
// if i and j are neighbors, communication is available
func MESH1(p int) func(i, j int) bool {
	return func(i, j int) bool {
		return int(math.Abs(float64(i - j))) == 1
	}
}

// 1-D Torus topology. Like 1-D mesh, but circularly
// linked. Function must account for first and last
// nodes connected
func TORUS1(p int) func(i, j int) bool {
	return func(i, j int) bool {
		return (i - j + p) % p == 1 || (j - i + p) % p == 1
	}
}

//2-D Mesh.
func MESH2(p int) func(i, j int) bool {
	q := int(math.Floor(math.Sqrt(float64(p)) + 0.1))
	return func(i, j int) bool {
		return xor(int(math.Abs(float64(i%q-j%q))), int(math.Abs(float64(i/q-j/q))), 1)
	}
}

func TORUS2(p int) func(i, j int) bool {
	q := int(math.Floor(math.Sqrt(float64(p)) + 0.1))
	return func(i, j int) bool {
		return xor(((i%q - j%q + q) %q), ((i/q - j/q + q) %q), 1) || xor(((j%q - i%q + q) %q), ((j/q - i/q + q) %q), 1)
	}
}

func TREE(i, j int) bool {
	return i == int(math.Ceil(float64(j-1) /2.0)) || j == int(math.Ceil(float64(i-1) /2.0))
}

// ====== PSim type and Functions ======

type PSim struct {
	P int
	Topology func(i, j int) (bool)
	initialized bool
	Pipes [][]chan interface {}
}

func (psim PSim) Run(f func(rank int, comm PSim)) {
	var wg sync.WaitGroup

	// init arrays
	psim.Pipes = make([][]chan interface {}, psim.P)

	// default p num procs
	if psim.P == 0 {
		psim.P = 1
	}

	// default topology
	if psim.Topology == nil {
		psim.Topology = SWITCH
	}

	// initialize channels
	for i := 0; i < psim.P; i++ {
		psim.Pipes[i] = make([]chan interface {}, psim.P)
		for j := 0; j < psim.P; j++ {
			psim.Pipes[i][j] = make(chan interface {})
		}
	}

	wg.Add(psim.P)

	// start go threads
	for r := 0; r < psim.P; r++ {
		go func (r int) {
			defer wg.Done()
			f(r, psim)
		}(r)
	}

	// psim object has been initialized
	psim.initialized = true

	wg.Wait()
}

// ======= Reflective Decorators =======

// Casts --
// 	using go's type assertions to make
//  casting functions as to keep the channels
//  and hence all of the message passing generic
//  conversions used on Recv

func ToInt(a interface {}) int{
	if v, ok := a.(int); ok {
		return v
	}

	// TODO Error
	fmt.Printf("Cannot cast to int")
	return 0
}

func ToFloat64(a interface {}) float64 {
	if v, ok := a.(float64); ok {
		return v
	}

	// TODO Error
	fmt.Printf("Cannot cast to int")
	return 0
}

func ToFloat(a interface {}) float64 {
	return ToFloat64(a)
}

func ToFloat32(a interface {}) float32 {
	if v, ok := a.(float32); ok {
		return v
	}

	// TODO Error
	fmt.Printf("Cannot cast to int")
	return 0
}

func InterfaceToSlice(slice interface{}) []interface{} {
	s := reflect.ValueOf(slice)
	if s.Kind() != reflect.Slice {
		panic("InterfaceSlice() given a non-slice type")
	}

	ret := make([]interface{}, s.Len())

	for i:=0; i<s.Len(); i++ {
		ret[i] = s.Index(i).Interface()
	}

	return ret
}

func ToIntArray(a []interface {}) []int {
	l := len(a)
	res := make([]int, l, l)

	for i := range a {
		res[i] = ToInt(a[i])
	}

	return res
}

func ToFloat64Array(a []interface {}) []float64 {
	l := len(a)
	res := make([]float64, l, l)

	for i := range a {
		res[i] = ToFloat64(a[i])
	}

	return res
}

func ToFloatArray(a []interface {}) []float64 {
	return ToFloat64Array(a)
}

func ToFloat32Array(a []interface {}) []float32 {
	l := len(a)
	res := make([]float32, l, l)

	for i := range a {
		res[i] = ToFloat32(a[i])
	}

	return res
}

// Reduce Operations --
// 	for the reduce operations, an associative
//  binary function is necessary. Unfortunately
//  type checking must be made so here are some basic
//  operations for users

func IntSum(a, b interface {}) int {
	if x, ok := a.(int); ok {
		if y, ok := b.(int); ok {
			return x + y
		}
	}

	fmt.Printf("Illegal add")
	return 0
}

func IntProd(a, b interface {}) int {
	if x, ok := a.(int); ok {
		if y, ok := b.(int); ok {
			return x * y
		}
	}

	fmt.Printf("Illegal prod")
	return 0
}

func FloatSum(a, b interface {}) float64 {
	if x, ok := a.(float64); ok {
		if y, ok := b.(float64); ok {
			return x + y
		}
	}

	fmt.Printf("Illegal add")
	return 0
}

func FloatProd(a, b interface {}) float64 {
	if x, ok := a.(float64); ok {
		if y, ok := b.(float64); ok {
			return x * y
		}
	}

	fmt.Printf("Illegal prod")
	return 0
}

func PMax(a, b interface {}) float64 {
	if x, ok := a.(float64); ok {
		if y, ok := b.(float64); ok {
			return math.Max(x, y)
		}
	}

	fmt.Printf("Illegal prod")
	return 0
}

func PMin(a, b interface {}) float64 {
	if x, ok := a.(float64); ok {
		if y, ok := b.(float64); ok {
			return math.Min(x, y)
		}
	}

	fmt.Printf("Illegal prod")
	return 0
}

// Decorators --
// 	using the functions created above, we make
//  decorator functions for Recv to return the
//  expected type directly rather than cast

// Recv
func (psim PSim) RecvInt(rank, source int) int {
	return ToInt(psim.Recv(rank, source))
}

func (psim PSim) RecvFloat64(rank, source int) float64 {
	return ToFloat64(psim.Recv(rank, source))
}

func (psim PSim) RecvFloat(rank, source int) float64 {
	return ToFloat64(psim.Recv(rank, source))
}

func (psim PSim) RecvFloat32(rank, source int) float32 {
	return ToFloat32(psim.Recv(rank, source))
}

func (psim PSim) RecvIntArray(rank, source int) []int {
	return ToIntArray(InterfaceToSlice(psim.Recv(rank, source)))
}

func (psim PSim) RecvFloat64Array(rank, source int) []float64 {
	return ToFloat64Array(InterfaceToSlice(psim.Recv(rank, source)))
}

func (psim PSim) RecvFloatArray(rank, source int) []float64 {
	return ToFloatArray(InterfaceToSlice(psim.Recv(rank, source)))
}

func (psim PSim) RecvFloat32Array(rank, source int) []float32 {
	return ToFloat32Array(InterfaceToSlice(psim.Recv(rank, source)))
}

// a2aBcast
func (psim PSim) All2all_broadcastInt(rank int, data int) []int {
	return ToIntArray(psim.All2all_broadcast(rank, data))
}

func (psim PSim) All2all_broadcastFloat(rank int, data float64) []float64 {
	return ToFloat64Array(psim.All2all_broadcast(rank, data))
}

// scattera
func (psim PSim) One2all_scatterInt(rank, source int, data []interface {}) []int {
	return ToIntArray(psim.One2all_scatter(rank, source, data))
}

func (psim PSim) One2all_scatterFloat(rank, source int, data []interface {}) []float64 {
	return ToFloatArray(psim.One2all_scatter(rank, source, data))
}

// collect
func (psim PSim) All2one_collectInt(rank, dest int, data int) []int {
	return ToIntArray(psim.All2one_collect(rank, dest, data))
}

func (psim PSim) All2one_collectFloat(rank, dest int, data float64) []float64 {
	return ToFloatArray(psim.All2one_collect(rank, dest, data))
}

// Message Passing functions

func (psim PSim) Send(source, dest int, data interface {}) {
	// if i or j less than 0 or greater then nprocs error
	// if i -> j communication unavailable, error
	if source < 0 || source > psim.P-1 || dest < 0 || dest > psim.P-1 || !psim.Topology(source, dest) {
		fmt.Printf("Send ERR:\nOut of range, i: %d; j: %d\n", source, dest);
	} else {
		// send data
		psim.Pipes[source][dest] <-data
	}
}

func (psim PSim) Recv(rank, source int) interface {} {
	// if i or j less than 0 or greater then nprocs error
	// if i -> j communication unavailable, error
	if rank < 0 || rank > psim.P-1 || source < 0 || source > psim.P-1 || !psim.Topology(rank, source) {
		fmt.Printf("Recv ERR:\nOut of range, i: %d; j: %d\n", rank, source);
	} else {
		// recv data
		return <-psim.Pipes[source][rank]
	}
	return nil
}

func (psim PSim) One2all_broadcast(rank, source int, data interface {}) interface {} {
	if rank == source {
		for i := 0; i < psim.P; i++ {
			if i != source {
				psim.Send(source, i, data)
			}
		}
		return data
	} else {
		return psim.Recv(rank, source)
	}
}

func (psim PSim) All2all_broadcast(rank int, data interface {}) []interface {} {
	vector := psim.All2one_collect(rank, 0, data)
	v := psim.One2all_broadcast(rank, 0, vector)
	switch t:= v.(type){
	case []interface {}:
		return t
	case interface {}:
		return []interface {} {t}
	default:
		// TODO error
	}
	return nil
}

func (psim PSim) One2all_scatter(rank, source int, data []interface {}) []interface {} {
	if rank == source {
		h, reminder := divmod(len(data), psim.P)

		if reminder > 0 {
			h += 1
		}

		for i := 1; i < psim.P; i++ {
			psim.Send(rank, i, data[i*h:i*h+h])
		}
		return data[0:h]
	} else {
		v := psim.Recv(rank, source)
		if vector, ok := v.([]interface {}); ok {
			return vector
		} else {
			return nil
		}
	}
}

func (psim PSim) All2one_collect(rank, dest int, data interface {}) []interface {} {
	var result []interface {}

	if rank == dest {
		for i := 0; i < psim.P; i++ {
			if i == rank {
				result = append(result, data)
			} else {
				result = append(result, psim.Recv(rank, i))
			}
		}
	} else {
		psim.Send(rank, dest, data)
	}

	return result
}

func (psim PSim) All2one_reduce(
	rank, dest int,
	data interface {},
	op func(a, b interface {}) interface {}) interface {} {
	if rank == dest {
		result := data
		for i := 1; i< psim.P; i++ {
			if i != rank {
				result = op(result, psim.Recv(rank, i))
			}
		}
		return result
	} else {
		psim.Send(rank, dest, data)
		return 0
	}
}

func (psim PSim) All2all_reduce(rank int, data interface {}, op func(a, b interface {}) interface {}) interface {} {
	result := psim.All2one_reduce(rank, 0, data, op)
	result = psim.One2all_broadcast(rank, 0, result)
	return result
}

func (psim PSim) Barrier(rank int) {
	psim.All2all_broadcast(rank, 0)
}
<|endoftext|>


<commit_before>// This file is part of *kellner*
//
// Copyright (C) 2015, Travelping GmbH <copyright@travelping.com>
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync/atomic"
	"time"
)

func scanRoot(root, cache string, nworkers int, doMD5, doSHA1 bool, gzipper gzWrite) {

	filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {

		if fi == nil || !fi.IsDir() {

			// not existing root-directory. we don't crash or exit here, the
			// operator might create it later on and trigger a rescan.
			if fi == nil && root == path {
				log.Printf("warning: not existent root directory %q\n", root)
			}

			return nil
		}

		// skip cache-directory
		if strings.HasPrefix(path, cache) {
			return filepath.SkipDir
		}

		var (
			now          = time.Now()
			relPath, _   = filepath.Rel(root, path)
			cachePath, _ = filepath.Abs(filepath.Join(cache, relPath))
			scanner      = packageScanner{
				cache:  cachePath,
				doMD5:  doMD5,
				doSHA1: doSHA1,
			}
		)

		if err = scanner.scan(path, nworkers); err != nil {
			log.Printf("error: %v", err)
			return nil
		}

		log.Printf("done building index for %q", path)
		log.Printf("time to parse %d packages in %q: %s\n",
			scanner.packages.Len(), path, time.Since(now))

		//
		// write the package index
		//

		var indexName = filepath.Join(cachePath, "Packages")
		var indexNameGz = indexName + ".gz"
		var indexNameStamps = indexName + ".stamps"

		var packagesFile, packagesFileGz, packagesFileStamps *os.File
		if packagesFile, err = ioutil.TempFile(cachePath, "Packages"); err != nil {
			log.Printf("error: can't create the Packages file in %q: %v\n", cachePath, err)
			return nil
		}
		defer packagesFile.Close()
		if packagesFileGz, err = os.Create(packagesFile.Name() + ".gz"); err != nil {
			log.Printf("error: can't create the Packages.gz file in %q: %v\n", cachePath, err)
			return nil
		}
		defer packagesFileGz.Close()

		if packagesFileStamps, err = os.Create(packagesFile.Name() + ".stamps"); err != nil {
			log.Printf("error: can't create the Packages.stamps file in %q: %v\n", cachePath, err)
			return nil
		}
		defer packagesFileStamps.Close()

		var index = scanner.packages.String()

		io.Copy(packagesFile, strings.NewReader(index))
		packagesFile.Sync()
		gzipper(packagesFileGz, strings.NewReader(index))
		packagesFileGz.Sync()
		scanner.packages.StampsTo(packagesFileStamps)
		packagesFileStamps.Sync()

		os.Remove(indexName)
		os.Remove(indexNameGz)
		os.Remove(indexNameStamps)
		os.Rename(packagesFile.Name(), indexName)
		os.Rename(packagesFileGz.Name(), indexNameGz)
		os.Rename(packagesFileStamps.Name(), indexNameStamps)

		return nil
	})
}

type packageScanner struct {
	packages *packageIndex
	nScanned int64
	nCached  int64

	root   string
	cache  string
	doSHA1 bool
	doMD5  bool
}

func (s *packageScanner) clear() {
	s.packages = &packageIndex{Entries: make(map[string]*ipkArchive)}
}

// scanPackages scans all files in root for packages. if it finds one, it
// checks the cache folder, it a cached version of the meta-information exists
// or if the meta-information is out-of-date. only if it's needed the meta
// information will be extracted from the packages and stored in the
// cache folder
func (s *packageScanner) scan(dirPath string, nworkers int) error {

	var dir, err = os.Open(dirPath)
	if err != nil {
		return fmt.Errorf("opening %q: %v\n", dirPath, err)
	}
	defer dir.Close()

	if err = s.provideCachePath(); err != nil {
		return err
	}

	var entries []os.FileInfo
	if entries, err = dir.Readdir(-1); err != nil {
		return fmt.Errorf("reading entries %q: %v\n", dirPath, err)
	}

	if s.packages == nil {
		s.clear()
	}
	var worker = newWorkerPool(nworkers)

	for _, entry := range entries {

		if s.skipNonPackage(entry) {
			continue
		}

		if s.fromCache(entry) {
			continue
		}

		worker.Hire()
		go s.scanAndCache(path.Join(dirPath, entry.Name()), worker)
	}
	worker.Wait()

	log.Printf("scanned %d packages (fresh %d|%d from cache) in %q.",
		s.packages.Len(), s.nScanned, s.nCached, s.root)

	return nil
}

func (s *packageScanner) scanAndCache(filePath string, worker *workerPool) {

	var n = time.Now()
	defer func() {
		worker.Release()
		log.Println("processed", filePath, time.Now().Sub(n))
	}()

	var archive, err = newIpkFromFile(filePath, s.doMD5, s.doSHA1)
	if err != nil {
		log.Printf("error: %v\n", err)
		return
	}
	s.packages.Add(filepath.Base(filePath), archive)
	atomic.AddInt64(&s.nScanned, 1)

	if s.cache == "" {
		return
	}

	var cacheName = genCachedControlName(filepath.Base(filePath), s.cache)
	if err = ioutil.WriteFile(cacheName, []byte(archive.Control), 0644); err != nil {
		log.Println(cacheName, err)
	}
}

func (s *packageScanner) fromCache(entry os.FileInfo) bool {

	var controlName = genCachedControlName(entry.Name(), s.cache)
	var cacheEntry, err = os.Stat(controlName)
	if err != nil {
		return false
	}
	if entry.ModTime().Before(cacheEntry.ModTime()) {
		var archive, err = newIpkFromCache(entry.Name(), s.cache, s.doMD5, s.doSHA1)
		if err != nil {
			log.Printf("error: %v\n", err)
			return false
		}
		s.packages.Add(entry.Name(), archive)
		s.nCached += 1
		return true
	}
	return false
}

func (s *packageScanner) skipNonPackage(fi os.FileInfo) bool {
	return path.Ext(fi.Name()) != ".ipk" || fi.IsDir()
}

func (s *packageScanner) provideCachePath() error {
	if s.cache == "" {
		return nil
	}

	if err := os.MkdirAll(s.cache, 0755); err != nil {
		return fmt.Errorf("creating cache-folder %q: %v\n", s.cache, err)
	}
	return nil
}

func genCachedControlName(name, cache string) string {
	var cacheName = filepath.Join(cache, name)
	return cacheName + ".control"
}
<commit_msg>comment update<commit_after>// This file is part of *kellner*
//
// Copyright (C) 2015, Travelping GmbH <copyright@travelping.com>
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync/atomic"
	"time"
)

func scanRoot(root, cache string, nworkers int, doMD5, doSHA1 bool, gzipper gzWrite) {

	filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {

		if fi == nil || !fi.IsDir() {

			// not existing root-directory. we don't crash or exit here, the
			// operator might create it later on and trigger a rescan.
			if fi == nil && root == path {
				log.Printf("warning: not existent root directory %q\n", root)
			}

			return nil
		}

		// skip cache-directory
		if strings.HasPrefix(path, cache) {
			return filepath.SkipDir
		}

		var (
			now          = time.Now()
			relPath, _   = filepath.Rel(root, path)
			cachePath, _ = filepath.Abs(filepath.Join(cache, relPath))
			scanner      = packageScanner{
				cache:  cachePath,
				doMD5:  doMD5,
				doSHA1: doSHA1,
			}
		)

		if err = scanner.scan(path, nworkers); err != nil {
			log.Printf("error: %v", err)
			return nil
		}

		log.Printf("done building index for %q", path)
		log.Printf("time to parse %d packages in %q: %s\n",
			scanner.packages.Len(), path, time.Since(now))

		//
		// write the package index
		//

		var indexName = filepath.Join(cachePath, "Packages")
		var indexNameGz = indexName + ".gz"
		var indexNameStamps = indexName + ".stamps"

		var packagesFile, packagesFileGz, packagesFileStamps *os.File
		if packagesFile, err = ioutil.TempFile(cachePath, "Packages"); err != nil {
			log.Printf("error: can't create the Packages file in %q: %v\n", cachePath, err)
			return nil
		}
		defer packagesFile.Close()
		if packagesFileGz, err = os.Create(packagesFile.Name() + ".gz"); err != nil {
			log.Printf("error: can't create the Packages.gz file in %q: %v\n", cachePath, err)
			return nil
		}
		defer packagesFileGz.Close()

		if packagesFileStamps, err = os.Create(packagesFile.Name() + ".stamps"); err != nil {
			log.Printf("error: can't create the Packages.stamps file in %q: %v\n", cachePath, err)
			return nil
		}
		defer packagesFileStamps.Close()

		var index = scanner.packages.String()

		io.Copy(packagesFile, strings.NewReader(index))
		packagesFile.Sync()
		gzipper(packagesFileGz, strings.NewReader(index))
		packagesFileGz.Sync()
		scanner.packages.StampsTo(packagesFileStamps)
		packagesFileStamps.Sync()

		os.Remove(indexName)
		os.Remove(indexNameGz)
		os.Remove(indexNameStamps)
		os.Rename(packagesFile.Name(), indexName)
		os.Rename(packagesFileGz.Name(), indexNameGz)
		os.Rename(packagesFileStamps.Name(), indexNameStamps)

		return nil
	})
}

type packageScanner struct {
	packages *packageIndex
	nScanned int64
	nCached  int64

	root   string
	cache  string
	doSHA1 bool
	doMD5  bool
}

func (s *packageScanner) clear() {
	s.packages = &packageIndex{Entries: make(map[string]*ipkArchive)}
}

// scanPackages scans all files in dirPath for packages. if it finds one, it
// checks the cache folder, if a cached version of the meta-information exists
// or if the meta-information is out-of-date. only when needed the meta
// information will be extracted from the packages and stored in the
// cache folder
func (s *packageScanner) scan(dirPath string, nworkers int) error {

	var dir, err = os.Open(dirPath)
	if err != nil {
		return fmt.Errorf("opening %q: %v\n", dirPath, err)
	}
	defer dir.Close()

	if err = s.provideCachePath(); err != nil {
		return err
	}

	var entries []os.FileInfo
	if entries, err = dir.Readdir(-1); err != nil {
		return fmt.Errorf("reading entries %q: %v\n", dirPath, err)
	}

	if s.packages == nil {
		s.clear()
	}
	var worker = newWorkerPool(nworkers)

	for _, entry := range entries {

		if s.skipNonPackage(entry) {
			continue
		}

		if s.fromCache(entry) {
			continue
		}

		worker.Hire()
		go s.scanAndCache(path.Join(dirPath, entry.Name()), worker)
	}
	worker.Wait()

	log.Printf("scanned %d packages (fresh %d|%d from cache) in %q.",
		s.packages.Len(), s.nScanned, s.nCached, s.root)

	return nil
}

func (s *packageScanner) scanAndCache(filePath string, worker *workerPool) {

	var n = time.Now()
	defer func() {
		worker.Release()
		log.Println("processed", filePath, time.Now().Sub(n))
	}()

	var archive, err = newIpkFromFile(filePath, s.doMD5, s.doSHA1)
	if err != nil {
		log.Printf("error: %v\n", err)
		return
	}
	s.packages.Add(filepath.Base(filePath), archive)
	atomic.AddInt64(&s.nScanned, 1)

	if s.cache == "" {
		return
	}

	var cacheName = genCachedControlName(filepath.Base(filePath), s.cache)
	if err = ioutil.WriteFile(cacheName, []byte(archive.Control), 0644); err != nil {
		log.Println(cacheName, err)
	}
}

func (s *packageScanner) fromCache(entry os.FileInfo) bool {

	var controlName = genCachedControlName(entry.Name(), s.cache)
	var cacheEntry, err = os.Stat(controlName)
	if err != nil {
		return false
	}
	if entry.ModTime().Before(cacheEntry.ModTime()) {
		var archive, err = newIpkFromCache(entry.Name(), s.cache, s.doMD5, s.doSHA1)
		if err != nil {
			log.Printf("error: %v\n", err)
			return false
		}
		s.packages.Add(entry.Name(), archive)
		s.nCached += 1
		return true
	}
	return false
}

func (s *packageScanner) skipNonPackage(fi os.FileInfo) bool {
	return path.Ext(fi.Name()) != ".ipk" || fi.IsDir()
}

func (s *packageScanner) provideCachePath() error {
	if s.cache == "" {
		return nil
	}

	if err := os.MkdirAll(s.cache, 0755); err != nil {
		return fmt.Errorf("creating cache-folder %q: %v\n", s.cache, err)
	}
	return nil
}

func genCachedControlName(name, cache string) string {
	var cacheName = filepath.Join(cache, name)
	return cacheName + ".control"
}
<|endoftext|>


<commit_before>// Copyright 2014, Hǎiliàng Wáng. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package flow

import (
	"io"

	"github.com/hailiang/gombi/scan"
)

type scanner struct {
	*scan.Scanner
}

func (s *scanner) Scan() bool {
	for s.Scanner.Scan() {
		if s.Token().Type != tokenSpace {
			return true
		}
	}
	return false
}

func newScanner(r io.Reader) *scanner {
	var (
		char   = scan.Char
		pat    = scan.Pat
		merge  = scan.Merge
		or     = scan.Or
		con    = scan.Con
		Tokens = scan.Tokens

		nonctrl   = char(`[:cntrl:]`).Negate()
		indent    = char(`\t `)
		lineBreak = char(`\n\r`)
		space     = merge(indent, lineBreak)
		any       = merge(nonctrl, space)
		invalid   = any.Negate()
		inline    = any.Exclude(lineBreak)
		delim     = char(`,{}`)
		empty     = pat(``)

		newline        = or(lineBreak, pat(`\r\n`))
		inlineComment  = con(pat(`//`), inline.ZeroOrMore(), or(newline, empty))
		quoted         = or(inline.Exclude(char(`"`)), pat(`\\"`))
		quotedString   = con(pat(`"`), quoted.ZeroOrMore(), pat(`"`))
		unquoted       = any.Exclude(delim, space)
		unquotedString = unquoted.OneOrMore()
		generalString  = or(quotedString, unquotedString)

		tokens = Tokens(
			invalid,
			inlineComment,
			char(`{`),
			char(`}`),
			char(`,`),
			generalString,
			space.OneOrMore(),
		)
	)
	s, err := scan.NewScanner(tokens.String(), r)
	if err != nil {
		panic(err)
	}
	return &scanner{s}
}

const (
	tokenError = iota
	tokenComment
	tokenLeftBrace
	tokenRightBrace
	tokenComma
	tokenString
	tokenSpace
	tokenEOF
)

type tokenType int

func (t tokenType) String() string {
	switch t {
	case tokenError:
		return "tokenError"
	case tokenEOF:
		return "tokenEOF"
	case tokenComment:
		return "tokenComment"
	case tokenString:
		return "tokenString"
	case tokenLeftBrace:
		return "tokenLeftBrace"
	case tokenRightBrace:
		return "tokenRightBrace"
	case tokenComma:
		return "tokenComma"
	}
	return "token unkown"
}
<commit_msg>adjust scanner api.<commit_after>// Copyright 2014, Hǎiliàng Wáng. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package flow

import (
	"io"

	"github.com/hailiang/gombi/scan"
)

const (
	tokenError = iota
	tokenComment
	tokenLeftBrace
	tokenRightBrace
	tokenComma
	tokenString
	tokenSpace
	tokenEOF
)

type tokenType int

func (t tokenType) String() string {
	switch t {
	case tokenError:
		return "tokenError"
	case tokenEOF:
		return "tokenEOF"
	case tokenComment:
		return "tokenComment"
	case tokenString:
		return "tokenString"
	case tokenLeftBrace:
		return "tokenLeftBrace"
	case tokenRightBrace:
		return "tokenRightBrace"
	case tokenComma:
		return "tokenComma"
	}
	return "token unkown"
}

type scanner struct {
	*scan.UTF8Scanner
}

func (s *scanner) Scan() bool {
	for s.UTF8Scanner.Scan() {
		if s.Token().Type != tokenSpace {
			return true
		}
	}
	return false
}

func newScanner(r io.Reader) *scanner {
	var (
		char   = scan.Char
		pat    = scan.Pat
		merge  = scan.Merge
		or     = scan.Or
		con    = scan.Con
		Tokens = scan.Tokens

		nonctrl   = char(`[:cntrl:]`).Negate()
		indent    = char(`\t `)
		lineBreak = char(`\n\r`)
		space     = merge(indent, lineBreak)
		any       = merge(nonctrl, space)
		invalid   = any.Negate()
		inline    = any.Exclude(lineBreak)
		delim     = char(`,{}`)
		empty     = pat(``)

		newline        = or(lineBreak, pat(`\r\n`))
		inlineComment  = con(pat(`//`), inline.ZeroOrMore(), or(newline, empty))
		quoted         = or(inline.Exclude(char(`"`)), pat(`\\"`))
		quotedString   = con(pat(`"`), quoted.ZeroOrMore(), pat(`"`))
		unquoted       = any.Exclude(delim, space)
		unquotedString = unquoted.OneOrMore()
		generalString  = or(quotedString, unquotedString)

		tokens = Tokens(
			invalid,
			inlineComment,
			char(`{`),
			char(`}`),
			char(`,`),
			generalString,
			space.OneOrMore(),
		)
	)
	s := scan.NewUTF8Scanner(tokens.String())
	err := s.Init(r)
	if err != nil {
		panic(err)
	}
	return &scanner{s}
}
<|endoftext|>


<commit_before>package gocr

import (
	"bufio"
	"fmt"
	"image"
	"io/ioutil"
	"log"
	"math"
	"os"
	"path/filepath"

	tf "github.com/tensorflow/tensorflow/tensorflow/go"
	"github.com/ugorji/go/codec"
)

type Marker struct {
	Start int
	End   int
}

func (m Marker) thickness() int {
	return m.End - m.Start
}

// Read the model from a file and return the Model
func ReadModel(path string) (Model, error) {
	file, err := os.Open(path)
	if err != nil {
		return Model{}, err
	}
	defer file.Close()

	decoder := codec.NewDecoder(file, new(codec.CborHandle))
	model := Model{}
	decoder.Decode(&model)

	return model, nil
}

// Find Marker of DarkSquare from given Matrix
// Direction 0 means it will iterate every rows
// Direction 1 means it will iterate every columns
func MarkersOfMatrix(data ImageMatrix, threshold float64, direction int) []Marker {
	r, c := data.Dims()
	markers := []Marker{}
	startMarker := -1
	isDarkSquare := false

	n := r
	if direction == 1 {
		n = c
	}

	for i := 0; i < n; i++ {

		var colAvg float64 = 1

		if direction == 0 {
			colAvg = float64(data.Row(i).Sum()) / float64(c)
		} else {
			colAvg = float64(data.Col(i).Sum()) / float64(r)
		}

		if colAvg >= threshold {
			if isDarkSquare && startMarker > 0 {
				markers = append(markers, Marker{
					Start: startMarker,
					End:   i,
				})
				startMarker = -1
				isDarkSquare = false
			} else {
				startMarker = i
			}
		} else {
			isDarkSquare = true
		}
	}

	return markers
}

// Scan the DarkSquare and return it as []mat64.Dense for each line
// And [][]mat64.Dense for each characther
func LinearScan(data ImageMatrix) [][]ImageMatrix {
	r, c := data.Dims()

	markers := MarkersOfMatrix(data, 0.9, 0)
	lines := []ImageMatrix{}

	rowPadding := 10
	for _, marker := range markers {

		if marker.thickness() <= 5 {
			continue
		}

		start := marker.Start - rowPadding
		if start < 0 {
			start = 0
		}

		end := marker.End + rowPadding
		if end >= r {
			end = r
		}

		line := data.Slice(start, end, 0, c)
		lines = append(lines, line)
	}

	charss := [][]ImageMatrix{}

	for _, line := range lines {
		markers := MarkersOfMatrix(line, 0.95, 1)
		r, c := line.Dims()
		chars := []ImageMatrix{}

		columnPadding := 2
		for _, marker := range markers {

			if marker.thickness() <= 5 {
				continue
			}

			start := marker.Start - columnPadding
			if start < 0 {
				start = 0
			}

			end := marker.End + columnPadding
			if end >= c {
				end = c
			}

			char := line.Slice(0, r, start, end)
			chars = append(chars, char)
		}

		charss = append(charss, chars)
	}

	return charss
}

func CirucularScan(image ImageMatrix) ([][]*Square, [][]ImageMatrix) {
	r, c := image.Dims()
	start := NewCoordinate(0, 0)
	imageSquare := NewSquare(start, NewCoordinate(r, c))
	exploredSquares := []*Square{}
	resultsSquare := []*Square{}

	for i := 0; i < c; i++ {
		for j := 0; j < r; j++ {
			exist := false
			for _, ea := range exploredSquares {
				if ea.Include(j, i) {
					exist = true
					break
				}
			}

			if exist {
				continue
			}

			if image.At(j, i) == 0 {
				coor := NewCoordinate(j, i)
				result := NewSquare(coor, coor)
				vcs := []*Coordinate{}

				circleRun(image, coor, &vcs, imageSquare, result)
				resultsSquare = append(resultsSquare, result)
				exploredSquares = append(exploredSquares, result)
			}
		}
	}

	charss := [][]ImageMatrix{}
	squaress := [][]*Square{}

	for _, result := range resultsSquare {
		if result.Width() < result.Height() {
			hat, i := findTopSquare(result, resultsSquare)
			if hat != nil {
				result.Merge(hat)
				resultsSquare = append(resultsSquare[:i], resultsSquare[i+1:]...)
			}
		}
	}

	for _, result := range resultsSquare {
		match := false
		for i, squares := range squaress {
			if result.AverageVerticalDistanceTo(squares[0]) < float64(squares[0].Height()) {
				charss[i] = append(charss[i], image.SliceSquare(result))
				squaress[i] = append(squaress[i], result)
				match = true
				break
			}
		}

		if !match {
			charss = append(charss, []ImageMatrix{image.SliceSquare(result)})
			squaress = append(squaress, []*Square{result})
		}
	}

	return squaress, charss
}

func circleRun(i ImageMatrix, c *Coordinate, vcs *[]*Coordinate, ia, rs *Square) {
	if c.IsInside(ia) {
		for _, cs := range *vcs {
			if c.row == cs.row && c.col == cs.col {
				return
			}
		}
		*vcs = append(*vcs, c)

		if i.At(c.row, c.col) == 0 {
			rs.Expand(c)

			circleRun(i, c.N(), vcs, ia, rs)
			circleRun(i, c.NE(), vcs, ia, rs)
			circleRun(i, c.E(), vcs, ia, rs)
			circleRun(i, c.SE(), vcs, ia, rs)
			circleRun(i, c.S(), vcs, ia, rs)
			circleRun(i, c.SW(), vcs, ia, rs)
			circleRun(i, c.W(), vcs, ia, rs)
			circleRun(i, c.NW(), vcs, ia, rs)
		}
	}
}

func findTopSquare(s *Square, squares []*Square) (*Square, int) {
	m := s.topLeft.col + (s.bottomRight.col-s.topLeft.col)/2
	ni := -1
	nd := float64((s.bottomRight.row - s.topLeft.row)) / 2

	for i := 0; i < len(squares); i++ {
		if squares[i].topLeft.col < m && m < squares[i].bottomRight.col {
			cd := s.NearestVerticalDistanceTo(squares[i])
			if nd > cd && cd > 0 {
				nd = cd
				ni = i
			}
		}
	}

	if ni < 0 {
		return nil, ni
	} else {
		return squares[ni], ni
	}
}

// ================================= Predictor =================================

type Predictor interface {
	inputWidth() int
	inputHeight() int
	Predicts(ImageMatrixs) []string
}

// ================================= Nearest Neighbor Predictor =================================
type NNPredictor struct {
	model *Model
}

func NewNNPredictor(model *Model) *NNPredictor {
	return &NNPredictor{
		model: model,
	}
}

func NewNNPredictorFromFile(path string) *NNPredictor {
	model, err := ReadModel(path)
	if err != nil {
		panic(err)
	}

	return &NNPredictor{
		model: &model,
	}
}

func (p *NNPredictor) inputHeight() int {
	r, _ := p.model.ModelImages[0].Data.Dims()
	return r
}

func (p *NNPredictor) inputWidth() int {
	_, c := p.model.ModelImages[0].Data.Dims()
	return c
}

func (p *NNPredictor) Predicts(images ImageMatrixs) []string {
	predictedLabels := make([]string, len(images))
	mr, mc := p.model.ModelImages[0].Data.Dims()

	for i, image := range images {
		resizedMatrix := PadAndResize(image, mr, mc)

		min := math.MaxFloat64
		for _, modelImage := range p.model.ModelImages {
			distance := EuclideanDistance(resizedMatrix, modelImage.Data)

			fmt.Println(modelImage.Label, distance, " ")
			if min > distance {
				min = distance
				predictedLabels[i] = modelImage.Label
			}
		}
	}

	return predictedLabels
}

// ================================= Tensor CNN Predictor =================================

type CNNPredictor struct {
	graph       *tf.Graph
	labels      []string
	InputHeight int
	InputWidth  int
}

func NewCNNPredictor(graph *tf.Graph, labels []string) *CNNPredictor {
	return &CNNPredictor{
		graph:  graph,
		labels: labels,
	}
}

func NewCNNPredictorFromDir(dir string) *CNNPredictor {
	graph, labels := loadModel(dir)
	return NewCNNPredictor(graph, labels)
}

func (p *CNNPredictor) inputHeight() int {
	return p.InputHeight
}

func (p *CNNPredictor) inputWidth() int {
	return p.InputWidth
}

func (p *CNNPredictor) Predicts(images ImageMatrixs) []string {

	session, err := tf.NewSession(p.graph, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer session.Close()

	kerasFlag, _ := tf.NewTensor(false)
	tensorImages, err := makeTensorFromImage(images)
	if err != nil {
		log.Fatal(err)
	}

	output, err := session.Run(
		map[tf.Output]*tf.Tensor{
			p.graph.Operation("convolution2d_input_1").Output(0): tensorImages,
			p.graph.Operation("keras_learning_phase").Output(0):  kerasFlag,
		},
		[]tf.Output{
			p.graph.Operation("ArgMax_1").Output(0),
		},
		nil)

	if err != nil {
		log.Fatal(err)
	}

	predictions := output[0].Value().([]int64)
	result := make([]string, len(predictions))
	for i, prediction := range predictions {
		result[i] = p.labels[prediction]
	}

	return result
}

func makeTensorFromImage(images ImageMatrixs) (*tf.Tensor, error) {

	o := make([][][][]float32, len(images))

	for k := 0; k < len(images); k++ {
		o[k] = make([][][]float32, len(images[k]))
		for i := 0; i < len(images[k]); i++ {
			o[k][i] = make([][]float32, len(images[k][i]))
			for j := 0; j < len(images[k][i]); j++ {
				o[k][i][j] = make([]float32, 1)
				o[k][i][j][0] = float32(images[k][i][j])
			}
		}
	}

	tensor, err := tf.NewTensor(o)
	if err != nil {
		return nil, err
	}

	return tensor, nil
}

func loadModel(dir string) (*tf.Graph, []string) {
	var (
		modelFile  = filepath.Join(dir, "model.pb")
		labelsFile = filepath.Join(dir, "labels.txt")
	)

	model, err := ioutil.ReadFile(modelFile)
	if err != nil {
		log.Fatal(err)
	}

	graph := tf.NewGraph()
	if err = graph.Import(model, ""); err != nil {
		log.Fatal(err)
	}

	file, err := os.Open(labelsFile)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	var labels []string
	for scanner.Scan() {
		labels = append(labels, scanner.Text())
	}

	return graph, labels
}

func ScanToStrings(p Predictor, image image.Image) []string {
	im := ImageToBinaryArray(image)
	squaress, charss := CirucularScan(im)
	results := []string{}

	for k, chars := range charss {
		datas := make([]ImageMatrix, len(chars))
		for i := 0; i < len(chars); i++ {
			datas[i] = PadAndResize(chars[i], p.inputHeight(), p.inputWidth())
		}

		texts := p.Predicts(datas)
		result := ""
		for i, text := range texts {
			if i < len(squaress[k])-1 {
				if squaress[k][i].NearestHorizontalDistanceTo(squaress[k][i+1]) > float64(squaress[k][i].Width()) {
					result += text + " "
					continue
				}
			}

			result += text
		}

		results = append(results, result)
	}

	return results
}
<commit_msg>Improve whitespace detection<commit_after>package gocr

import (
	"bufio"
	"fmt"
	"image"
	"io/ioutil"
	"log"
	"math"
	"os"
	"path/filepath"

	tf "github.com/tensorflow/tensorflow/tensorflow/go"
	"github.com/ugorji/go/codec"
)

type Marker struct {
	Start int
	End   int
}

func (m Marker) thickness() int {
	return m.End - m.Start
}

// Read the model from a file and return the Model
func ReadModel(path string) (Model, error) {
	file, err := os.Open(path)
	if err != nil {
		return Model{}, err
	}
	defer file.Close()

	decoder := codec.NewDecoder(file, new(codec.CborHandle))
	model := Model{}
	decoder.Decode(&model)

	return model, nil
}

// Find Marker of DarkSquare from given Matrix
// Direction 0 means it will iterate every rows
// Direction 1 means it will iterate every columns
func MarkersOfMatrix(data ImageMatrix, threshold float64, direction int) []Marker {
	r, c := data.Dims()
	markers := []Marker{}
	startMarker := -1
	isDarkSquare := false

	n := r
	if direction == 1 {
		n = c
	}

	for i := 0; i < n; i++ {

		var colAvg float64 = 1

		if direction == 0 {
			colAvg = float64(data.Row(i).Sum()) / float64(c)
		} else {
			colAvg = float64(data.Col(i).Sum()) / float64(r)
		}

		if colAvg >= threshold {
			if isDarkSquare && startMarker > 0 {
				markers = append(markers, Marker{
					Start: startMarker,
					End:   i,
				})
				startMarker = -1
				isDarkSquare = false
			} else {
				startMarker = i
			}
		} else {
			isDarkSquare = true
		}
	}

	return markers
}

// Scan the DarkSquare and return it as []mat64.Dense for each line
// And [][]mat64.Dense for each characther
func LinearScan(data ImageMatrix) [][]ImageMatrix {
	r, c := data.Dims()

	markers := MarkersOfMatrix(data, 0.9, 0)
	lines := []ImageMatrix{}

	rowPadding := 10
	for _, marker := range markers {

		if marker.thickness() <= 5 {
			continue
		}

		start := marker.Start - rowPadding
		if start < 0 {
			start = 0
		}

		end := marker.End + rowPadding
		if end >= r {
			end = r
		}

		line := data.Slice(start, end, 0, c)
		lines = append(lines, line)
	}

	charss := [][]ImageMatrix{}

	for _, line := range lines {
		markers := MarkersOfMatrix(line, 0.95, 1)
		r, c := line.Dims()
		chars := []ImageMatrix{}

		columnPadding := 2
		for _, marker := range markers {

			if marker.thickness() <= 5 {
				continue
			}

			start := marker.Start - columnPadding
			if start < 0 {
				start = 0
			}

			end := marker.End + columnPadding
			if end >= c {
				end = c
			}

			char := line.Slice(0, r, start, end)
			chars = append(chars, char)
		}

		charss = append(charss, chars)
	}

	return charss
}

func CirucularScan(image ImageMatrix) ([][]*Square, [][]ImageMatrix) {
	r, c := image.Dims()
	start := NewCoordinate(0, 0)
	imageSquare := NewSquare(start, NewCoordinate(r, c))
	exploredSquares := []*Square{}
	resultsSquare := []*Square{}

	for i := 0; i < c; i++ {
		for j := 0; j < r; j++ {
			exist := false
			for _, ea := range exploredSquares {
				if ea.Include(j, i) {
					exist = true
					break
				}
			}

			if exist {
				continue
			}

			if image.At(j, i) == 0 {
				coor := NewCoordinate(j, i)
				result := NewSquare(coor, coor)
				vcs := []*Coordinate{}

				circleRun(image, coor, &vcs, imageSquare, result)
				resultsSquare = append(resultsSquare, result)
				exploredSquares = append(exploredSquares, result)
			}
		}
	}

	charss := [][]ImageMatrix{}
	squaress := [][]*Square{}

	for _, result := range resultsSquare {
		if result.Width() < result.Height() {
			hat, i := findTopSquare(result, resultsSquare)
			if hat != nil {
				result.Merge(hat)
				resultsSquare = append(resultsSquare[:i], resultsSquare[i+1:]...)
			}
		}
	}

	for _, result := range resultsSquare {
		match := false
		for i, squares := range squaress {
			if result.AverageVerticalDistanceTo(squares[0]) < float64(squares[0].Height()) {
				charss[i] = append(charss[i], image.SliceSquare(result))
				squaress[i] = append(squaress[i], result)
				match = true
				break
			}
		}

		if !match {
			charss = append(charss, []ImageMatrix{image.SliceSquare(result)})
			squaress = append(squaress, []*Square{result})
		}
	}

	return squaress, charss
}

func circleRun(i ImageMatrix, c *Coordinate, vcs *[]*Coordinate, ia, rs *Square) {
	if c.IsInside(ia) {
		for _, cs := range *vcs {
			if c.row == cs.row && c.col == cs.col {
				return
			}
		}
		*vcs = append(*vcs, c)

		if i.At(c.row, c.col) == 0 {
			rs.Expand(c)

			circleRun(i, c.N(), vcs, ia, rs)
			circleRun(i, c.NE(), vcs, ia, rs)
			circleRun(i, c.E(), vcs, ia, rs)
			circleRun(i, c.SE(), vcs, ia, rs)
			circleRun(i, c.S(), vcs, ia, rs)
			circleRun(i, c.SW(), vcs, ia, rs)
			circleRun(i, c.W(), vcs, ia, rs)
			circleRun(i, c.NW(), vcs, ia, rs)
		}
	}
}

func findTopSquare(s *Square, squares []*Square) (*Square, int) {
	m := s.topLeft.col + (s.bottomRight.col-s.topLeft.col)/2
	ni := -1
	nd := float64((s.bottomRight.row - s.topLeft.row)) / 2

	for i := 0; i < len(squares); i++ {
		if squares[i].topLeft.col < m && m < squares[i].bottomRight.col {
			cd := s.NearestVerticalDistanceTo(squares[i])
			if nd > cd && cd > 0 {
				nd = cd
				ni = i
			}
		}
	}

	if ni < 0 {
		return nil, ni
	} else {
		return squares[ni], ni
	}
}

// ================================= Predictor =================================

type Predictor interface {
	inputWidth() int
	inputHeight() int
	Predicts(ImageMatrixs) []string
}

// ================================= Nearest Neighbor Predictor =================================
type NNPredictor struct {
	model *Model
}

func NewNNPredictor(model *Model) *NNPredictor {
	return &NNPredictor{
		model: model,
	}
}

func NewNNPredictorFromFile(path string) *NNPredictor {
	model, err := ReadModel(path)
	if err != nil {
		panic(err)
	}

	return &NNPredictor{
		model: &model,
	}
}

func (p *NNPredictor) inputHeight() int {
	r, _ := p.model.ModelImages[0].Data.Dims()
	return r
}

func (p *NNPredictor) inputWidth() int {
	_, c := p.model.ModelImages[0].Data.Dims()
	return c
}

func (p *NNPredictor) Predicts(images ImageMatrixs) []string {
	predictedLabels := make([]string, len(images))
	mr, mc := p.model.ModelImages[0].Data.Dims()

	for i, image := range images {
		resizedMatrix := PadAndResize(image, mr, mc)

		min := math.MaxFloat64
		for _, modelImage := range p.model.ModelImages {
			distance := EuclideanDistance(resizedMatrix, modelImage.Data)

			fmt.Println(modelImage.Label, distance, " ")
			if min > distance {
				min = distance
				predictedLabels[i] = modelImage.Label
			}
		}
	}

	return predictedLabels
}

// ================================= Tensor CNN Predictor =================================

type CNNPredictor struct {
	graph       *tf.Graph
	labels      []string
	InputHeight int
	InputWidth  int
}

func NewCNNPredictor(graph *tf.Graph, labels []string) *CNNPredictor {
	return &CNNPredictor{
		graph:  graph,
		labels: labels,
	}
}

func NewCNNPredictorFromDir(dir string) *CNNPredictor {
	graph, labels := loadModel(dir)
	return NewCNNPredictor(graph, labels)
}

func (p *CNNPredictor) inputHeight() int {
	return p.InputHeight
}

func (p *CNNPredictor) inputWidth() int {
	return p.InputWidth
}

func (p *CNNPredictor) Predicts(images ImageMatrixs) []string {

	session, err := tf.NewSession(p.graph, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer session.Close()

	kerasFlag, _ := tf.NewTensor(false)
	tensorImages, err := makeTensorFromImage(images)
	if err != nil {
		log.Fatal(err)
	}

	output, err := session.Run(
		map[tf.Output]*tf.Tensor{
			p.graph.Operation("convolution2d_input_1").Output(0): tensorImages,
			p.graph.Operation("keras_learning_phase").Output(0):  kerasFlag,
		},
		[]tf.Output{
			p.graph.Operation("ArgMax_1").Output(0),
		},
		nil)

	if err != nil {
		log.Fatal(err)
	}

	predictions := output[0].Value().([]int64)
	result := make([]string, len(predictions))
	for i, prediction := range predictions {
		result[i] = p.labels[prediction]
	}

	return result
}

func makeTensorFromImage(images ImageMatrixs) (*tf.Tensor, error) {

	o := make([][][][]float32, len(images))

	for k := 0; k < len(images); k++ {
		o[k] = make([][][]float32, len(images[k]))
		for i := 0; i < len(images[k]); i++ {
			o[k][i] = make([][]float32, len(images[k][i]))
			for j := 0; j < len(images[k][i]); j++ {
				o[k][i][j] = make([]float32, 1)
				o[k][i][j][0] = float32(images[k][i][j])
			}
		}
	}

	tensor, err := tf.NewTensor(o)
	if err != nil {
		return nil, err
	}

	return tensor, nil
}

func loadModel(dir string) (*tf.Graph, []string) {
	var (
		modelFile  = filepath.Join(dir, "model.pb")
		labelsFile = filepath.Join(dir, "labels.txt")
	)

	model, err := ioutil.ReadFile(modelFile)
	if err != nil {
		log.Fatal(err)
	}

	graph := tf.NewGraph()
	if err = graph.Import(model, ""); err != nil {
		log.Fatal(err)
	}

	file, err := os.Open(labelsFile)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	var labels []string
	for scanner.Scan() {
		labels = append(labels, scanner.Text())
	}

	return graph, labels
}

func ScanToStrings(p Predictor, image image.Image) []string {
	im := ImageToBinaryArray(image)
	squaress, charss := CirucularScan(im)
	results := []string{}

	for k, chars := range charss {
		datas := make([]ImageMatrix, len(chars))
		for i := 0; i < len(chars); i++ {
			datas[i] = PadAndResize(chars[i], p.inputHeight(), p.inputWidth())
		}

		texts := p.Predicts(datas)
		result := ""
		dist := 0.0

		for i := 0; i < len(texts)-1; i++ {
			dist += squaress[k][i].NearestHorizontalDistanceTo(squaress[k][i+1])
		}

		avgDist := dist / float64(len(texts)-1)

		for i, text := range texts {
			if i < len(squaress[k])-1 {
				if squaress[k][i].NearestHorizontalDistanceTo(squaress[k][i+1]) > avgDist {
					result += text + " "
					continue
				}
			}

			result += text
		}

		results = append(results, result)
	}

	return results
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"go/build"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"sourcegraph.com/sourcegraph/srclib"
	"sourcegraph.com/sourcegraph/srclib/unit"
)

func init() {
	_, err := parser.AddCommand("scan",
		"scan for Go packages",
		"Scan the directory tree rooted at the current directory for Go packages.",
		&scanCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

type ScanCmd struct {
	Repo   string `long:"repo" description:"repository URI" value-name:"URI"`
	Subdir string `long:"subdir" description:"subdirectory in repository" value-name:"DIR"`
}

var scanCmd ScanCmd

func (c *ScanCmd) Execute(args []string) error {
	if c.Repo == "" && os.Getenv("IN_DOCKER_CONTAINER") != "" {
		log.Println("Warning: no --repo specified, and tool is running in a Docker container (i.e., without awareness of host's GOPATH). Go import paths in source units produced by the scanner may be inaccurate. To fix this, ensure that the --repo URI is specified. Report this issue if you are seeing it unexpectedly.")
	}

	if err := json.NewDecoder(os.Stdin).Decode(&config); err != nil {
		return err
	}
	if err := os.Stdin.Close(); err != nil {
		return err
	}

	// Automatically detect vendored dirs (check for vendor/src and
	// Godeps/_workspace/src) and set up GOPATH pointing to them if
	// they exist.
	var setAutoGOPATH bool
	if config.GOPATH == "" {
		vendorDirs := []string{"vendor", "Godeps/_workspace"}
		var foundGOPATHs []string
		for _, vdir := range vendorDirs {
			if fi, err := os.Stat(filepath.Join(cwd, vdir, "src")); err == nil && fi.Mode().IsDir() {
				foundGOPATHs = append(foundGOPATHs, vdir)
				setAutoGOPATH = true
				log.Printf("Adding %s to GOPATH (auto-detected Go vendored dependencies source dir %s). If you don't want this, make a Srcfile with a GOPATH property set to something other than the empty string.", vdir, filepath.Join(vdir, "src"))
			}
		}
		config.GOPATH = strings.Join(foundGOPATHs, string(filepath.ListSeparator))
	}

	if err := config.apply(); err != nil {
		return err
	}

	cwd, err := filepath.EvalSymlinks(getCWD())
	if err != nil {
		return err
	}
	scanDir := cwd
	if !isInGopath(scanDir) {
		scanDir = filepath.Join(cwd, srclibGopath, "src", filepath.FromSlash(c.Repo))
		buildContext.GOPATH = filepath.Join(cwd, srclibGopath) + string(os.PathListSeparator) + buildContext.GOPATH

		os.RemoveAll(srclibGopath) // ignore error
		if err := os.MkdirAll(filepath.Dir(scanDir), 0777); err != nil {
			return err
		}
		rel, err := filepath.Rel(filepath.Dir(scanDir), cwd)
		if err != nil {
			return err
		}
		if err := os.Symlink(rel, scanDir); err != nil {
			return err
		}
	}

	units, err := scan(scanDir)
	if err != nil {
		return err
	}

	if len(config.PkgPatterns) != 0 {
		matchers := make([]func(name string) bool, len(config.PkgPatterns))
		for i, pattern := range config.PkgPatterns {
			matchers[i] = matchPattern(pattern)
		}

		var filteredUnits []*unit.SourceUnit
		for _, unit := range units {
			for _, m := range matchers {
				if m(unit.Name) {
					filteredUnits = append(filteredUnits, unit)
					break
				}
			}
		}
		units = filteredUnits
	}

	// Fix up import paths to be consistent when running as a program and as
	// a Docker container. But if a GOROOT is set, then we probably want import
	// paths to not contain the repo, so only do this if there's no GOROOT set
	// in the Srcfile.
	if os.Getenv("IN_DOCKER_CONTAINER") != "" && config.GOROOT == "" {
		for _, u := range units {
			pkg := u.Data.(*build.Package)
			pkg.ImportPath = filepath.Join(c.Repo, c.Subdir, pkg.Dir)
			u.Name = pkg.ImportPath
		}
	}

	// Make vendored dep unit names (package import paths) relative to
	// vendored src dir, not to top-level dir.
	if config.GOPATH != "" {
		dirs := filepath.SplitList(config.GOPATH)
		for _, dir := range dirs {
			relDir, err := filepath.Rel(cwd, dir)
			if err != nil {
				return err
			}
			srcDir := filepath.Join(relDir, "src")
			for _, u := range units {
				pkg := u.Data.(*build.Package)
				if strings.HasPrefix(pkg.Dir, srcDir) {
					relImport, err := filepath.Rel(srcDir, pkg.Dir)
					if err != nil {
						return err
					}
					pkg.ImportPath = relImport
					u.Name = pkg.ImportPath
				}
			}
		}
	}

	// make files relative to repository root
	for _, u := range units {
		pkgSubdir := filepath.Join(c.Subdir, u.Data.(*build.Package).Dir)
		for i, f := range u.Files {
			u.Files[i] = filepath.Join(pkgSubdir, f)
		}
	}

	// If we automatically set the GOPATH based on the presence of
	// vendor dirs, then we need to pass the GOPATH to the units
	// because it is not persisted in the Srcfile. Otherwise the other
	// tools would never see the auto-set GOPATH.
	if setAutoGOPATH {
		for _, u := range units {
			if u.Config == nil {
				u.Config = map[string]interface{}{}
			}

			dirs := filepath.SplitList(config.GOPATH)
			for i, dir := range dirs {
				relDir, err := filepath.Rel(cwd, dir)
				if err != nil {
					return err
				}
				dirs[i] = relDir
			}
			u.Config["GOPATH"] = strings.Join(dirs, string(filepath.ListSeparator))
		}
	}

	b, err := json.MarshalIndent(units, "", "  ")
	if err != nil {
		return err
	}
	if _, err := os.Stdout.Write(b); err != nil {
		return err
	}
	return nil
}

func isInGopath(path string) bool {
	for _, gopath := range filepath.SplitList(buildContext.GOPATH) {
		if strings.HasPrefix(evalSymlinks(path), filepath.Join(evalSymlinks(gopath), "src")) {
			return true
		}
	}
	return false
}

func scan(scanDir string) ([]*unit.SourceUnit, error) {
	// TODO(sqs): include xtest, but we'll have to make them have a distinctly
	// namespaced def path from the non-xtest pkg.

	pkgs, err := scanForPackages(scanDir)
	if err != nil {
		return nil, err
	}

	var units []*unit.SourceUnit
	for _, pkg := range pkgs {
		// Collect all files
		var files []string
		files = append(files, pkg.GoFiles...)
		files = append(files, pkg.CgoFiles...)
		files = append(files, pkg.IgnoredGoFiles...)
		files = append(files, pkg.CFiles...)
		files = append(files, pkg.CXXFiles...)
		files = append(files, pkg.MFiles...)
		files = append(files, pkg.HFiles...)
		files = append(files, pkg.SFiles...)
		files = append(files, pkg.SwigFiles...)
		files = append(files, pkg.SwigCXXFiles...)
		files = append(files, pkg.SysoFiles...)
		files = append(files, pkg.TestGoFiles...)
		files = append(files, pkg.XTestGoFiles...)

		// Collect all imports. We use a map to remove duplicates.
		var imports []string
		imports = append(imports, pkg.Imports...)
		imports = append(imports, pkg.TestImports...)
		imports = append(imports, pkg.XTestImports...)
		imports = uniq(imports)
		sort.Strings(imports)

		// Create appropriate type for (unit).SourceUnit
		deps := make([]interface{}, len(imports))
		for i, imp := range imports {
			deps[i] = imp
		}

		pkg.Dir, err = filepath.Rel(scanDir, pkg.Dir)
		if err != nil {
			return nil, err
		}
		pkg.BinDir = ""
		pkg.ConflictDir = ""

		// Root differs depending on the system, so it's hard to compare results
		// across environments (when running as a program). Clear it so we can
		// compare results in tests more easily.
		pkg.Root = ""

		units = append(units, &unit.SourceUnit{
			Name:         pkg.ImportPath,
			Type:         "GoPackage",
			Dir:          pkg.Dir,
			Files:        files,
			Data:         pkg,
			Dependencies: deps,
			Ops:          map[string]*srclib.ToolRef{"depresolve": nil, "graph": nil},
		})
	}

	return units, nil
}

func scanForPackages(dir string) ([]*build.Package, error) {
	var pkgs []*build.Package

	pkg, err := buildContext.ImportDir(dir, 0)
	if _, isNoGoError := err.(*build.NoGoError); err != nil && !isNoGoError {
		return nil, err
	}
	if err == nil {
		pkgs = append(pkgs, pkg)
	}

	infos, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	for _, info := range infos {
		name := info.Name()
		fullPath := filepath.Join(dir, name)
		if info.IsDir() && ((name[0] != '.' && name[0] != '_' && name != "testdata") || (strings.HasSuffix(fullPath, "Godeps/_workspace") && !config.SkipGodeps)) {
			subPkgs, err := scanForPackages(fullPath)
			if err != nil {
				return nil, err
			}
			pkgs = append(pkgs, subPkgs...)
		}
	}

	return pkgs, nil
}

// matchPattern(pattern)(name) reports whether
// name matches pattern.  Pattern is a limited glob
// pattern in which '...' means 'any string' and there
// is no other special syntax.
func matchPattern(pattern string) func(name string) bool {
	re := regexp.QuoteMeta(pattern)
	re = strings.Replace(re, `\.\.\.`, `.*`, -1)
	// Special case: foo/... matches foo too.
	if strings.HasSuffix(re, `/.*`) {
		re = re[:len(re)-len(`/.*`)] + `(/.*)?`
	}
	reg := regexp.MustCompile(`^` + re + `$`)
	return func(name string) bool {
		return reg.MatchString(name)
	}
}
<commit_msg>don't store pkg.SrcRoot and pkg.PkgRoot<commit_after>package main

import (
	"encoding/json"
	"go/build"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"sourcegraph.com/sourcegraph/srclib"
	"sourcegraph.com/sourcegraph/srclib/unit"
)

func init() {
	_, err := parser.AddCommand("scan",
		"scan for Go packages",
		"Scan the directory tree rooted at the current directory for Go packages.",
		&scanCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

type ScanCmd struct {
	Repo   string `long:"repo" description:"repository URI" value-name:"URI"`
	Subdir string `long:"subdir" description:"subdirectory in repository" value-name:"DIR"`
}

var scanCmd ScanCmd

func (c *ScanCmd) Execute(args []string) error {
	if c.Repo == "" && os.Getenv("IN_DOCKER_CONTAINER") != "" {
		log.Println("Warning: no --repo specified, and tool is running in a Docker container (i.e., without awareness of host's GOPATH). Go import paths in source units produced by the scanner may be inaccurate. To fix this, ensure that the --repo URI is specified. Report this issue if you are seeing it unexpectedly.")
	}

	if err := json.NewDecoder(os.Stdin).Decode(&config); err != nil {
		return err
	}
	if err := os.Stdin.Close(); err != nil {
		return err
	}

	// Automatically detect vendored dirs (check for vendor/src and
	// Godeps/_workspace/src) and set up GOPATH pointing to them if
	// they exist.
	var setAutoGOPATH bool
	if config.GOPATH == "" {
		vendorDirs := []string{"vendor", "Godeps/_workspace"}
		var foundGOPATHs []string
		for _, vdir := range vendorDirs {
			if fi, err := os.Stat(filepath.Join(cwd, vdir, "src")); err == nil && fi.Mode().IsDir() {
				foundGOPATHs = append(foundGOPATHs, vdir)
				setAutoGOPATH = true
				log.Printf("Adding %s to GOPATH (auto-detected Go vendored dependencies source dir %s). If you don't want this, make a Srcfile with a GOPATH property set to something other than the empty string.", vdir, filepath.Join(vdir, "src"))
			}
		}
		config.GOPATH = strings.Join(foundGOPATHs, string(filepath.ListSeparator))
	}

	if err := config.apply(); err != nil {
		return err
	}

	cwd, err := filepath.EvalSymlinks(getCWD())
	if err != nil {
		return err
	}
	scanDir := cwd
	if !isInGopath(scanDir) {
		scanDir = filepath.Join(cwd, srclibGopath, "src", filepath.FromSlash(c.Repo))
		buildContext.GOPATH = filepath.Join(cwd, srclibGopath) + string(os.PathListSeparator) + buildContext.GOPATH

		os.RemoveAll(srclibGopath) // ignore error
		if err := os.MkdirAll(filepath.Dir(scanDir), 0777); err != nil {
			return err
		}
		rel, err := filepath.Rel(filepath.Dir(scanDir), cwd)
		if err != nil {
			return err
		}
		if err := os.Symlink(rel, scanDir); err != nil {
			return err
		}
	}

	units, err := scan(scanDir)
	if err != nil {
		return err
	}

	if len(config.PkgPatterns) != 0 {
		matchers := make([]func(name string) bool, len(config.PkgPatterns))
		for i, pattern := range config.PkgPatterns {
			matchers[i] = matchPattern(pattern)
		}

		var filteredUnits []*unit.SourceUnit
		for _, unit := range units {
			for _, m := range matchers {
				if m(unit.Name) {
					filteredUnits = append(filteredUnits, unit)
					break
				}
			}
		}
		units = filteredUnits
	}

	// Fix up import paths to be consistent when running as a program and as
	// a Docker container. But if a GOROOT is set, then we probably want import
	// paths to not contain the repo, so only do this if there's no GOROOT set
	// in the Srcfile.
	if os.Getenv("IN_DOCKER_CONTAINER") != "" && config.GOROOT == "" {
		for _, u := range units {
			pkg := u.Data.(*build.Package)
			pkg.ImportPath = filepath.Join(c.Repo, c.Subdir, pkg.Dir)
			u.Name = pkg.ImportPath
		}
	}

	// Make vendored dep unit names (package import paths) relative to
	// vendored src dir, not to top-level dir.
	if config.GOPATH != "" {
		dirs := filepath.SplitList(config.GOPATH)
		for _, dir := range dirs {
			relDir, err := filepath.Rel(cwd, dir)
			if err != nil {
				return err
			}
			srcDir := filepath.Join(relDir, "src")
			for _, u := range units {
				pkg := u.Data.(*build.Package)
				if strings.HasPrefix(pkg.Dir, srcDir) {
					relImport, err := filepath.Rel(srcDir, pkg.Dir)
					if err != nil {
						return err
					}
					pkg.ImportPath = relImport
					u.Name = pkg.ImportPath
				}
			}
		}
	}

	// make files relative to repository root
	for _, u := range units {
		pkgSubdir := filepath.Join(c.Subdir, u.Data.(*build.Package).Dir)
		for i, f := range u.Files {
			u.Files[i] = filepath.Join(pkgSubdir, f)
		}
	}

	// If we automatically set the GOPATH based on the presence of
	// vendor dirs, then we need to pass the GOPATH to the units
	// because it is not persisted in the Srcfile. Otherwise the other
	// tools would never see the auto-set GOPATH.
	if setAutoGOPATH {
		for _, u := range units {
			if u.Config == nil {
				u.Config = map[string]interface{}{}
			}

			dirs := filepath.SplitList(config.GOPATH)
			for i, dir := range dirs {
				relDir, err := filepath.Rel(cwd, dir)
				if err != nil {
					return err
				}
				dirs[i] = relDir
			}
			u.Config["GOPATH"] = strings.Join(dirs, string(filepath.ListSeparator))
		}
	}

	b, err := json.MarshalIndent(units, "", "  ")
	if err != nil {
		return err
	}
	if _, err := os.Stdout.Write(b); err != nil {
		return err
	}
	return nil
}

func isInGopath(path string) bool {
	for _, gopath := range filepath.SplitList(buildContext.GOPATH) {
		if strings.HasPrefix(evalSymlinks(path), filepath.Join(evalSymlinks(gopath), "src")) {
			return true
		}
	}
	return false
}

func scan(scanDir string) ([]*unit.SourceUnit, error) {
	// TODO(sqs): include xtest, but we'll have to make them have a distinctly
	// namespaced def path from the non-xtest pkg.

	pkgs, err := scanForPackages(scanDir)
	if err != nil {
		return nil, err
	}

	var units []*unit.SourceUnit
	for _, pkg := range pkgs {
		// Collect all files
		var files []string
		files = append(files, pkg.GoFiles...)
		files = append(files, pkg.CgoFiles...)
		files = append(files, pkg.IgnoredGoFiles...)
		files = append(files, pkg.CFiles...)
		files = append(files, pkg.CXXFiles...)
		files = append(files, pkg.MFiles...)
		files = append(files, pkg.HFiles...)
		files = append(files, pkg.SFiles...)
		files = append(files, pkg.SwigFiles...)
		files = append(files, pkg.SwigCXXFiles...)
		files = append(files, pkg.SysoFiles...)
		files = append(files, pkg.TestGoFiles...)
		files = append(files, pkg.XTestGoFiles...)

		// Collect all imports. We use a map to remove duplicates.
		var imports []string
		imports = append(imports, pkg.Imports...)
		imports = append(imports, pkg.TestImports...)
		imports = append(imports, pkg.XTestImports...)
		imports = uniq(imports)
		sort.Strings(imports)

		// Create appropriate type for (unit).SourceUnit
		deps := make([]interface{}, len(imports))
		for i, imp := range imports {
			deps[i] = imp
		}

		pkg.Dir, err = filepath.Rel(scanDir, pkg.Dir)
		if err != nil {
			return nil, err
		}
		pkg.BinDir = ""
		pkg.ConflictDir = ""

		// Root differs depending on the system, so it's hard to compare results
		// across environments (when running as a program). Clear it so we can
		// compare results in tests more easily.
		pkg.Root = ""
		pkg.SrcRoot = ""
		pkg.PkgRoot = ""

		units = append(units, &unit.SourceUnit{
			Name:         pkg.ImportPath,
			Type:         "GoPackage",
			Dir:          pkg.Dir,
			Files:        files,
			Data:         pkg,
			Dependencies: deps,
			Ops:          map[string]*srclib.ToolRef{"depresolve": nil, "graph": nil},
		})
	}

	return units, nil
}

func scanForPackages(dir string) ([]*build.Package, error) {
	var pkgs []*build.Package

	pkg, err := buildContext.ImportDir(dir, 0)
	if _, isNoGoError := err.(*build.NoGoError); err != nil && !isNoGoError {
		return nil, err
	}
	if err == nil {
		pkgs = append(pkgs, pkg)
	}

	infos, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	for _, info := range infos {
		name := info.Name()
		fullPath := filepath.Join(dir, name)
		if info.IsDir() && ((name[0] != '.' && name[0] != '_' && name != "testdata") || (strings.HasSuffix(fullPath, "Godeps/_workspace") && !config.SkipGodeps)) {
			subPkgs, err := scanForPackages(fullPath)
			if err != nil {
				return nil, err
			}
			pkgs = append(pkgs, subPkgs...)
		}
	}

	return pkgs, nil
}

// matchPattern(pattern)(name) reports whether
// name matches pattern.  Pattern is a limited glob
// pattern in which '...' means 'any string' and there
// is no other special syntax.
func matchPattern(pattern string) func(name string) bool {
	re := regexp.QuoteMeta(pattern)
	re = strings.Replace(re, `\.\.\.`, `.*`, -1)
	// Special case: foo/... matches foo too.
	if strings.HasSuffix(re, `/.*`) {
		re = re[:len(re)-len(`/.*`)] + `(/.*)?`
	}
	reg := regexp.MustCompile(`^` + re + `$`)
	return func(name string) bool {
		return reg.MatchString(name)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"time"

	"github.com/andybons/hipchat"
)

const (
	timeout time.Duration = 2
)

func SendMessage(msg string, config Config) {
	c := hipchat.Client{AuthToken: config.Token}
	request := hipchat.MessageRequest{
		RoomId:        config.Room,
		From:          config.From,
		Message:       msg,
		Color:         hipchat.ColorPurple,
		MessageFormat: hipchat.FormatText,
		Notify:        true,
	}

	errc := make(chan error, 1)
	okc := make(chan bool, 1)
	go func() {
		err := c.PostMessage(request)
		if err != nil {
			errc <- err
		} else {
			okc <- true
		}
	}()

	select {
	case <-okc:
	case err := <-errc:
		log.Printf("Cannot send message: %s %q", msg, err)
	case <-time.After(time.Second * timeout):
		log.Printf("Timout after %d seconds", timeout)
	}
}
<commit_msg>Added message sent upon successfull sent message<commit_after>package main

import (
	"log"
	"time"

	"github.com/andybons/hipchat"
)

const (
	timeout time.Duration = 2
)

func SendMessage(msg string, config Config) {
	c := hipchat.Client{AuthToken: config.Token}
	request := hipchat.MessageRequest{
		RoomId:        config.Room,
		From:          config.From,
		Message:       msg,
		Color:         hipchat.ColorPurple,
		MessageFormat: hipchat.FormatText,
		Notify:        true,
	}

	errc := make(chan error, 1)
	okc := make(chan bool, 1)
	go func() {
		err := c.PostMessage(request)
		if err != nil {
			errc <- err
		} else {
			okc <- true
		}
	}()

	select {
	case <-okc:
		log.Println("Message sent successfully \U0001f604")
	case err := <-errc:
		log.Printf("Cannot send message: %s %q", msg, err)
	case <-time.After(time.Second * timeout):
		log.Printf("Timout after %d seconds", timeout)
	}
}
<|endoftext|>


<commit_before>package main

// sērus m (feminine sēra, neuter sērum); first/second declension
//
// late, too late
// slow, tardy
//
import (
	"errors"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"syscall"
	"time"
)

func main() {
	// Call realMain instead of doing the work here so we can use
	// `defer` statements within the function and have them work properly.
	// (defers aren't called with os.Exit)
	os.Exit(realMain())
}

func realMain() int {

	logger := &VerboseLog{}

	keyName := strings.Join(os.Args[2:], " ")

	mutex, err := NewMutex("/etc/sera.json", keyName, logger)
	if err != nil {
		logger.Printf("%s\n", err)
		return 2
	}

	keyExpiry, err := expiryArg()
	if err != nil {
		logger.Printf("%s\n", err)
		return 2
	}

	mutex.SetExpiry(keyExpiry)
	mutex.SetTries(10)
	mutex.SetDelay(keyExpiry / 10)
	mutex.SetFactor(0.01)

	if err = mutex.Lock(); err != nil {
		logger.Printf("%s\n", err)
		return 2
	}
	defer mutex.Unlock()

	cmd := exec.Command(os.Args[2:][0])
	cmd.Args = os.Args[2:]

	err = PipeCommandOutput(cmd)
	if err != nil {
		logger.Printf("%s\n", err)
		return 2
	}

	exitStatus, err := RunCommand(cmd)
	if err != nil {
		logger.Printf("%s\n", err)
		return exitStatus
	}

	return exitStatus
}

func RunCommand(cmd *exec.Cmd) (existatus int, err error) {
	// generic failure
	if err := cmd.Start(); err != nil {
		return 2, err
	}

	if err := cmd.Wait(); err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			// The program has exited with an exit code != 0
			// This works on both Unix and Windows. Although package
			// syscall is generally platform dependent, WaitStatus is
			// defined for both Unix and Windows and in both cases has
			// an ExitStatus() method with the same signature.
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				return status.ExitStatus(), nil
			}
		}
		return 2, err
	}
	return 0, nil
}

// get the commands stdout and stderr
func PipeCommandOutput(cmd *exec.Cmd) (err error) {

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	// get the commands stdout and stderr
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	// As soon the command prints to its stdout/stderr, print to the "real" stdout/stderr
	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)

	return nil
}

func expiryArg() (expiry time.Duration, err error) {
	args := os.Args[1:]
	if len(args) < 1 {
		err = errors.New("Usage: sera <expiry in sec> <command>")
		return
	}

	seconds, err := strconv.Atoi(args[0])
	if err != nil {
		return 0, err
	}
	expiry = time.Duration(seconds) * time.Second
	return
}
<commit_msg>If not enough arguments are passed in, show usage information<commit_after>package main

// sērus m (feminine sēra, neuter sērum); first/second declension
//
// late, too late
// slow, tardy
//
import (
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"syscall"
	"time"
)

func main() {
	// Call realMain instead of doing the work here so we can use
	// `defer` statements within the function and have them work properly.
	// (defers aren't called with os.Exit)
	os.Exit(realMain())
}

func realMain() int {

	logger := &VerboseLog{}

    if len(os.Args) < 2 {
        logger.Printf("Usage: sera <expiry in sec> <command>\n")
        return 2
    }

	keyName := strings.Join(os.Args[2:], " ")

	mutex, err := NewMutex("/etc/sera.json", keyName, logger)
	if err != nil {
		logger.Printf("%s\n", err)
		return 2
	}

	keyExpiry, err := expiryArg()
	if err != nil {
		logger.Printf("%s\n", err)
		return 2
	}

	mutex.SetExpiry(keyExpiry)
	mutex.SetTries(10)
	mutex.SetDelay(keyExpiry / 10)
	mutex.SetFactor(0.01)

	if err = mutex.Lock(); err != nil {
		logger.Printf("%s\n", err)
		return 2
	}
	defer mutex.Unlock()

	cmd := exec.Command(os.Args[2:][0])
	cmd.Args = os.Args[2:]

	err = PipeCommandOutput(cmd)
	if err != nil {
		logger.Printf("%s\n", err)
		return 2
	}

	exitStatus, err := RunCommand(cmd)
	if err != nil {
		logger.Printf("%s\n", err)
		return exitStatus
	}

	return exitStatus
}

func RunCommand(cmd *exec.Cmd) (existatus int, err error) {
	// generic failure
	if err := cmd.Start(); err != nil {
		return 2, err
	}

	if err := cmd.Wait(); err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			// The program has exited with an exit code != 0
			// This works on both Unix and Windows. Although package
			// syscall is generally platform dependent, WaitStatus is
			// defined for both Unix and Windows and in both cases has
			// an ExitStatus() method with the same signature.
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				return status.ExitStatus(), nil
			}
		}
		return 2, err
	}
	return 0, nil
}

// get the commands stdout and stderr
func PipeCommandOutput(cmd *exec.Cmd) (err error) {

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	// get the commands stdout and stderr
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	// As soon the command prints to its stdout/stderr, print to the "real" stdout/stderr
	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)

	return nil
}

func expiryArg() (expiry time.Duration, err error) {
	args := os.Args[1:]

	seconds, err := strconv.Atoi(args[0])
	if err != nil {
		return 0, err
	}
	expiry = time.Duration(seconds) * time.Second
	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"log"
	"net/http"
	"os"
)

func applyHeadersHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")
		h.ServeHTTP(w, r)
	})
}

func registerHandlers(mux *http.ServeMux, paths map[string]string) {
	var handler http.Handler
	for pattern, root := range paths {
		fileInfo, err := os.Stat(root)

		if err != nil {
			log.Println("Error: path doesn't exist: " + root)
		} else {
			if fileInfo.IsDir() {
				log.Printf("Registering handler with pattern: %s, root path: %s",
					pattern, root)
				handler = applyHeadersHandler(http.FileServer(http.Dir(root)))
				mux.Handle(pattern, http.StripPrefix(pattern, handler))
			} else {
				log.Printf("Registering handler with pattern: %s, file: %s",
					pattern, root)
				handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					http.ServeFile(w, r, root)
				})
				mux.Handle(pattern, applyHeadersHandler(handler))
			}
		}
	}
}

func parsePaths(paths string) (pathMap map[string]string, err error) {
	err = json.Unmarshal([]byte(paths), &pathMap)
	return
}

func main() {
	var listen = flag.String("listen", ":8080",
		"Interface/port to listen on. eg. :8080 or 127.0.0.1:8080")
	var pathsRaw = flag.String("paths", `{"/": "."}`,
		"Paths to serve. A json object with the keys as the url pattern, and "+
			"the value as the root. Default serves current folder.")
	flag.Parse()

	paths, err := parsePaths(*pathsRaw)

	if err != nil {
		log.Fatal(err)
	}

	mux := http.NewServeMux()
	registerHandlers(mux, paths)
	log.Println("Listening on: ", *listen)

	log.Fatal(http.ListenAndServe(*listen, mux))
}
<commit_msg>Fix cache headers in case used as origin.<commit_after>package main

import (
	"encoding/json"
	"flag"
	"log"
	"net/http"
	"os"
)

func applyHeadersHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Cache-Control", "No-Cache")
		h.ServeHTTP(w, r)
	})
}

func registerHandlers(mux *http.ServeMux, paths map[string]string) {
	var handler http.Handler
	for pattern, root := range paths {
		fileInfo, err := os.Stat(root)

		if err != nil {
			log.Println("Error: path doesn't exist: " + root)
		} else {
			if fileInfo.IsDir() {
				log.Printf("Registering handler with pattern: %s, root path: %s",
					pattern, root)
				handler = applyHeadersHandler(http.FileServer(http.Dir(root)))
				mux.Handle(pattern, http.StripPrefix(pattern, handler))
			} else {
				log.Printf("Registering handler with pattern: %s, file: %s",
					pattern, root)
				handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					http.ServeFile(w, r, root)
				})
				mux.Handle(pattern, applyHeadersHandler(handler))
			}
		}
	}
}

func parsePaths(paths string) (pathMap map[string]string, err error) {
	err = json.Unmarshal([]byte(paths), &pathMap)
	return
}

func main() {
	var listen = flag.String("listen", ":8080",
		"Interface/port to listen on. eg. :8080 or 127.0.0.1:8080")
	var pathsRaw = flag.String("paths", `{"/": "."}`,
		"Paths to serve. A json object with the keys as the url pattern, and "+
			"the value as the root. Default serves current folder.")
	flag.Parse()

	paths, err := parsePaths(*pathsRaw)

	if err != nil {
		log.Fatal(err)
	}

	mux := http.NewServeMux()
	registerHandlers(mux, paths)
	log.Println("Listening on: ", *listen)

	log.Fatal(http.ListenAndServe(*listen, mux))
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package virtcontainers

import (
	"fmt"
	"os"
	"os/exec"
	"syscall"
	"time"

	ns "github.com/containers/virtcontainers/pkg/nsenter"
	"github.com/mitchellh/mapstructure"
	"github.com/sirupsen/logrus"
)

// ShimType describes a shim type.
type ShimType string

const (
	// CCShimType is the ccShim.
	CCShimType ShimType = "ccShim"

	// NoopShimType is the noopShim.
	NoopShimType ShimType = "noopShim"

	// KataShimType is the Kata Containers shim type.
	KataShimType ShimType = "kataShim"
)

var waitForShimTimeout = 10.0
var consoleFileMode = os.FileMode(0660)

// ShimParams is the structure providing specific parameters needed
// for the execution of the shim binary.
type ShimParams struct {
	Container string
	Token     string
	URL       string
	Console   string
	Terminal  bool
	Detach    bool
	PID       int
	CreateNS  []ns.NSType
}

// ShimConfig is the structure providing specific configuration
// for shim implementations.
type ShimConfig struct {
	Path  string
	Debug bool
}

// Set sets a shim type based on the input string.
func (pType *ShimType) Set(value string) error {
	switch value {
	case "noopShim":
		*pType = NoopShimType
		return nil
	case "ccShim":
		*pType = CCShimType
		return nil
	case "kataShim":
		*pType = KataShimType
		return nil
	default:
		return fmt.Errorf("Unknown shim type %s", value)
	}
}

// String converts a shim type to a string.
func (pType *ShimType) String() string {
	switch *pType {
	case NoopShimType:
		return string(NoopShimType)
	case CCShimType:
		return string(CCShimType)
	case KataShimType:
		return string(KataShimType)
	default:
		return ""
	}
}

// newShim returns a shim from a shim type.
func newShim(pType ShimType) (shim, error) {
	switch pType {
	case NoopShimType:
		return &noopShim{}, nil
	case CCShimType:
		return &ccShim{}, nil
	case KataShimType:
		return &kataShim{}, nil
	default:
		return &noopShim{}, nil
	}
}

// newShimConfig returns a shim config from a generic PodConfig interface.
func newShimConfig(config PodConfig) interface{} {
	switch config.ShimType {
	case NoopShimType:
		return nil
	case CCShimType, KataShimType:
		var shimConfig ShimConfig
		err := mapstructure.Decode(config.ShimConfig, &shimConfig)
		if err != nil {
			return err
		}
		return shimConfig
	default:
		return nil
	}
}

func shimLogger() *logrus.Entry {
	return virtLog.WithField("subsystem", "shim")
}

func signalShim(pid int, sig syscall.Signal) error {
	if pid <= 0 {
		return nil
	}

	shimLogger().WithFields(
		logrus.Fields{
			"shim-pid":    pid,
			"shim-signal": sig,
		}).Info("Signalling shim")

	return syscall.Kill(pid, sig)
}

func stopShim(pid int) error {
	if err := signalShim(pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH {
		return err
	}

	return nil
}

func prepareAndStartShim(pod *Pod, shim shim, cid, token, url string, cmd Cmd) (*Process, error) {
	process := &Process{
		Token:     token,
		StartTime: time.Now().UTC(),
	}

	shimParams := ShimParams{
		Container: cid,
		Token:     token,
		URL:       url,
		Console:   cmd.Console,
		Terminal:  cmd.Interactive,
		Detach:    cmd.Detach,
	}

	var pid int
	if err := pod.network.run(pod.networkNS.NetNsPath, func() (shimErr error) {
		pid, shimErr = shim.start(*pod, shimParams)
		return
	}); err != nil {
		return nil, err
	}

	process.Pid = pid

	return process, nil
}

func startShim(args []string, params ShimParams) (int, error) {
	cmd := exec.Command(args[0], args[1:]...)

	if !params.Detach {
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}

	cloneFlags := 0
	for _, nsType := range params.CreateNS {
		cloneFlags |= ns.CloneFlagsTable[nsType]
	}

	cmd.SysProcAttr = &syscall.SysProcAttr{
		Cloneflags: uintptr(cloneFlags),
	}

	var f *os.File
	var err error
	if params.Console != "" {
		f, err = os.OpenFile(params.Console, os.O_RDWR, consoleFileMode)
		if err != nil {
			return -1, err
		}

		cmd.Stdin = f
		cmd.Stdout = f
		cmd.Stderr = f
		// Create Session
		cmd.SysProcAttr.Setsid = true
		// Set Controlling terminal to Ctty
		cmd.SysProcAttr.Setctty = true
		cmd.SysProcAttr.Ctty = int(f.Fd())
	}
	defer func() {
		if f != nil {
			f.Close()
		}
	}()

	if err := cmd.Start(); err != nil {
		return -1, err
	}

	return cmd.Process.Pid, nil
}

func isShimRunning(pid int) (bool, error) {
	process, err := os.FindProcess(pid)
	if err != nil {
		return false, err
	}

	if err := process.Signal(syscall.Signal(0)); err != nil {
		return false, nil
	}

	return true, nil
}

// waitForShim waits for the end of the shim unless it reaches the timeout
// first, returning an error in that case.
func waitForShim(pid int) error {
	if pid <= 0 {
		return nil
	}

	tInit := time.Now()
	for {
		running, err := isShimRunning(pid)
		if err != nil {
			return err
		}

		if !running {
			break
		}

		if time.Since(tInit).Seconds() >= waitForShimTimeout {
			return fmt.Errorf("Shim still running, timeout %f s has been reached", waitForShimTimeout)
		}

		// Let's avoid to run a too busy loop
		time.Sleep(time.Duration(100) * time.Millisecond)
	}

	return nil
}

// shim is the virtcontainers shim interface.
type shim interface {
	// start starts the shim relying on its configuration and on
	// parameters provided.
	start(pod Pod, params ShimParams) (int, error)
}
<commit_msg>shim: Add ability to enter any set of namespaces<commit_after>//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package virtcontainers

import (
	"fmt"
	"os"
	"os/exec"
	"syscall"
	"time"

	ns "github.com/containers/virtcontainers/pkg/nsenter"
	"github.com/mitchellh/mapstructure"
	"github.com/sirupsen/logrus"
)

// ShimType describes a shim type.
type ShimType string

const (
	// CCShimType is the ccShim.
	CCShimType ShimType = "ccShim"

	// NoopShimType is the noopShim.
	NoopShimType ShimType = "noopShim"

	// KataShimType is the Kata Containers shim type.
	KataShimType ShimType = "kataShim"
)

var waitForShimTimeout = 10.0
var consoleFileMode = os.FileMode(0660)

// ShimParams is the structure providing specific parameters needed
// for the execution of the shim binary.
type ShimParams struct {
	Container string
	Token     string
	URL       string
	Console   string
	Terminal  bool
	Detach    bool
	PID       int
	CreateNS  []ns.NSType
	EnterNS   []ns.Namespace
}

// ShimConfig is the structure providing specific configuration
// for shim implementations.
type ShimConfig struct {
	Path  string
	Debug bool
}

// Set sets a shim type based on the input string.
func (pType *ShimType) Set(value string) error {
	switch value {
	case "noopShim":
		*pType = NoopShimType
		return nil
	case "ccShim":
		*pType = CCShimType
		return nil
	case "kataShim":
		*pType = KataShimType
		return nil
	default:
		return fmt.Errorf("Unknown shim type %s", value)
	}
}

// String converts a shim type to a string.
func (pType *ShimType) String() string {
	switch *pType {
	case NoopShimType:
		return string(NoopShimType)
	case CCShimType:
		return string(CCShimType)
	case KataShimType:
		return string(KataShimType)
	default:
		return ""
	}
}

// newShim returns a shim from a shim type.
func newShim(pType ShimType) (shim, error) {
	switch pType {
	case NoopShimType:
		return &noopShim{}, nil
	case CCShimType:
		return &ccShim{}, nil
	case KataShimType:
		return &kataShim{}, nil
	default:
		return &noopShim{}, nil
	}
}

// newShimConfig returns a shim config from a generic PodConfig interface.
func newShimConfig(config PodConfig) interface{} {
	switch config.ShimType {
	case NoopShimType:
		return nil
	case CCShimType, KataShimType:
		var shimConfig ShimConfig
		err := mapstructure.Decode(config.ShimConfig, &shimConfig)
		if err != nil {
			return err
		}
		return shimConfig
	default:
		return nil
	}
}

func shimLogger() *logrus.Entry {
	return virtLog.WithField("subsystem", "shim")
}

func signalShim(pid int, sig syscall.Signal) error {
	if pid <= 0 {
		return nil
	}

	shimLogger().WithFields(
		logrus.Fields{
			"shim-pid":    pid,
			"shim-signal": sig,
		}).Info("Signalling shim")

	return syscall.Kill(pid, sig)
}

func stopShim(pid int) error {
	if err := signalShim(pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH {
		return err
	}

	return nil
}

func prepareAndStartShim(pod *Pod, shim shim, cid, token, url string, cmd Cmd) (*Process, error) {
	process := &Process{
		Token:     token,
		StartTime: time.Now().UTC(),
	}

	shimParams := ShimParams{
		Container: cid,
		Token:     token,
		URL:       url,
		Console:   cmd.Console,
		Terminal:  cmd.Interactive,
		Detach:    cmd.Detach,
		EnterNS:   []ns.Namespace{
			{
				Path: pod.networkNS.NetNsPath,
				Type: ns.NSTypeNet,
			},
		},
	}

	pid, err := shim.start(*pod, shimParams)
	if err != nil {
		return nil, err
	}

	process.Pid = pid

	return process, nil
}

func startShim(args []string, params ShimParams) (int, error) {
	cmd := exec.Command(args[0], args[1:]...)

	if !params.Detach {
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}

	cloneFlags := 0
	for _, nsType := range params.CreateNS {
		cloneFlags |= ns.CloneFlagsTable[nsType]
	}

	cmd.SysProcAttr = &syscall.SysProcAttr{
		Cloneflags: uintptr(cloneFlags),
	}

	var f *os.File
	var err error
	if params.Console != "" {
		f, err = os.OpenFile(params.Console, os.O_RDWR, consoleFileMode)
		if err != nil {
			return -1, err
		}

		cmd.Stdin = f
		cmd.Stdout = f
		cmd.Stderr = f
		// Create Session
		cmd.SysProcAttr.Setsid = true
		// Set Controlling terminal to Ctty
		cmd.SysProcAttr.Setctty = true
		cmd.SysProcAttr.Ctty = int(f.Fd())
	}
	defer func() {
		if f != nil {
			f.Close()
		}
	}()

	if err := ns.NsEnter(params.EnterNS, func() error {
		return cmd.Start()
	}); err != nil {
		return -1, err
	}

	return cmd.Process.Pid, nil
}

func isShimRunning(pid int) (bool, error) {
	process, err := os.FindProcess(pid)
	if err != nil {
		return false, err
	}

	if err := process.Signal(syscall.Signal(0)); err != nil {
		return false, nil
	}

	return true, nil
}

// waitForShim waits for the end of the shim unless it reaches the timeout
// first, returning an error in that case.
func waitForShim(pid int) error {
	if pid <= 0 {
		return nil
	}

	tInit := time.Now()
	for {
		running, err := isShimRunning(pid)
		if err != nil {
			return err
		}

		if !running {
			break
		}

		if time.Since(tInit).Seconds() >= waitForShimTimeout {
			return fmt.Errorf("Shim still running, timeout %f s has been reached", waitForShimTimeout)
		}

		// Let's avoid to run a too busy loop
		time.Sleep(time.Duration(100) * time.Millisecond)
	}

	return nil
}

// shim is the virtcontainers shim interface.
type shim interface {
	// start starts the shim relying on its configuration and on
	// parameters provided.
	start(pod Pod, params ShimParams) (int, error)
}
<|endoftext|>