<commit_before>package main

import (
	"errors"
	"net/http"
	"os"
	"testing"

	"github.com/skeswa/gophr/common"
	"github.com/skeswa/gophr/common/io"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
)

func TestDownloadPackage(t *testing.T) {
	mockIO := io.NewMockIO()
	mockIO.
		On("Mkdir", mock.AnythingOfType("string"), os.FileMode(0644)).
		Return(errors.New("this is an error"))
	args := packageDownloaderArgs{
		io:                   mockIO,
		author:               "myauthor",
		repo:                 "myrepo",
		sha:                  "mysha",
		constructionZonePath: "/my/cons/zone",
	}
	_, err := downloadPackage(args)
	assert.NotNil(t, err)
	mockIO.AssertExpectations(t)

	mockIO = io.NewMockIO()
	mockIO.
		On("Mkdir", mock.AnythingOfType("string"), os.FileMode(0644)).
		Return(nil)
	zipResp := &http.Response{
		StatusCode: 500,
		Body:       common.NewMockHTTPResponseBody(nil),
	}
	deleteWorkDirCalled := false
	args = packageDownloaderArgs{
		io:                   mockIO,
		author:               "myauthor",
		repo:                 "myrepo",
		sha:                  "mysha",
		constructionZonePath: "/my/cons/zone",

		doHTTPGet: func(url string) (*http.Response, error) {
			assert.Equal(t, "https://github.com/myauthor/myrepo/archive/mysha.zip", url)
			return zipResp, errors.New("this is an error")
		},
		deleteWorkDir: func(folderPath string) {
			deleteWorkDirCalled = true
		},
	}
	_, err = downloadPackage(args)
	assert.NotNil(t, err)
	mockIO.AssertExpectations(t)
	assert.True(t, zipResp.Body.(*common.MockHTTPResponseBody).WasClosed())
	assert.True(t, deleteWorkDirCalled)

	mockIO = io.NewMockIO()
	mockIO.
		On("Mkdir", mock.AnythingOfType("string"), os.FileMode(0644)).
		Return(nil)
	zipResp = &http.Response{
		StatusCode: 404,
		Body:       common.NewMockHTTPResponseBody(nil),
	}
	deleteWorkDirCalled = false
	args = packageDownloaderArgs{
		io:                   mockIO,
		author:               "myauthor",
		repo:                 "myrepo",
		sha:                  "mysha",
		constructionZonePath: "/my/cons/zone",

		doHTTPGet: func(url string) (*http.Response, error) {
			assert.Equal(t, "https://github.com/myauthor/myrepo/archive/mysha.zip", url)
			return zipResp, nil
		},
		deleteWorkDir: func(folderPath string) {
			deleteWorkDirCalled = true
		},
	}
	_, err = downloadPackage(args)
	assert.NotNil(t, err)
	mockIO.AssertExpectations(t)
	assert.True(t, zipResp.Body.(*common.MockHTTPResponseBody).WasClosed())
	assert.True(t, deleteWorkDirCalled)

	mockIO = io.NewMockIO()
	mockIO.
		On("Mkdir", mock.AnythingOfType("string"), os.FileMode(0644)).
		Return(nil)
	mockIO.
		On("Create", mock.AnythingOfType("string")).
		Return((*os.File)(nil), errors.New("oh no"))
	zipResp = &http.Response{
		StatusCode: 200,
		Body:       common.NewMockHTTPResponseBody([]byte("this is a zip")),
	}
	deleteWorkDirCalled = false
	args = packageDownloaderArgs{
		io:                   mockIO,
		author:               "myauthor",
		repo:                 "myrepo",
		sha:                  "mysha",
		constructionZonePath: "/my/cons/zone",

		doHTTPGet: func(url string) (*http.Response, error) {
			assert.Equal(t, "https://github.com/myauthor/myrepo/archive/mysha.zip", url)
			return zipResp, nil
		},
		deleteWorkDir: func(folderPath string) {
			deleteWorkDirCalled = true
		},
	}
	_, err = downloadPackage(args)
	assert.NotNil(t, err)
	mockIO.AssertExpectations(t)
	assert.True(t, zipResp.Body.(*common.MockHTTPResponseBody).WasClosed())
	assert.True(t, deleteWorkDirCalled)
}
<commit_msg>tests pass so far - about half way through<commit_after>package main

import (
	"errors"
	"net/http"
	"os"
	"testing"

	"github.com/skeswa/gophr/common"
	"github.com/skeswa/gophr/common/io"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
)

func TestDownloadPackage(t *testing.T) {
	mockIO := io.NewMockIO()
	mockIO.
		On("Mkdir", mock.AnythingOfType("string"), os.FileMode(0644)).
		Return(errors.New("this is an error"))
	args := packageDownloaderArgs{
		io:                   mockIO,
		author:               "myauthor",
		repo:                 "myrepo",
		sha:                  "mysha",
		constructionZonePath: "/my/cons/zone",
	}
	_, err := downloadPackage(args)
	assert.NotNil(t, err)
	mockIO.AssertExpectations(t)

	mockIO = io.NewMockIO()
	mockIO.
		On("Mkdir", mock.AnythingOfType("string"), os.FileMode(0644)).
		Return(nil)
	zipResp := &http.Response{
		StatusCode: 500,
		Body:       common.NewMockHTTPResponseBody(nil),
	}
	deleteWorkDirCalled := false
	args = packageDownloaderArgs{
		io:                   mockIO,
		author:               "myauthor",
		repo:                 "myrepo",
		sha:                  "mysha",
		constructionZonePath: "/my/cons/zone",

		doHTTPGet: func(url string) (*http.Response, error) {
			assert.Equal(t, "https://github.com/myauthor/myrepo/archive/mysha.zip", url)
			return zipResp, errors.New("this is an error")
		},
		deleteWorkDir: func(folderPath string) {
			deleteWorkDirCalled = true
		},
	}
	_, err = downloadPackage(args)
	assert.NotNil(t, err)
	mockIO.AssertExpectations(t)
	assert.True(t, zipResp.Body.(*common.MockHTTPResponseBody).WasClosed())
	assert.True(t, deleteWorkDirCalled)

	mockIO = io.NewMockIO()
	mockIO.
		On("Mkdir", mock.AnythingOfType("string"), os.FileMode(0644)).
		Return(nil)
	zipResp = &http.Response{
		StatusCode: 404,
		Body:       common.NewMockHTTPResponseBody(nil),
	}
	deleteWorkDirCalled = false
	args = packageDownloaderArgs{
		io:                   mockIO,
		author:               "myauthor",
		repo:                 "myrepo",
		sha:                  "mysha",
		constructionZonePath: "/my/cons/zone",

		doHTTPGet: func(url string) (*http.Response, error) {
			assert.Equal(t, "https://github.com/myauthor/myrepo/archive/mysha.zip", url)
			return zipResp, nil
		},
		deleteWorkDir: func(folderPath string) {
			deleteWorkDirCalled = true
		},
	}
	_, err = downloadPackage(args)
	assert.NotNil(t, err)
	mockIO.AssertExpectations(t)
	assert.True(t, zipResp.Body.(*common.MockHTTPResponseBody).WasClosed())
	assert.True(t, deleteWorkDirCalled)

	mockIO = io.NewMockIO()
	mockIO.
		On("Mkdir", mock.AnythingOfType("string"), os.FileMode(0644)).
		Return(nil)
	mockIO.
		On("Create", mock.AnythingOfType("string")).
		Return((*os.File)(nil), errors.New("oh no"))
	zipResp = &http.Response{
		StatusCode: 200,
		Body:       common.NewMockHTTPResponseBody([]byte("this is a zip")),
	}
	deleteWorkDirCalled = false
	args = packageDownloaderArgs{
		io:                   mockIO,
		author:               "myauthor",
		repo:                 "myrepo",
		sha:                  "mysha",
		constructionZonePath: "/my/cons/zone",

		doHTTPGet: func(url string) (*http.Response, error) {
			assert.Equal(t, "https://github.com/myauthor/myrepo/archive/mysha.zip", url)
			return zipResp, nil
		},
		deleteWorkDir: func(folderPath string) {
			deleteWorkDirCalled = true
		},
	}
	_, err = downloadPackage(args)
	assert.NotNil(t, err)
	mockIO.AssertExpectations(t)
	assert.True(t, zipResp.Body.(*common.MockHTTPResponseBody).WasClosed())
	assert.True(t, deleteWorkDirCalled)

	// TODO(skeswa): come up with a mock file to make sure it is closed.
	mockFile := &os.File{}
	zipResp = &http.Response{
		StatusCode: 200,
		Body:       common.NewMockHTTPResponseBody([]byte("this is a zip")),
	}
	mockIO = io.NewMockIO()
	mockIO.
		On("Mkdir", mock.AnythingOfType("string"), os.FileMode(0644)).
		Return(nil)
	mockIO.
		On("Create", mock.AnythingOfType("string")).
		Return(mockFile, error(nil))
	mockIO.
		On("Copy", mockFile, zipResp.Body).
		Return(int64(0), errors.New("the copy didnt work"))
	deleteWorkDirCalled = false
	args = packageDownloaderArgs{
		io:                   mockIO,
		author:               "myauthor",
		repo:                 "myrepo",
		sha:                  "mysha",
		constructionZonePath: "/my/cons/zone",

		doHTTPGet: func(url string) (*http.Response, error) {
			assert.Equal(t, "https://github.com/myauthor/myrepo/archive/mysha.zip", url)
			return zipResp, nil
		},
		deleteWorkDir: func(folderPath string) {
			deleteWorkDirCalled = true
		},
	}
	_, err = downloadPackage(args)
	assert.NotNil(t, err)
	mockIO.AssertExpectations(t)
	assert.True(t, zipResp.Body.(*common.MockHTTPResponseBody).WasClosed())
	assert.True(t, deleteWorkDirCalled)
}
<|endoftext|>


<commit_before><commit_msg>routing: fix memory corruption in MC store<commit_after><|endoftext|>


<commit_before>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file contains the frame primitive functions.

package golisp

import (
	"errors"
	"fmt"
)

func RegisterFramePrimitives() {
	MakePrimitiveFunction("make-frame", -1, MakeFrameImpl)
	MakePrimitiveFunction("has-slot?", 2, HasSlotImpl)
	MakePrimitiveFunction("get-slot", 2, GetSlotImpl)
	MakePrimitiveFunction("get-slot-or-nil", 2, GetSlotOrNilImpl)
	MakePrimitiveFunction("remove-slot!", 2, RemoveSlotImpl)
	MakePrimitiveFunction("set-slot!", 3, SetSlotImpl)
	MakePrimitiveFunction("send", -1, SendImpl)
	MakePrimitiveFunction("send-super", -1, SendSuperImpl)
	MakePrimitiveFunction("clone", 1, CloneImpl)
}

func MakeFrameImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	if Length(args)%2 != 0 {
		err = errors.New("Frames must be initialized with an even number of arguments.")
	}
	m := make(FrameMap)
	for c := args; NotNilP(c); c = Cddr(c) {
		k := Car(c)
		if !NakedP(k) {
			err = errors.New(fmt.Sprintf("Frame keys must be naked symbols, but was given %s.", String(k)))
			return
		}
		v, e := Eval(Cadr(c), env)
		if e != nil {
			return nil, e
		}
		m[StringValue(k)] = v
	}
	return FrameWithValue(&m), nil
}

func HasSlotImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f, err := Eval(Car(args), env)
	if err != nil {
		return
	}
	if !FrameP(f) {
		err = errors.New(fmt.Sprintf("has-slot requires a frame as it's first argument, but was given %s.", String(f)))
	}

	k, err := Eval(Cadr(args), env)
	if err != nil {
		return
	}
	if !SymbolP(k) {
		err = errors.New(fmt.Sprintf("has-slot requires a symbol as it's second argument, but was given %s.", String(k)))
		return
	}

	return BooleanWithValue(f.Frame.HasSlot(StringValue(k))), nil
}

func GetSlotImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f, err := Eval(Car(args), env)
	if err != nil {
		return
	}
	if !FrameP(f) {
		err = errors.New(fmt.Sprintf("get-slot requires a frame as it's first argument, but was given %s.", String(f)))
	}

	k, err := Eval(Cadr(args), env)
	if err != nil {
		return
	}
	if !SymbolP(k) {
		err = errors.New(fmt.Sprintf("get-slot requires a symbol as it's second argument, but was given %s.", String(k)))
		return
	}

	if !f.Frame.HasSlot(StringValue(k)) {
		err = errors.New(fmt.Sprintf("get-slot requires an existing slot, but was given %s.", String(k)))
		return
	}

	return f.Frame.Get(StringValue(k)), nil
}

func GetSlotOrNilImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f, err := Eval(Car(args), env)
	if err != nil {
		return
	}
	if !FrameP(f) {
		err = errors.New(fmt.Sprintf("get-slot-or-nil requires a frame as it's first argument, but was given %s.", String(f)))
	}

	k, err := Eval(Cadr(args), env)
	if err != nil {
		return
	}
	if !SymbolP(k) {
		err = errors.New(fmt.Sprintf("get-slot-or-nil requires a symbol as it's second argument, but was given %s.", String(k)))
		return
	}

	return f.Frame.Get(StringValue(k)), nil
}

func RemoveSlotImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f, err := Eval(Car(args), env)
	if err != nil {
		return
	}
	if !FrameP(f) {
		err = errors.New(fmt.Sprintf("remove-slot! requires a frame as it's first argument, but was given %s.", String(f)))
	}

	k, err := Eval(Cadr(args), env)
	if err != nil {
		return
	}
	if !SymbolP(k) {
		err = errors.New(fmt.Sprintf("remove-slot! requires a symbol as it's second argument, but was given %s.", String(k)))
		return
	}

	return BooleanWithValue(f.Frame.Remove(StringValue(k))), nil
}

func SetSlotImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f, err := Eval(Car(args), env)
	if err != nil {
		return
	}
	if !FrameP(f) {
		err = errors.New(fmt.Sprintf("set-slot requires a frame as it's first argument, but was given %s.", String(f)))
	}

	k, err := Eval(Cadr(args), env)
	if err != nil {
		return
	}
	if !NakedP(k) {
		err = errors.New(fmt.Sprintf("set-slot requires a naked symbol as it's second argument, but was given %s.", String(k)))
		return
	}

	v, err := Eval(Caddr(args), env)
	if err != nil {
		return
	}

	return f.Frame.Set(StringValue(k), v), nil
}

func SendImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f, err := Eval(Car(args), env)
	if err != nil {
		return
	}
	if !FrameP(f) {
		err = errors.New(fmt.Sprintf("send requires a frame as it's first argument, but was given %s.", String(f)))
	}

	k, err := Eval(Cadr(args), env)
	if err != nil {
		return
	}
	if !SymbolP(k) {
		err = errors.New(fmt.Sprintf("send requires a symbol as it's second argument, but was given %s.", String(k)))
		return
	}

	if !f.Frame.HasSlot(StringValue(k)) {
		err = errors.New(fmt.Sprintf("send requires an existing slot, but was given %s.", String(k)))
		return
	}

	fun := f.Frame.Get(StringValue(k))
	if !FunctionP(fun) {
		err = errors.New(fmt.Sprintf("send requires a function slot, but was given a slot containing a %s.", TypeName(TypeOf(fun))))
	}

	params := Cddr(args)
	frameEnv := NewSymbolTableFrameBelowWithFrame(env, f.Frame)
	frameEnv.BindLocallyTo(SymbolWithName("self"), f)
	return fun.Func.Apply(params, frameEnv)
}

func getSuperFunction(selector string, env *SymbolTableFrame) *Data {
	f := env.Frame
	if f == nil {
		return nil
	}

	for _, p := range f.Parents() {
		fun := p.Get(selector)
		if fun != nil {
			return fun
		}
	}

	return nil
}

func SendSuperImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	if !env.HasFrame() {
		err = errors.New("send-super can only be used within the context of a frame.")
		return
	}

	selector, err := Eval(Car(args), env)
	if err != nil {
		return
	}
	if !SymbolP(selector) {
		err = errors.New(fmt.Sprintf("Selector must be a symbol but was %s.", TypeName(TypeOf(selector))))
		return
	}

	fun := getSuperFunction(StringValue(selector), env)
	if fun == nil || !FunctionP(fun) {
		err = errors.New(fmt.Sprintf("Message sent must select a function slot but was %s.", TypeName(TypeOf(fun))))
		return
	}

	params := Cdr(args)
	frameEnv := NewSymbolTableFrameBelowWithFrame(env, env.Frame)
	frameEnv.BindLocallyTo(SymbolWithName("self"), FrameWithValue(env.Frame))
	return fun.Func.Apply(params, frameEnv)
}

func CloneImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f, err := Eval(Car(args), env)
	if err != nil {
		return
	}
	if !FrameP(f) {
		err = errors.New(fmt.Sprintf("clone requires a frame as it's argument, but was given %s.", String(f)))
	}

	return FrameWithValue(f.Frame.Clone()), nil
}
<commit_msg>Added conversion functions<commit_after>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file contains the frame primitive functions.

package golisp

import (
	"errors"
	"fmt"
)

func RegisterFramePrimitives() {
	MakePrimitiveFunction("make-frame", -1, MakeFrameImpl)
	MakePrimitiveFunction("has-slot?", 2, HasSlotImpl)
	MakePrimitiveFunction("get-slot", 2, GetSlotImpl)
	MakePrimitiveFunction("get-slot-or-nil", 2, GetSlotOrNilImpl)
	MakePrimitiveFunction("remove-slot!", 2, RemoveSlotImpl)
	MakePrimitiveFunction("set-slot!", 3, SetSlotImpl)
	MakePrimitiveFunction("send", -1, SendImpl)
	MakePrimitiveFunction("send-super", -1, SendSuperImpl)
	MakePrimitiveFunction("clone", 1, CloneImpl)
	MakePrimitiveFunction("json->lisp", 1, JsonToLispImpl)
	MakePrimitiveFunction("lisp->json", 1, LispToJsonImpl)
}

func MakeFrameImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	if Length(args)%2 != 0 {
		err = errors.New("Frames must be initialized with an even number of arguments.")
	}
	m := make(FrameMap)
	for c := args; NotNilP(c); c = Cddr(c) {
		k := Car(c)
		if !NakedP(k) {
			err = errors.New(fmt.Sprintf("Frame keys must be naked symbols, but was given %s.", String(k)))
			return
		}
		v, e := Eval(Cadr(c), env)
		if e != nil {
			return nil, e
		}
		m[StringValue(k)] = v
	}
	return FrameWithValue(&m), nil
}

func HasSlotImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f, err := Eval(Car(args), env)
	if err != nil {
		return
	}
	if !FrameP(f) {
		err = errors.New(fmt.Sprintf("has-slot requires a frame as it's first argument, but was given %s.", String(f)))
	}

	k, err := Eval(Cadr(args), env)
	if err != nil {
		return
	}
	if !SymbolP(k) {
		err = errors.New(fmt.Sprintf("has-slot requires a symbol as it's second argument, but was given %s.", String(k)))
		return
	}

	return BooleanWithValue(f.Frame.HasSlot(StringValue(k))), nil
}

func GetSlotImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f, err := Eval(Car(args), env)
	if err != nil {
		return
	}
	if !FrameP(f) {
		err = errors.New(fmt.Sprintf("get-slot requires a frame as it's first argument, but was given %s.", String(f)))
	}

	k, err := Eval(Cadr(args), env)
	if err != nil {
		return
	}
	if !SymbolP(k) {
		err = errors.New(fmt.Sprintf("get-slot requires a symbol as it's second argument, but was given %s.", String(k)))
		return
	}

	if !f.Frame.HasSlot(StringValue(k)) {
		err = errors.New(fmt.Sprintf("get-slot requires an existing slot, but was given %s.", String(k)))
		return
	}

	return f.Frame.Get(StringValue(k)), nil
}

func GetSlotOrNilImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f, err := Eval(Car(args), env)
	if err != nil {
		return
	}
	if !FrameP(f) {
		err = errors.New(fmt.Sprintf("get-slot-or-nil requires a frame as it's first argument, but was given %s.", String(f)))
	}

	k, err := Eval(Cadr(args), env)
	if err != nil {
		return
	}
	if !SymbolP(k) {
		err = errors.New(fmt.Sprintf("get-slot-or-nil requires a symbol as it's second argument, but was given %s.", String(k)))
		return
	}

	return f.Frame.Get(StringValue(k)), nil
}

func RemoveSlotImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f, err := Eval(Car(args), env)
	if err != nil {
		return
	}
	if !FrameP(f) {
		err = errors.New(fmt.Sprintf("remove-slot! requires a frame as it's first argument, but was given %s.", String(f)))
	}

	k, err := Eval(Cadr(args), env)
	if err != nil {
		return
	}
	if !SymbolP(k) {
		err = errors.New(fmt.Sprintf("remove-slot! requires a symbol as it's second argument, but was given %s.", String(k)))
		return
	}

	return BooleanWithValue(f.Frame.Remove(StringValue(k))), nil
}

func SetSlotImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f, err := Eval(Car(args), env)
	if err != nil {
		return
	}
	if !FrameP(f) {
		err = errors.New(fmt.Sprintf("set-slot requires a frame as it's first argument, but was given %s.", String(f)))
	}

	k, err := Eval(Cadr(args), env)
	if err != nil {
		return
	}
	if !NakedP(k) {
		err = errors.New(fmt.Sprintf("set-slot requires a naked symbol as it's second argument, but was given %s.", String(k)))
		return
	}

	v, err := Eval(Caddr(args), env)
	if err != nil {
		return
	}

	return f.Frame.Set(StringValue(k), v), nil
}

func SendImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f, err := Eval(Car(args), env)
	if err != nil {
		return
	}
	if !FrameP(f) {
		err = errors.New(fmt.Sprintf("send requires a frame as it's first argument, but was given %s.", String(f)))
	}

	k, err := Eval(Cadr(args), env)
	if err != nil {
		return
	}
	if !SymbolP(k) {
		err = errors.New(fmt.Sprintf("send requires a symbol as it's second argument, but was given %s.", String(k)))
		return
	}

	if !f.Frame.HasSlot(StringValue(k)) {
		err = errors.New(fmt.Sprintf("send requires an existing slot, but was given %s.", String(k)))
		return
	}

	fun := f.Frame.Get(StringValue(k))
	if !FunctionP(fun) {
		err = errors.New(fmt.Sprintf("send requires a function slot, but was given a slot containing a %s.", TypeName(TypeOf(fun))))
	}

	params := Cddr(args)
	frameEnv := NewSymbolTableFrameBelowWithFrame(env, f.Frame)
	frameEnv.BindLocallyTo(SymbolWithName("self"), f)
	return fun.Func.Apply(params, frameEnv)
}

func getSuperFunction(selector string, env *SymbolTableFrame) *Data {
	f := env.Frame
	if f == nil {
		return nil
	}

	for _, p := range f.Parents() {
		fun := p.Get(selector)
		if fun != nil {
			return fun
		}
	}

	return nil
}

func SendSuperImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	if !env.HasFrame() {
		err = errors.New("send-super can only be used within the context of a frame.")
		return
	}

	selector, err := Eval(Car(args), env)
	if err != nil {
		return
	}
	if !SymbolP(selector) {
		err = errors.New(fmt.Sprintf("Selector must be a symbol but was %s.", TypeName(TypeOf(selector))))
		return
	}

	fun := getSuperFunction(StringValue(selector), env)
	if fun == nil || !FunctionP(fun) {
		err = errors.New(fmt.Sprintf("Message sent must select a function slot but was %s.", TypeName(TypeOf(fun))))
		return
	}

	params := Cdr(args)
	frameEnv := NewSymbolTableFrameBelowWithFrame(env, env.Frame)
	frameEnv.BindLocallyTo(SymbolWithName("self"), FrameWithValue(env.Frame))
	return fun.Func.Apply(params, frameEnv)
}

func CloneImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f, err := Eval(Car(args), env)
	if err != nil {
		return
	}
	if !FrameP(f) {
		err = errors.New(fmt.Sprintf("clone requires a frame as it's argument, but was given %s.", String(f)))
		return
	}

	return FrameWithValue(f.Frame.Clone()), nil
}

func JsonToLispImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	j, err := Eval(Car(args), env)
	if err != nil {
		return
	}
	if !StringP(j) {
		err = errors.New(fmt.Sprintf("json->lisp requires a string as it's argument, but was given %s.", String(j)))
		return
	}

	return JsonStringToLispWithFrames(StringValue(j)), nil
}

func LispToJsonImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	l, err := Eval(Car(args), env)
	if err != nil {
		return
	}

	return StringWithValue(LispWithFramesToJsonString(l)), nil
}
<|endoftext|>


<commit_before>package prime

import (
	"testing"
)

func TestPrimes(t *testing.T) {
	p := Primes(1000000)
	if len(p) != 78498 {
		t.Error("Wrong number of prime numbers lower than 1M ", len(p))
	}
	if p[0] != uint64(2) {
		t.Error("1st prime number is not 2 ", p[0])
	}
	if p[1] != uint64(3) {
		t.Error("2nd prime number is not 2 ", p[1])
	}
	if p[25] != uint64(101) {
		t.Error("25th prime number is not 2 ", p[25])
	}
	if p[23423] != uint64(267391) {
		t.Error("23424th prime number is not 267391 ", p[23423])
	}
	p = Primes(100)
	if p[24] != uint64(97) {
		t.Error("23424th prime number is not 267391 ", p[24])
	}
	if len(p) != 25 {
		t.Error("Wrong number of prime numbers lower than 10 ", len(p))
	}
	p = Primes(1)
	if len(p) != 0 {
		t.Error("Edge case of 1 not correct ", len(p))
	}
	p = Primes(2)
	if len(p) != 1 {
		t.Error("Edge case of 2 not correct ", len(p))
	}
}

func TestFill(t *testing.T) {
	p := make([]uint64, 101)
	fill(p, 2, 100)
	var c uint64
	for _, v := range p {
		if v == 1 {
			c++
		}
	}
	if c != 49 {
		t.Error("Filled cells are wrong ", c)
	}
}
<commit_msg>correcting unit test texts<commit_after>package prime

import (
	"testing"
)

func TestPrimes(t *testing.T) {
	p := Primes(1000000)
	if len(p) != 78498 {
		t.Error("Wrong number of prime numbers lower than 1M,", len(p))
	}
	if p[0] != uint64(2) {
		t.Error("1st prime number is not 2,", p[0])
	}
	if p[1] != uint64(3) {
		t.Error("2nd prime number is not 3,", p[1])
	}
	if p[25] != uint64(101) {
		t.Error("25th prime number is not 101,", p[25])
	}
	if p[23423] != uint64(267391) {
		t.Error("23424th prime number is not 267391,", p[23423])
	}
	p = Primes(100)
	if p[24] != uint64(97) {
		t.Error("23424th prime number is not 97,", p[24])
	}
	if len(p) != 25 {
		t.Error("Wrong number of prime numbers lower than 100,", len(p))
	}
	p = Primes(1)
	if len(p) != 0 {
		t.Error("Edge case of 1 not correct,", len(p))
	}
	p = Primes(2)
	if len(p) != 1 {
		t.Error("Edge case of 2 not correct,", len(p))
	}
}

func TestFill(t *testing.T) {
	p := make([]uint64, 101)
	fill(p, 2, 100)
	var c uint64
	for _, v := range p {
		if v == 1 {
			c++
		}
	}
	if c != 49 {
		t.Error("Filled cells are wrong ", c)
	}
}
<|endoftext|>


<commit_before>// +build darwin linux

package misc

import (
	"os"
	"syscall"
)

// GetRootDiskUsageKB returns used and total space of the file system mounted on /. Returns 0 if they cannot be determined.
func GetRootDiskUsageKB() (usedKB, freeKB, totalKB int) {
	fs := syscall.Statfs_t{}
	err := syscall.Statfs("/", &fs)
	if err != nil {
		return
	}
	totalKB = int(int64(fs.Blocks) * int64(fs.Bsize) / 1024)
	freeKB = int(int64(fs.Bfree) * int64(fs.Bsize) / 1024)
	usedKB = totalKB - freeKB
	return
}

/*
InvokeProgram launches an external program with time constraints. The external program inherits laitos' environment
mixed with additional input environment variables. The additional variables take precedence over inherited ones.
Returns stdout+stderr output combined, and error if there is any.
*/
func InvokeProgram(envVars []string, timeoutSec int, program string, args ...string) (out string, err error) {
	if timeoutSec < 1 {
		return "", errors.New("invalid time limit")
	}
	// Make an environment variable array of common PATH, inherited values, and newly specified values.
	defaultOSEnv := os.Environ()
	combinedEnv := make([]string, 0, 1+len(defaultOSEnv))
	// Inherit environment variables from program environment
	combinedEnv = append(combinedEnv, defaultOSEnv...)
	/*
		Put common PATH values into the mix. Since go 1.9, when environment variables contain duplicated keys, only
		the last value of duplicated key is effective. This behaviour enables caller to override PATH if deemed
		necessary.
	*/
	combinedEnv = append(combinedEnv, "PATH="+CommonPATH)
	if envVars != nil {
		combinedEnv = append(combinedEnv, envVars...)
	}
	// Collect stdout and stderr all together in a single buffer
	var outBuf bytes.Buffer
	proc := exec.Command(program, args...)
	proc.Env = combinedEnv
	proc.Stdout = &outBuf
	proc.Stderr = &outBuf
	// Use process group so that child processes are also killed upon time out, Windows does not require this.
	proc.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
	// Monitor for time out
	var timedOut bool
	timeOutTimer := time.AfterFunc(time.Duration(timeoutSec)*time.Second, func() {
		timedOut = true
		if !KillProcess(proc.Process) {
			logger.Warning("InvokeProgram", program, nil, "failed to kill after time limit exceeded")
		}
	})
	err = proc.Run()
	timeOutTimer.Stop()
	if timedOut {
		err = errors.New("time limit exceeded")
	}
	out = outBuf.String()
	return
}

// KillProcess kills the process or the group of processes associated with it.
func KillProcess(proc *os.Process) (success bool) {
	// Kill process group if it is one
	if killErr := syscall.Kill(-proc.Pid, syscall.SIGKILL); killErr == nil {
		success = true
	}
	if killErr := syscall.Kill(proc.Pid, syscall.SIGKILL); killErr == nil {
		success = true
	}
	if proc.Kill() == nil {
		success = true
	}
	proc.Wait()
	return
}

// LockMemory locks program memory to prevent swapping, protecting sensitive user data.
func LockMemory() {
	// Lock all program memory into main memory to prevent sensitive data from leaking into swap.
	if os.Geteuid() == 0 {
		if err := syscall.Mlockall(syscall.MCL_CURRENT | syscall.MCL_FUTURE); err != nil {
			logger.Warning("LockMemory", "", err, "failed to lock memory")
			return
		}
		logger.Warning("LockMemory", "", nil, "program has been locked into memory for safety reasons")
	} else {
		logger.Warning("LockMemory", "", nil, "program is not running as root (UID 0) hence memory cannot be locked, your private information may leak onto disk.")
	}
}
<commit_msg>fix test case compilation error due to sys_unix.go<commit_after>// +build darwin linux

package misc

import (
	"bytes"
	"errors"
	"os"
	"os/exec"
	"syscall"
	"time"
)

// GetRootDiskUsageKB returns used and total space of the file system mounted on /. Returns 0 if they cannot be determined.
func GetRootDiskUsageKB() (usedKB, freeKB, totalKB int) {
	fs := syscall.Statfs_t{}
	err := syscall.Statfs("/", &fs)
	if err != nil {
		return
	}
	totalKB = int(int64(fs.Blocks) * int64(fs.Bsize) / 1024)
	freeKB = int(int64(fs.Bfree) * int64(fs.Bsize) / 1024)
	usedKB = totalKB - freeKB
	return
}

/*
InvokeProgram launches an external program with time constraints. The external program inherits laitos' environment
mixed with additional input environment variables. The additional variables take precedence over inherited ones.
Returns stdout+stderr output combined, and error if there is any.
*/
func InvokeProgram(envVars []string, timeoutSec int, program string, args ...string) (out string, err error) {
	if timeoutSec < 1 {
		return "", errors.New("invalid time limit")
	}
	// Make an environment variable array of common PATH, inherited values, and newly specified values.
	defaultOSEnv := os.Environ()
	combinedEnv := make([]string, 0, 1+len(defaultOSEnv))
	// Inherit environment variables from program environment
	combinedEnv = append(combinedEnv, defaultOSEnv...)
	/*
		Put common PATH values into the mix. Since go 1.9, when environment variables contain duplicated keys, only
		the last value of duplicated key is effective. This behaviour enables caller to override PATH if deemed
		necessary.
	*/
	combinedEnv = append(combinedEnv, "PATH="+CommonPATH)
	if envVars != nil {
		combinedEnv = append(combinedEnv, envVars...)
	}
	// Collect stdout and stderr all together in a single buffer
	var outBuf bytes.Buffer
	proc := exec.Command(program, args...)
	proc.Env = combinedEnv
	proc.Stdout = &outBuf
	proc.Stderr = &outBuf
	// Use process group so that child processes are also killed upon time out, Windows does not require this.
	proc.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
	// Monitor for time out
	var timedOut bool
	timeOutTimer := time.AfterFunc(time.Duration(timeoutSec)*time.Second, func() {
		timedOut = true
		if !KillProcess(proc.Process) {
			logger.Warning("InvokeProgram", program, nil, "failed to kill after time limit exceeded")
		}
	})
	err = proc.Run()
	timeOutTimer.Stop()
	if timedOut {
		err = errors.New("time limit exceeded")
	}
	out = outBuf.String()
	return
}

// KillProcess kills the process or the group of processes associated with it.
func KillProcess(proc *os.Process) (success bool) {
	// Kill process group if it is one
	if killErr := syscall.Kill(-proc.Pid, syscall.SIGKILL); killErr == nil {
		success = true
	}
	if killErr := syscall.Kill(proc.Pid, syscall.SIGKILL); killErr == nil {
		success = true
	}
	if proc.Kill() == nil {
		success = true
	}
	proc.Wait()
	return
}

// LockMemory locks program memory to prevent swapping, protecting sensitive user data.
func LockMemory() {
	// Lock all program memory into main memory to prevent sensitive data from leaking into swap.
	if os.Geteuid() == 0 {
		if err := syscall.Mlockall(syscall.MCL_CURRENT | syscall.MCL_FUTURE); err != nil {
			logger.Warning("LockMemory", "", err, "failed to lock memory")
			return
		}
		logger.Warning("LockMemory", "", nil, "program has been locked into memory for safety reasons")
	} else {
		logger.Warning("LockMemory", "", nil, "program is not running as root (UID 0) hence memory cannot be locked, your private information may leak onto disk.")
	}
}
<|endoftext|>


<commit_before>package mmhook

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mmbot/adapter"
	"mmbot/message"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/gorilla/schema"
)

// Client is a client for Mattermost.
type Client struct {
	config *adapter.Config
	logger *log.Logger
	http   *http.Client
	tokens map[string]int
	in     chan message.InMessage
	quit   chan bool
}

// NewClient returns new mattermost webhook client.
func NewClient(config *adapter.Config, logger *log.Logger) *Client {
	if logger == nil {
		logger = log.New(ioutil.Discard, "", 0)
	}
	c := &Client{
		config: config,
		logger: logger,
		in:     make(chan message.InMessage),
		quit:   make(chan bool),
	}

	tr := &http.Transport{
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
	}
	if config.InsecureSkipVerify {
		tr.TLSClientConfig = &tls.Config{
			InsecureSkipVerify: config.InsecureSkipVerify,
		}
	}
	c.http = &http.Client{Transport: tr}

	// build token lookup table
	c.tokens = make(map[string]int, len(config.Tokens))
	for i, token := range config.Tokens {
		token = strings.TrimSpace(token)
		if token != "" {
			c.tokens[token] = i
		}
	}

	return c
}

// Run starts the communication with Mattermost and blocks until stopped.
func (c *Client) Run() error {
	<-c.quit
	close(c.quit)
	return nil
}

// Stop terminates the communication.
func (c *Client) Stop() error {
	c.quit <- true
	return nil
}

// Send sends a message to Mattermost.
func (c *Client) Send(msg *message.OutMessage) error {
	om := translateOutMessage(msg)
	if c.config.OverrideUserName != "" && om.UserName == "" {
		om.UserName = c.config.OverrideUserName
	}
	if c.config.IconURL != "" && om.IconURL == "" {
		om.IconURL = c.config.IconURL
	}

	buf, err := json.Marshal(om)
	if err != nil {
		return err
	}

	res, err := c.http.Post(c.config.OutgoingURL, "application/json", bytes.NewReader(buf))
	if err != nil {
		return err
	}
	defer res.Body.Close()

	io.Copy(ioutil.Discard, res.Body)
	if res.StatusCode != 200 {
		return fmt.Errorf("Failed to send a message (%d %s)",
			res.StatusCode, res.Status)
	}

	return nil
}

// Receiver returns a channel that receives messages from chat service.
func (c *Client) Receiver() chan message.InMessage {
	return c.in
}

// IncomingWebHook returns webhook. It will be disabled if nil.
func (c *Client) IncomingWebHook() *adapter.IncomingWebHook {
	return &adapter.IncomingWebHook{
		Path:    c.config.IncomingPath,
		Handler: c.ServeHTTP,
	}
}

// ServeHTTP implements http.Handler interface.
// ServeHTTP receives a message from Mattermost outgoing webhook.
func (c *Client) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		c.logger.Printf("Invalid %q request from %q", r.Method, r.RemoteAddr)
		http.Error(w, "405 Method Not Allowed", http.StatusMethodNotAllowed)
		return
	}

	msg := InMessage{}
	if err := decodeForm(&msg, r); err != nil {
		c.logger.Printf("Invalid form data: %v", err)
		http.NotFound(w, r)
		return
	}

	if len(c.tokens) > 0 {
		if msg.Token == "" {
			c.logger.Printf("No token request from %q", r.RemoteAddr)
			http.Error(w, "401 Unauthorized", http.StatusUnauthorized)
			return
		} else if !c.validToken(msg.Token) {
			c.logger.Printf("Invalid token %q request from %q", msg.Token, r.RemoteAddr)
			http.Error(w, "401 Unauthorized", http.StatusUnauthorized)
			return
		}
	}

	im := translateInMessage(&msg)
	c.in <- *im
}

func (c *Client) validToken(token string) bool {
	_, ok := c.tokens[token]
	return ok
}

func decodeForm(msg *InMessage, r *http.Request) error {
	if err := r.ParseForm(); err != nil {
		return err
	}

	decoder := schema.NewDecoder()
	if err := decoder.Decode(msg, r.PostForm); err != nil {
		return err
	}

	return nil
}
<commit_msg>Fix HTTP response status code<commit_after>package mmhook

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mmbot/adapter"
	"mmbot/message"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/gorilla/schema"
)

// Client is a client for Mattermost.
type Client struct {
	config *adapter.Config
	logger *log.Logger
	http   *http.Client
	tokens map[string]int
	in     chan message.InMessage
	quit   chan bool
}

// NewClient returns new mattermost webhook client.
func NewClient(config *adapter.Config, logger *log.Logger) *Client {
	if logger == nil {
		logger = log.New(ioutil.Discard, "", 0)
	}
	c := &Client{
		config: config,
		logger: logger,
		in:     make(chan message.InMessage),
		quit:   make(chan bool),
	}

	tr := &http.Transport{
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
	}
	if config.InsecureSkipVerify {
		tr.TLSClientConfig = &tls.Config{
			InsecureSkipVerify: config.InsecureSkipVerify,
		}
	}
	c.http = &http.Client{Transport: tr}

	// build token lookup table
	c.tokens = make(map[string]int, len(config.Tokens))
	for i, token := range config.Tokens {
		token = strings.TrimSpace(token)
		if token != "" {
			c.tokens[token] = i
		}
	}

	return c
}

// Run starts the communication with Mattermost and blocks until stopped.
func (c *Client) Run() error {
	<-c.quit
	close(c.quit)
	return nil
}

// Stop terminates the communication.
func (c *Client) Stop() error {
	c.quit <- true
	return nil
}

// Send sends a message to Mattermost.
func (c *Client) Send(msg *message.OutMessage) error {
	om := translateOutMessage(msg)
	if c.config.OverrideUserName != "" && om.UserName == "" {
		om.UserName = c.config.OverrideUserName
	}
	if c.config.IconURL != "" && om.IconURL == "" {
		om.IconURL = c.config.IconURL
	}

	buf, err := json.Marshal(om)
	if err != nil {
		return err
	}

	res, err := c.http.Post(c.config.OutgoingURL, "application/json", bytes.NewReader(buf))
	if err != nil {
		return err
	}
	defer res.Body.Close()

	io.Copy(ioutil.Discard, res.Body)
	if res.StatusCode != 200 {
		return fmt.Errorf("Failed to send a message (%d %s)",
			res.StatusCode, res.Status)
	}

	return nil
}

// Receiver returns a channel that receives messages from chat service.
func (c *Client) Receiver() chan message.InMessage {
	return c.in
}

// IncomingWebHook returns webhook. It will be disabled if nil.
func (c *Client) IncomingWebHook() *adapter.IncomingWebHook {
	return &adapter.IncomingWebHook{
		Path:    c.config.IncomingPath,
		Handler: c.ServeHTTP,
	}
}

// ServeHTTP implements http.Handler interface.
// ServeHTTP receives a message from Mattermost outgoing webhook.
func (c *Client) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		c.logger.Printf("Invalid %q request from %q", r.Method, r.RemoteAddr)
		http.Error(w, "405 Method Not Allowed", http.StatusMethodNotAllowed)
		return
	}

	msg := InMessage{}
	if err := decodeForm(&msg, r); err != nil {
		c.logger.Printf("Invalid form data: %v", err)
		http.Error(w, "400 Bad Request", http.StatusBadRequest)
		return
	}

	if len(c.tokens) > 0 {
		if msg.Token == "" {
			c.logger.Printf("No token request from %q", r.RemoteAddr)
			http.Error(w, "400 Bad Request", http.StatusBadRequest)
			return
		} else if !c.validToken(msg.Token) {
			c.logger.Printf("Invalid token %q request from %q", msg.Token, r.RemoteAddr)
			http.Error(w, "400 Bad Request", http.StatusBadRequest)
			return
		}
	}

	im := translateInMessage(&msg)
	c.in <- *im
}

func (c *Client) validToken(token string) bool {
	_, ok := c.tokens[token]
	return ok
}

func decodeForm(msg *InMessage, r *http.Request) error {
	if err := r.ParseForm(); err != nil {
		return err
	}

	decoder := schema.NewDecoder()
	if err := decoder.Decode(msg, r.PostForm); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>/**
 * (C) Copyright 2013 MongoDB, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"os"
	"fmt"
	"bytes"
	"log/syslog"
)

// -------------------------------------
// The appender interface

type Appender interface {
	Append(log *Log) error
}

func FormatLog(log *Log) string {
	year, month, day := log.Timestamp.Date()
	hour, min, sec := log.Timestamp.Clock()

	return fmt.Sprintf("[%.4d/%.2d/%.2d %.2d:%.2d:%.2d] [%v.%v] [%v:%d] %v\n",
		year, month, day,
		hour, min, sec,
		log.Prefix, log.Level.Type(),
		log.Filename, log.Line,
		log.Message())
}

// -------------------------------------
// The syslog appender

type SyslogAppender struct {
	writer *syslog.Writer
}

func (self *SyslogAppender) Append(log *Log) error {
	switch log.Level {
		case Off: return nil
		case Debug: return self.writer.Debug(formatSyslogLog(log))
		case Info: return self.writer.Info(formatSyslogLog(log))
		case Warn: return self.writer.Warning(formatSyslogLog(log))
		case Error: return self.writer.Err(formatSyslogLog(log))
	}
	return nil
}

func NewSyslogAppender(network, raddr, appId string) (*SyslogAppender, error) {

	appender := &SyslogAppender{}
	var err error

	appender.writer, err = syslog.Dial(network, raddr, syslog.LOG_INFO|syslog.LOG_USER, appId)

	return appender, err
}

func formatSyslogLog(log *Log) string {
	year, month, day := log.Timestamp.Date()
	hour, min, sec := log.Timestamp.Clock()

	return fmt.Sprintf("[%.4d/%.2d/%.2d %.2d:%.2d:%.2d] [%v] [%v:%d] %v\n",
		year, month, day, hour, min, sec,
		log.Level.Type(),
		log.Filename, log.Line,
		log.Message())
}

// -------------------------------------
// The file appenders

type FileAppender struct {
	*os.File
}

func (self FileAppender) Append(log *Log) error {
	_, err := self.WriteString(FormatLog(log))
	return err
}

func StdOutAppender() *FileAppender {
	return &FileAppender{os.Stdout}
}

func StdErrAppender() *FileAppender {
	return &FileAppender{os.Stderr}
}

func DevNullAppender() (*FileAppender, error) {
	devNull, err := os.Open(os.DevNull)
	if err != nil {
		return nil, err
	}

	return &FileAppender{devNull}, nil
}

type StringAppender struct {
	*bytes.Buffer
}

func NewStringAppender(buffer *bytes.Buffer) *StringAppender {
	return &StringAppender{buffer}
}

func (self StringAppender) Append(log *Log) error {
	_, err := self.WriteString(FormatLog(log) + "\n")
	return err
}

// Return true if the log should be passed to the underlying
// `Appender`
type Filter func(log *Log) bool
type FilterAppender struct {
	Appender Appender
	Filter   Filter
}

func (self *FilterAppender) Append(log *Log) error {
	if self.Filter(log) == false {
		return nil
	}

	return self.Appender.Append(log)
}

func LevelFilter(threshold Level, appender Appender) *FilterAppender {
	filterFunc := func(log *Log) bool {
		return log.Level >= threshold
	}

	return &FilterAppender{
		Appender: appender,
		Filter:   filterFunc,
	}
}

<commit_msg>hacking<commit_after>/**
 * (C) Copyright 2013 MongoDB, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"os"
	"fmt"
	"bytes"
	"log/syslog"
)

// -------------------------------------
// The appender interface

type Appender interface {
	Append(log *Log) error
}

func FormatLog(log *Log) string {
	year, month, day := log.Timestamp.Date()
	hour, min, sec := log.Timestamp.Clock()

	return fmt.Sprintf("[%.4d/%.2d/%.2d %.2d:%.2d:%.2d] [%v.%v] [%v:%d] %v\n",
		year, month, day,
		hour, min, sec,
		log.Prefix, log.Level.Type(),
		log.Filename, log.Line,
		log.Message())
}

// -------------------------------------
// The syslog appender

type SyslogAppender struct {
	writer *syslog.Writer
}

func (self *SyslogAppender) Append(log *Log) error {
	switch log.Level {
		case Off: return nil
		case Debug: return self.writer.Debug(formatSyslogLog(log))
		case Info: return self.writer.Info(formatSyslogLog(log))
		case Warn: return self.writer.Warning(formatSyslogLog(log))
		case Error: return self.writer.Err(formatSyslogLog(log))
	}
	return nil
}

func NewSyslogAppender(network, raddr, appId string) (*SyslogAppender, error) {

	appender := &SyslogAppender{}
	var err error

	appender.writer, err = syslog.Dial(network, raddr, syslog.LOG_INFO|syslog.LOG_KERN, appId)

	return appender, err
}

func formatSyslogLog(log *Log) string {
	year, month, day := log.Timestamp.Date()
	hour, min, sec := log.Timestamp.Clock()

	return fmt.Sprintf("[%.4d/%.2d/%.2d %.2d:%.2d:%.2d] [%v] [%v:%d] %v\n",
		year, month, day, hour, min, sec,
		log.Level.Type(),
		log.Filename, log.Line,
		log.Message())
}

// -------------------------------------
// The file appenders

type FileAppender struct {
	*os.File
}

func (self FileAppender) Append(log *Log) error {
	_, err := self.WriteString(FormatLog(log))
	return err
}

func StdOutAppender() *FileAppender {
	return &FileAppender{os.Stdout}
}

func StdErrAppender() *FileAppender {
	return &FileAppender{os.Stderr}
}

func DevNullAppender() (*FileAppender, error) {
	devNull, err := os.Open(os.DevNull)
	if err != nil {
		return nil, err
	}

	return &FileAppender{devNull}, nil
}

type StringAppender struct {
	*bytes.Buffer
}

func NewStringAppender(buffer *bytes.Buffer) *StringAppender {
	return &StringAppender{buffer}
}

func (self StringAppender) Append(log *Log) error {
	_, err := self.WriteString(FormatLog(log) + "\n")
	return err
}

// Return true if the log should be passed to the underlying
// `Appender`
type Filter func(log *Log) bool
type FilterAppender struct {
	Appender Appender
	Filter   Filter
}

func (self *FilterAppender) Append(log *Log) error {
	if self.Filter(log) == false {
		return nil
	}

	return self.Appender.Append(log)
}

func LevelFilter(threshold Level, appender Appender) *FilterAppender {
	filterFunc := func(log *Log) bool {
		return log.Level >= threshold
	}

	return &FilterAppender{
		Appender: appender,
		Filter:   filterFunc,
	}
}

<|endoftext|>


<commit_before>// Copyright 2017 Microsoft Corporation and contributors.  All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.

// generator scours the github.com/Azure/azure-rest-api-specs repository in search for
// REST functionality that has Go configuration, then generates packages accordingly.
// This tool was developed with the intention that it be an internal tool for the
// Azure-SDK-for-Go team, but the usage of this package for public scenarios is in no
// way prohibited. For example, have a fork of https://github.com/Azure/autorest.go?
// Generate out and SDK with the same shape as our SDK, but using your patterns using
// this tool.
//
// Given that this code was developed as an internal tool, troubles with it that do not
// pertain to our usage of it may be slow to be fixed. Pull Requests are welcome though,
// feel free to contribute.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/marstr/collection"
)

const (
	defaultGenVer = "@microsoft.azure/autorest.go@v2.1.62"
)

var (
	targetFiles    collection.Enumerable
	packageVersion string
	outputBase     string
	logDirBase     string
	autorestVer    string
	errLog         *log.Logger
	statusLog      *log.Logger
	dryRun         bool
)

// version should be set by the linker when compiling this program by providing the following arguments:
// -X main.version=<version>
//
// If installing the generator in your machine, that means running the following the command:
//   go install -ldflags "-X main.version=<version>"
//
// The reason this is not controlled in source, is to allow for the git commit SHA1 to be used as the
// version of the string. To retrieve the currently checked-out git commit SHA1, you can use the command:
//   git rev-parse HEAD
//
// If this value is set, it is recommended that it be the value of the SHA1 git commit identifier for the
// source that was used at build time.
var version string

func main() {
	start := time.Now()

	type generationTuple struct {
		fileName     string
		packageName  string
		outputFolder string
	}

	literateFiles := collection.Where(targetFiles, func(subject interface{}) bool {
		return strings.EqualFold(path.Base(subject.(string)), "README.md")
	})

	stableAPIPaths := []string{}

	tuples := collection.SelectMany(literateFiles,
		// The following function compiles the regexp which finds Go related package tags in a literate file, and creates a collection.Unfolder.
		// This function has been declared this way so that the relatively expensive act of compiling a regexp is only done once.
		func() collection.Unfolder {
			const goSettingsPatternText = "```" + `\s+yaml\s+\$\(\s*tag\s*\)\s*==\s*'([\d\w\-\.]+)'\s*&&\s*\$\(go\)[\w\d\-\s:]*\s+output-folder:\s+\$\(go-sdk-folder\)([\w\d\-_\\/\.]+)\s+[\w\d\-\s:]*` + "```"
			goConfigPattern := regexp.MustCompile(goSettingsPatternText)

			const packagePatternText = `(?ms)\s+yaml\s+\$\(\s*tag\s*\)\s*==\s*'([\d\w\-\.]+)'\s*^input-file:(\s+.*?)\x60`
			packageConfigPattern := regexp.MustCompile(packagePatternText)

			// This function is a collection.Unfolder which takes a literate file as a path, and retrieves all configuration which applies to a package tag and Go.
			return func(subject interface{}) collection.Enumerator {
				results := make(chan interface{})

				go func() {
					defer close(results)

					targetContents, err := ioutil.ReadFile(subject.(string))
					if err != nil {
						errLog.Printf("Skipping %q because: %v", subject.(string), err)
						return
					}

					matches := goConfigPattern.FindAllStringSubmatch(string(targetContents), -1)

					if len(matches) == 0 {
						statusLog.Printf("Skipping %q because there were no package tags with go configuration found.", subject.(string))
					} else {
						packageMatches := packageConfigPattern.FindAllStringSubmatch(string(targetContents), -1)
						outputAPIFolders := map[string]string{}

						for _, submatch := range matches {

							packageName := normalizePath(submatch[1])
							outputFolder := normalizePath(submatch[2])
							leafFolder := getLeafOutputFolder(outputFolder)
							isStableAPI := isStableAPI(packageName, packageMatches)
							_, ok := outputAPIFolders[leafFolder]

							if !ok && isStableAPI {
								outputAPIFolders[leafFolder] = packageName
								stableAPIPaths = append(stableAPIPaths, path.Join(getRelativeOutputBase(), strings.Trim(outputFolder, "/")))
							}

							results <- generationTuple{
								fileName:     subject.(string),
								packageName:  packageName,
								outputFolder: outputFolder,
							}
						}
					}
				}()

				return results
			}
		}())

	if dryRun {
		for entry := range tuples.Enumerate(nil) {
			tuple := entry.(generationTuple)
			fmt.Printf("%q in %q to %q\n", tuple.packageName, tuple.fileName, tuple.outputFolder)
		}
	} else {
		var generatedCount, formattedCount, builtCount, vettedCount uint

		done := make(chan struct{})
		// Call AutoRest for each of the tags in each of the literate files, generating a Go package for calling that service.
		generated := tuples.Enumerate(done).ParallelSelect(func(subject interface{}) interface{} {
			tuple := subject.(generationTuple)
			args := []string{
				tuple.fileName,
				"--go",
				fmt.Sprintf("--go-sdk-folder='%s'", outputBase),
				"--verbose",
				"--tag=" + tuple.packageName,
				"--use=" + autorestVer,
				"--use-onever",
			}

			if packageVersion != "" {
				args = append(args, fmt.Sprintf("--package-version='%s'", packageVersion))
			}

			logFileLoc := filepath.Join(logDirBase, tuple.outputFolder)
			err := os.MkdirAll(logFileLoc, os.ModePerm)
			if err != nil {
				errLog.Printf("Could not create log directory %q", logFileLoc)
				return nil
			}

			logFile, err := os.Create(filepath.Join(logFileLoc, "autorestLog.txt"))
			if err != nil {
				errLog.Printf("Could not create log file %q for AutoRest generating from: %q in %q", logFileLoc, tuple.packageName, tuple.fileName)
				return nil
			}

			commandText := new(bytes.Buffer)
			fmt.Fprint(commandText, "Executing Command: \"")
			fmt.Fprint(commandText, "autorest ")
			for _, a := range args {
				fmt.Fprint(commandText, a)
				fmt.Fprint(commandText, " ")
			}
			commandText.Truncate(commandText.Len() - 1)
			fmt.Fprint(commandText, `"`)

			fmt.Fprintln(logFile, commandText.String())

			genProc := exec.Command("autorest", args...)
			genProc.Stdout = logFile
			genProc.Stderr = logFile

			err = genProc.Run()
			if err != nil {
				fmt.Fprintln(logFile, "Autorest Exectution Error: ", err)
				return nil
			}
			generatedCount++
			return path.Join(outputBase, tuple.outputFolder)
		}).Where(isntNil)

		// Take all of the generated packages and reformat them to adhere to Go's guidelines.
		formatted := generated.Select(func(subject interface{}) (result interface{}) {
			err := exec.Command("gofmt", "-w", subject.(string)).Run()
			if err == nil {
				formattedCount++
				result = subject
			} else {
				errLog.Printf("Failed to format: %q", subject.(string))
			}
			return
		}).Where(isntNil)

		// Build all of the packages as a sanity check.
		built := formatted.Select(func(subject interface{}) (result interface{}) {
			pkgName := strings.TrimPrefix(trimGoPath(subject.(string)), "/src/")
			err := exec.Command("go", "build", pkgName).Run()
			if err == nil {
				builtCount++
				return pkgName
			}
			errLog.Printf("Failed to build: %q", pkgName)
			return nil
		}).Where(isntNil)

		vetted := built.Select(func(subject interface{}) interface{} {
			err := exec.Command("go", "vet", subject.(string)).Run()
			if err == nil {
				vettedCount++
				return subject
			}
			errLog.Printf("Failed to vet: %q", subject.(string))
			return nil
		}).Where(isntNil)

		// Turn the crank. This loop forces evaluation of the chain of enumerators that were built up
		// in the code above.
		for range vetted {
			// Intenionally Left Blank
		}

		// Write list of stable apis.
		writeListToFile(path.Join(outputBase, "profiles", "latest", "stableApis.txt"), stableAPIPaths)

		fmt.Println("Execution Time: ", time.Now().Sub(start))
		fmt.Println("Generated: ", generatedCount)
		fmt.Println("Formatted: ", formattedCount)
		fmt.Println("Built: ", builtCount)
		fmt.Println("Vetted: ", vettedCount)
		fmt.Println("Stable APIs count: ", len(stableAPIPaths))
		close(done)
	}
}

func init() {
	var useRecursive, useStatus bool

	const automaticLogDirValue = "temp"
	const logDirUsage = "The root directory where all logs can be found. If the value `" + automaticLogDirValue + "` is supplied, a randomly named directory will be generated in the $TEMP directory."

	flag.BoolVar(&useRecursive, "r", false, "Recursively traverses the directories specified looking for literate files.")
	flag.StringVar(&outputBase, "o", getDefaultOutputBase(), "The root directory to use for the output of generated files. i.e. The value to be treated as the go-sdk-folder when AutoRest is called.")
	flag.BoolVar(&useStatus, "v", false, "Print status messages as generation takes place.")
	flag.BoolVar(&dryRun, "p", false, "Preview which packages would be generated instead of actaully calling autorest.")
	flag.StringVar(&packageVersion, "version", "", "The version that should be stamped on this SDK. This should be a semver.")
	flag.StringVar(&logDirBase, "l", getDefaultOutputBase(), logDirUsage)
	flag.StringVar(&autorestVer, "a", defaultGenVer, "The version of the AutoRest Go code generator to use, defaults to `"+defaultGenVer+"`.")

	// Override the default usage message, printing to stderr as the default one would.
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "generator [options] [target Literate files]...")
		flag.PrintDefaults()
	}

	flag.Parse()

	statusWriter := ioutil.Discard
	if useStatus {
		statusWriter = os.Stdout
	}
	statusLog = log.New(statusWriter, "[STATUS] ", 0)
	errLog = log.New(os.Stderr, "[ERROR] ", 0)

	if logDirBase == automaticLogDirValue {
		var err error
		logDirBase, err = ioutil.TempDir("", "az-go-sdk-logs")
		logDirBase = normalizePath(logDirBase)
		if err == nil {
			statusLog.Print("Generation logs can be found at: ", logDirBase)
		} else {
			errLog.Print("Logging disabled. Could not create directory: ", logDirBase)
		}
	}

	targetFiles = collection.AsEnumerable(flag.Args())
	targetFiles = collection.SelectMany(targetFiles, func(subject interface{}) collection.Enumerator {
		cast, ok := subject.(string)

		if !ok {
			return collection.Empty.Enumerate(nil)
		}
		pathInfo, err := os.Stat(cast)
		if err != nil {
			return collection.Empty.Enumerate(nil)
		}

		if pathInfo.IsDir() {
			traverser := collection.Directory{
				Location: cast,
				Options:  collection.DirectoryOptionsExcludeDirectories,
			}

			if useRecursive {
				traverser.Options |= collection.DirectoryOptionsRecursive
			}

			return traverser.Enumerate(nil)
		}

		return collection.AsEnumerable(cast).Enumerate(nil)
	})

	targetFiles = collection.Select(targetFiles, func(subject interface{}) interface{} {
		return normalizePath(subject.(string))
	})
}

// goPath returns the value of the environement variable GOPATH at the beginning of this
// program's execution. The actual enviroment variable is only queried once, before any calls
// are made to this function.
var goPath = func() func() string {
	val := normalizePath(os.Getenv("GOPATH"))
	return func() string {
		return val
	}
}()

// getDefaultOutputBase returns the default location of the Azure-SDK-for-Go on your filesystem.
func getDefaultOutputBase() string {
	return normalizePath(path.Join(goPath(), "src", "github.com", "Azure", "azure-sdk-for-go"))
}

// getDefaultOutputBase returns the default location of the Azure-SDK-for-Go on your filesystem relative to the 'GOPATH/src'.
func getRelativeOutputBase() string {
	return normalizePath(path.Join("github.com", "Azure", "azure-sdk-for-go"))
}

// trimGoPath operates like strings.TrimPrefix, where the prefix is always the value of GOPATH in the
// environment in which this program is being executed.
func trimGoPath(location string) string {
	return strings.TrimPrefix(normalizePath(location), goPath())
}

// normalizePath ensures that a path is expressed using forward slashes for sanity when working
// with Go Standard Library functions that seem to expect this.
func normalizePath(location string) (result string) {
	result = strings.Replace(location, `\`, "/", -1)
	return
}

//getLeafOutputFolder returns the leaf of a given path.
func getLeafOutputFolder(location string) string {
	_, leaf := filepath.Split(location)
	return leaf
}

//isStableAPI returns true if for the given package name there are no preview input files.
func isStableAPI(packageName string, packageInputFiles [][]string) bool {
	for _, packageDefinition := range packageInputFiles {
		if !strings.Contains(normalizePath(packageDefinition[2]), "/preview/") && packageName == packageDefinition[1] {
			return true
		}
	}
	return false
}

// isntNil is a simple `collection.Predicate` which filters out `nil` objects from an Enumerator.
func isntNil(subject interface{}) bool {
	return subject != nil
}

// writeListToFile prints a list of strings to the specified filepath.
func writeListToFile(filepath string, list []string) error {
	outputFile, err := os.Create(filepath)
	if err != nil {
		return err
	}
	for _, el := range list {
		fmt.Fprintln(outputFile, el)
	}
	return nil
}
<commit_msg>Use Regexp.Copy function to avoid lock contention<commit_after>// Copyright 2017 Microsoft Corporation and contributors.  All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.

// generator scours the github.com/Azure/azure-rest-api-specs repository in search for
// REST functionality that has Go configuration, then generates packages accordingly.
// This tool was developed with the intention that it be an internal tool for the
// Azure-SDK-for-Go team, but the usage of this package for public scenarios is in no
// way prohibited. For example, have a fork of https://github.com/Azure/autorest.go?
// Generate out and SDK with the same shape as our SDK, but using your patterns using
// this tool.
//
// Given that this code was developed as an internal tool, troubles with it that do not
// pertain to our usage of it may be slow to be fixed. Pull Requests are welcome though,
// feel free to contribute.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/marstr/collection"
)

const (
	defaultGenVer = "@microsoft.azure/autorest.go@v2.1.62"
)

var (
	targetFiles    collection.Enumerable
	packageVersion string
	outputBase     string
	logDirBase     string
	autorestVer    string
	errLog         *log.Logger
	statusLog      *log.Logger
	dryRun         bool
)

// version should be set by the linker when compiling this program by providing the following arguments:
// -X main.version=<version>
//
// If installing the generator in your machine, that means running the following the command:
//   go install -ldflags "-X main.version=<version>"
//
// The reason this is not controlled in source, is to allow for the git commit SHA1 to be used as the
// version of the string. To retrieve the currently checked-out git commit SHA1, you can use the command:
//   git rev-parse HEAD
//
// If this value is set, it is recommended that it be the value of the SHA1 git commit identifier for the
// source that was used at build time.
var version string

func main() {
	start := time.Now()

	type generationTuple struct {
		fileName     string
		packageName  string
		outputFolder string
	}

	literateFiles := collection.Where(targetFiles, func(subject interface{}) bool {
		return strings.EqualFold(path.Base(subject.(string)), "README.md")
	})

	stableAPIPaths := []string{}

	tuples := collection.SelectMany(literateFiles,
		// The following function compiles the regexp which finds Go related package tags in a literate file, and creates a collection.Unfolder.
		// This function has been declared this way so that the relatively expensive act of compiling a regexp is only done once.
		func() collection.Unfolder {
			const goSettingsPatternText = "```" + `\s+yaml\s+\$\(\s*tag\s*\)\s*==\s*'([\d\w\-\.]+)'\s*&&\s*\$\(go\)[\w\d\-\s:]*\s+output-folder:\s+\$\(go-sdk-folder\)([\w\d\-_\\/\.]+)\s+[\w\d\-\s:]*` + "```"
			goConfigPattern := regexp.MustCompile(goSettingsPatternText)

			const packagePatternText = `(?ms)\s+yaml\s+\$\(\s*tag\s*\)\s*==\s*'([\d\w\-\.]+)'\s*^input-file:(\s+.*?)\x60`
			packageConfigPattern := regexp.MustCompile(packagePatternText)

			// This function is a collection.Unfolder which takes a literate file as a path, and retrieves all configuration which applies to a package tag and Go.
			return func(subject interface{}) collection.Enumerator {
				results := make(chan interface{})

				go func() {
					defer close(results)

					targetContents, err := ioutil.ReadFile(subject.(string))
					if err != nil {
						errLog.Printf("Skipping %q because: %v", subject.(string), err)
						return
					}

					matches := goConfigPattern.Copy().FindAllStringSubmatch(string(targetContents), -1)

					if len(matches) == 0 {
						statusLog.Printf("Skipping %q because there were no package tags with go configuration found.", subject.(string))
					} else {
						packageMatches := packageConfigPattern.Copy().FindAllStringSubmatch(string(targetContents), -1)
						outputAPIFolders := map[string]string{}

						for _, submatch := range matches {

							packageName := normalizePath(submatch[1])
							outputFolder := normalizePath(submatch[2])
							leafFolder := getLeafOutputFolder(outputFolder)
							isStableAPI := isStableAPI(packageName, packageMatches)
							_, ok := outputAPIFolders[leafFolder]

							if !ok && isStableAPI {
								outputAPIFolders[leafFolder] = packageName
								stableAPIPaths = append(stableAPIPaths, path.Join(getRelativeOutputBase(), strings.Trim(outputFolder, "/")))
							}

							results <- generationTuple{
								fileName:     subject.(string),
								packageName:  packageName,
								outputFolder: outputFolder,
							}
						}
					}
				}()

				return results
			}
		}())

	if dryRun {
		for entry := range tuples.Enumerate(nil) {
			tuple := entry.(generationTuple)
			fmt.Printf("%q in %q to %q\n", tuple.packageName, tuple.fileName, tuple.outputFolder)
		}
	} else {
		var generatedCount, formattedCount, builtCount, vettedCount uint

		done := make(chan struct{})
		// Call AutoRest for each of the tags in each of the literate files, generating a Go package for calling that service.
		generated := tuples.Enumerate(done).ParallelSelect(func(subject interface{}) interface{} {
			tuple := subject.(generationTuple)
			args := []string{
				tuple.fileName,
				"--go",
				fmt.Sprintf("--go-sdk-folder='%s'", outputBase),
				"--verbose",
				"--tag=" + tuple.packageName,
				"--use=" + autorestVer,
				"--use-onever",
			}

			if packageVersion != "" {
				args = append(args, fmt.Sprintf("--package-version='%s'", packageVersion))
			}

			logFileLoc := filepath.Join(logDirBase, tuple.outputFolder)
			err := os.MkdirAll(logFileLoc, os.ModePerm)
			if err != nil {
				errLog.Printf("Could not create log directory %q", logFileLoc)
				return nil
			}

			logFile, err := os.Create(filepath.Join(logFileLoc, "autorestLog.txt"))
			if err != nil {
				errLog.Printf("Could not create log file %q for AutoRest generating from: %q in %q", logFileLoc, tuple.packageName, tuple.fileName)
				return nil
			}

			commandText := new(bytes.Buffer)
			fmt.Fprint(commandText, "Executing Command: \"")
			fmt.Fprint(commandText, "autorest ")
			for _, a := range args {
				fmt.Fprint(commandText, a)
				fmt.Fprint(commandText, " ")
			}
			commandText.Truncate(commandText.Len() - 1)
			fmt.Fprint(commandText, `"`)

			fmt.Fprintln(logFile, commandText.String())

			genProc := exec.Command("autorest", args...)
			genProc.Stdout = logFile
			genProc.Stderr = logFile

			err = genProc.Run()
			if err != nil {
				fmt.Fprintln(logFile, "Autorest Exectution Error: ", err)
				return nil
			}
			generatedCount++
			return path.Join(outputBase, tuple.outputFolder)
		}).Where(isntNil)

		// Take all of the generated packages and reformat them to adhere to Go's guidelines.
		formatted := generated.Select(func(subject interface{}) (result interface{}) {
			err := exec.Command("gofmt", "-w", subject.(string)).Run()
			if err == nil {
				formattedCount++
				result = subject
			} else {
				errLog.Printf("Failed to format: %q", subject.(string))
			}
			return
		}).Where(isntNil)

		// Build all of the packages as a sanity check.
		built := formatted.Select(func(subject interface{}) (result interface{}) {
			pkgName := strings.TrimPrefix(trimGoPath(subject.(string)), "/src/")
			err := exec.Command("go", "build", pkgName).Run()
			if err == nil {
				builtCount++
				return pkgName
			}
			errLog.Printf("Failed to build: %q", pkgName)
			return nil
		}).Where(isntNil)

		vetted := built.Select(func(subject interface{}) interface{} {
			err := exec.Command("go", "vet", subject.(string)).Run()
			if err == nil {
				vettedCount++
				return subject
			}
			errLog.Printf("Failed to vet: %q", subject.(string))
			return nil
		}).Where(isntNil)

		// Turn the crank. This loop forces evaluation of the chain of enumerators that were built up
		// in the code above.
		for range vetted {
			// Intenionally Left Blank
		}

		// Write list of stable apis.
		writeListToFile(path.Join(outputBase, "profiles", "latest", "stableApis.txt"), stableAPIPaths)

		fmt.Println("Execution Time: ", time.Now().Sub(start))
		fmt.Println("Generated: ", generatedCount)
		fmt.Println("Formatted: ", formattedCount)
		fmt.Println("Built: ", builtCount)
		fmt.Println("Vetted: ", vettedCount)
		fmt.Println("Stable APIs count: ", len(stableAPIPaths))
		close(done)
	}
}

func init() {
	var useRecursive, useStatus bool

	const automaticLogDirValue = "temp"
	const logDirUsage = "The root directory where all logs can be found. If the value `" + automaticLogDirValue + "` is supplied, a randomly named directory will be generated in the $TEMP directory."

	flag.BoolVar(&useRecursive, "r", false, "Recursively traverses the directories specified looking for literate files.")
	flag.StringVar(&outputBase, "o", getDefaultOutputBase(), "The root directory to use for the output of generated files. i.e. The value to be treated as the go-sdk-folder when AutoRest is called.")
	flag.BoolVar(&useStatus, "v", false, "Print status messages as generation takes place.")
	flag.BoolVar(&dryRun, "p", false, "Preview which packages would be generated instead of actaully calling autorest.")
	flag.StringVar(&packageVersion, "version", "", "The version that should be stamped on this SDK. This should be a semver.")
	flag.StringVar(&logDirBase, "l", getDefaultOutputBase(), logDirUsage)
	flag.StringVar(&autorestVer, "a", defaultGenVer, "The version of the AutoRest Go code generator to use, defaults to `"+defaultGenVer+"`.")

	// Override the default usage message, printing to stderr as the default one would.
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "generator [options] [target Literate files]...")
		flag.PrintDefaults()
	}

	flag.Parse()

	statusWriter := ioutil.Discard
	if useStatus {
		statusWriter = os.Stdout
	}
	statusLog = log.New(statusWriter, "[STATUS] ", 0)
	errLog = log.New(os.Stderr, "[ERROR] ", 0)

	if logDirBase == automaticLogDirValue {
		var err error
		logDirBase, err = ioutil.TempDir("", "az-go-sdk-logs")
		logDirBase = normalizePath(logDirBase)
		if err == nil {
			statusLog.Print("Generation logs can be found at: ", logDirBase)
		} else {
			errLog.Print("Logging disabled. Could not create directory: ", logDirBase)
		}
	}

	targetFiles = collection.AsEnumerable(flag.Args())
	targetFiles = collection.SelectMany(targetFiles, func(subject interface{}) collection.Enumerator {
		cast, ok := subject.(string)

		if !ok {
			return collection.Empty.Enumerate(nil)
		}
		pathInfo, err := os.Stat(cast)
		if err != nil {
			return collection.Empty.Enumerate(nil)
		}

		if pathInfo.IsDir() {
			traverser := collection.Directory{
				Location: cast,
				Options:  collection.DirectoryOptionsExcludeDirectories,
			}

			if useRecursive {
				traverser.Options |= collection.DirectoryOptionsRecursive
			}

			return traverser.Enumerate(nil)
		}

		return collection.AsEnumerable(cast).Enumerate(nil)
	})

	targetFiles = collection.Select(targetFiles, func(subject interface{}) interface{} {
		return normalizePath(subject.(string))
	})
}

// goPath returns the value of the environement variable GOPATH at the beginning of this
// program's execution. The actual enviroment variable is only queried once, before any calls
// are made to this function.
var goPath = func() func() string {
	val := normalizePath(os.Getenv("GOPATH"))
	return func() string {
		return val
	}
}()

// getDefaultOutputBase returns the default location of the Azure-SDK-for-Go on your filesystem.
func getDefaultOutputBase() string {
	return normalizePath(path.Join(goPath(), "src", "github.com", "Azure", "azure-sdk-for-go"))
}

// getDefaultOutputBase returns the default location of the Azure-SDK-for-Go on your filesystem relative to the 'GOPATH/src'.
func getRelativeOutputBase() string {
	return normalizePath(path.Join("github.com", "Azure", "azure-sdk-for-go"))
}

// trimGoPath operates like strings.TrimPrefix, where the prefix is always the value of GOPATH in the
// environment in which this program is being executed.
func trimGoPath(location string) string {
	return strings.TrimPrefix(normalizePath(location), goPath())
}

// normalizePath ensures that a path is expressed using forward slashes for sanity when working
// with Go Standard Library functions that seem to expect this.
func normalizePath(location string) (result string) {
	result = strings.Replace(location, `\`, "/", -1)
	return
}

//getLeafOutputFolder returns the leaf of a given path.
func getLeafOutputFolder(location string) string {
	_, leaf := filepath.Split(location)
	return leaf
}

//isStableAPI returns true if for the given package name there are no preview input files.
func isStableAPI(packageName string, packageInputFiles [][]string) bool {
	for _, packageDefinition := range packageInputFiles {
		if !strings.Contains(normalizePath(packageDefinition[2]), "/preview/") && packageName == packageDefinition[1] {
			return true
		}
	}
	return false
}

// isntNil is a simple `collection.Predicate` which filters out `nil` objects from an Enumerator.
func isntNil(subject interface{}) bool {
	return subject != nil
}

// writeListToFile prints a list of strings to the specified filepath.
func writeListToFile(filepath string, list []string) error {
	outputFile, err := os.Create(filepath)
	if err != nil {
		return err
	}
	for _, el := range list {
		fmt.Fprintln(outputFile, el)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"context"
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
	"syscall"
	"time"

	"pixur.org/pixur/be/schema"
	sdb "pixur.org/pixur/be/schema/db"
	tab "pixur.org/pixur/be/schema/tables"
	"pixur.org/pixur/be/server/config"
	"pixur.org/pixur/be/tasks"

	"golang.org/x/crypto/ssh/terminal"
)

var (
	initTables    = flag.Bool("init_tables", false, "If set, initialize database tables")
	createNewUser = flag.Bool("create_first_user", true, "If set, creates a new")
)

func run() error {
	log.Println("Opening Database")
	db, err := sdb.Open(config.Conf.DbName, config.Conf.DbConfig)
	if err != nil {
		return err
	}
	defer db.Close()
	if *initTables {
		var stmts []string
		stmts = append(stmts, tab.SqlTables[db.Adapter().Name()]...)
		stmts = append(stmts, tab.SqlInitTables[db.Adapter().Name()]...)
		log.Println("Initializing tables")
		if err := db.InitSchema(stmts); err != nil {
			return err
		}
	}
	if *createNewUser {
		r := bufio.NewReader(os.Stdin)
		fmt.Print("Admin Ident (e.g. foo@bar.com): ")
		ident, err := r.ReadString('\n')
		if err != nil {
			return err
		}

		fmt.Print("Admin Secret (e.g. 12345): ")
		secret, err := terminal.ReadPassword(int(syscall.Stdin))
		if err != nil {
			return err
		}

		task := &tasks.CreateUserTask{
			DB:  db,
			Now: time.Now,

			Ident:  strings.TrimSpace(ident),
			Secret: string(secret),
			Ctx:    context.Background(),
			Capability: append(schema.UserNewCap,
				schema.User_PIC_SOFT_DELETE,
				schema.User_USER_UPDATE_CAPABILITY),
		}

		// Presumably there is nobody in the database yet, so we need to temporarily relax permissions
		// on the anonymous user.
		oldcap := schema.AnonymousUser.Capability
		schema.AnonymousUser.Capability = []schema.User_Capability{schema.User_USER_CREATE}
		sts := new(tasks.TaskRunner).Run(task)
		schema.AnonymousUser.Capability = oldcap
		if sts != nil {
			return sts
		}

		log.Println("Created user")
	}

	return nil
}

func main() {
	flag.Parse()

	if err := run(); err != nil {
		log.Println(err)
	}
}
<commit_msg>tools: update initsite to use glog<commit_after>package main

import (
	"bufio"
	"context"
	"flag"
	"fmt"
	"os"
	"strings"
	"syscall"
	"time"

	"pixur.org/pixur/be/schema"
	sdb "pixur.org/pixur/be/schema/db"
	tab "pixur.org/pixur/be/schema/tables"
	"pixur.org/pixur/be/server/config"
	"pixur.org/pixur/be/tasks"

	"github.com/golang/glog"
	"golang.org/x/crypto/ssh/terminal"
)

var (
	initTables    = flag.Bool("init_tables", false, "If set, initialize database tables")
	createNewUser = flag.Bool("create_first_user", true, "If set, creates a new")
)

func run() error {
	glog.Info("Opening Database")
	db, err := sdb.Open(config.Conf.DbName, config.Conf.DbConfig)
	if err != nil {
		return err
	}
	defer db.Close()
	if *initTables {
		var stmts []string
		stmts = append(stmts, tab.SqlTables[db.Adapter().Name()]...)
		stmts = append(stmts, tab.SqlInitTables[db.Adapter().Name()]...)
		glog.Info("Initializing tables")
		if err := db.InitSchema(stmts); err != nil {
			return err
		}
	}
	if *createNewUser {
		r := bufio.NewReader(os.Stdin)
		fmt.Print("Admin Ident (e.g. foo@bar.com): ")
		ident, err := r.ReadString('\n')
		if err != nil {
			return err
		}

		fmt.Print("Admin Secret (e.g. 12345): ")
		secret, err := terminal.ReadPassword(int(syscall.Stdin))
		if err != nil {
			return err
		}

		task := &tasks.CreateUserTask{
			DB:  db,
			Now: time.Now,

			Ident:  strings.TrimSpace(ident),
			Secret: string(secret),
			Ctx:    context.Background(),
			Capability: append(schema.UserNewCap,
				schema.User_PIC_SOFT_DELETE,
				schema.User_USER_UPDATE_CAPABILITY),
		}

		// Presumably there is nobody in the database yet, so we need to temporarily relax permissions
		// on the anonymous user.
		oldcap := schema.AnonymousUser.Capability
		schema.AnonymousUser.Capability = []schema.User_Capability{schema.User_USER_CREATE}
		sts := new(tasks.TaskRunner).Run(task)
		schema.AnonymousUser.Capability = oldcap
		if sts != nil {
			return sts
		}

		glog.Info("Created user")
	}

	return nil
}

func main() {
	args := []string{"-logtostderr"}
	args = append(args, os.Args[1:]...)
	flag.CommandLine.Parse(args)

	if err := run(); err != nil {
		glog.Error(err)
	}
}
<|endoftext|>


<commit_before>package logging

import (
	"github.com/tychoish/grip/level"
	"github.com/tychoish/grip/message"
)

// Internal helpers to manage sending interaction

func (g *Grip) conditionalSend(conditional bool, m message.Composer) {
	if !conditional {
		return
	}

	g.Send(m)
	return
}

/////////////

func (g *Grip) LogWhen(conditional bool, l level.Priority, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(l, m))
}
func (g *Grip) LogWhenln(conditional bool, l level.Priority, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(l, msg...))
}
func (g *Grip) LogWhenf(conditional bool, l level.Priority, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(l, msg, args...))
}

/////////////

func (g *Grip) DefaultWhen(conditional bool, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(g.Level().Default, m))
}
func (g *Grip) DefaultWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(g.Level().Default, msg...))
}
func (g *Grip) DefaultWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(g.Level().Default, msg, args...))
}

/////////////

func (g *Grip) EmergencyWhen(conditional bool, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(level.Emergency, m))
}
func (g *Grip) EmergencyWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(level.Emergency, msg...))
}
func (g *Grip) EmergencyWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(level.Emergency, msg, args...))
}

/////////////

func (g *Grip) AlertWhen(conditional bool, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(level.Alert, m))
}
func (g *Grip) AlertWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(level.Alert, msg...))
}
func (g *Grip) AlertWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(level.Alert, msg, args...))
}

/////////////

func (g *Grip) CriticalWhen(conditional bool, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(level.Critical, m))
}
func (g *Grip) CriticalWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(level.Critical, msg...))
}
func (g *Grip) CriticalWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(level.Critical, msg, args...))
}

/////////////

func (g *Grip) ErrorWhen(conditional bool, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(level.Critical, m))
}
func (g *Grip) ErrorWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(level.Critical, msg...))
}
func (g *Grip) ErrorWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(level.Critical, msg, args...))
}

/////////////

func (g *Grip) WarningWhen(conditional bool, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(level.Warning, m))
}
func (g *Grip) WarningWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(level.Warning, msg...))
}
func (g *Grip) WarningWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(level.Warning, msg, args...))
}

/////////////

func (g *Grip) NoticeWhen(conditional bool, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(level.Notice, m))
}
func (g *Grip) NoticeWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(level.Notice, msg...))
}
func (g *Grip) NoticeWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(level.Notice, msg, args...))
}

/////////////

func (g *Grip) InfoWhen(conditional bool, msg interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(level.Info, msg))
}
func (g *Grip) InfoWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(level.Info, msg...))
}
func (g *Grip) InfoWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(level.Info, msg, args...))
}

/////////////

func (g *Grip) DebugWhen(conditional bool, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(level.Debug, m))
}
func (g *Grip) DebugWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(level.Debug, msg...))
}
func (g *Grip) DebugWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(level.Debug, msg, args...))
}
<commit_msg>fix inconsistent variable name<commit_after>package logging

import (
	"github.com/tychoish/grip/level"
	"github.com/tychoish/grip/message"
)

// Internal helpers to manage sending interaction

func (g *Grip) conditionalSend(conditional bool, m message.Composer) {
	if !conditional {
		return
	}

	g.Send(m)
	return
}

/////////////

func (g *Grip) LogWhen(conditional bool, l level.Priority, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(l, m))
}
func (g *Grip) LogWhenln(conditional bool, l level.Priority, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(l, msg...))
}
func (g *Grip) LogWhenf(conditional bool, l level.Priority, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(l, msg, args...))
}

/////////////

func (g *Grip) DefaultWhen(conditional bool, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(g.Level().Default, m))
}
func (g *Grip) DefaultWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(g.Level().Default, msg...))
}
func (g *Grip) DefaultWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(g.Level().Default, msg, args...))
}

/////////////

func (g *Grip) EmergencyWhen(conditional bool, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(level.Emergency, m))
}
func (g *Grip) EmergencyWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(level.Emergency, msg...))
}
func (g *Grip) EmergencyWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(level.Emergency, msg, args...))
}

/////////////

func (g *Grip) AlertWhen(conditional bool, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(level.Alert, m))
}
func (g *Grip) AlertWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(level.Alert, msg...))
}
func (g *Grip) AlertWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(level.Alert, msg, args...))
}

/////////////

func (g *Grip) CriticalWhen(conditional bool, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(level.Critical, m))
}
func (g *Grip) CriticalWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(level.Critical, msg...))
}
func (g *Grip) CriticalWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(level.Critical, msg, args...))
}

/////////////

func (g *Grip) ErrorWhen(conditional bool, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(level.Critical, m))
}
func (g *Grip) ErrorWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(level.Critical, msg...))
}
func (g *Grip) ErrorWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(level.Critical, msg, args...))
}

/////////////

func (g *Grip) WarningWhen(conditional bool, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(level.Warning, m))
}
func (g *Grip) WarningWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(level.Warning, msg...))
}
func (g *Grip) WarningWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(level.Warning, msg, args...))
}

/////////////

func (g *Grip) NoticeWhen(conditional bool, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(level.Notice, m))
}
func (g *Grip) NoticeWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(level.Notice, msg...))
}
func (g *Grip) NoticeWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(level.Notice, msg, args...))
}

/////////////

func (g *Grip) InfoWhen(conditional bool, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(level.Info, m))
}
func (g *Grip) InfoWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(level.Info, msg...))
}
func (g *Grip) InfoWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(level.Info, msg, args...))
}

/////////////

func (g *Grip) DebugWhen(conditional bool, m interface{}) {
	g.conditionalSend(conditional, message.ConvertToComposer(level.Debug, m))
}
func (g *Grip) DebugWhenln(conditional bool, msg ...interface{}) {
	g.conditionalSend(conditional, message.NewLinesMessage(level.Debug, msg...))
}
func (g *Grip) DebugWhenf(conditional bool, msg string, args ...interface{}) {
	g.conditionalSend(conditional, message.NewFormattedMessage(level.Debug, msg, args...))
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logstream

import (
	"bufio"
	"encoding/json"
	"fmt"
	"strings"
	"sync"
	"testing"
	"time"

	"golang.org/x/sync/errgroup"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/knative/pkg/ptr"
	"github.com/knative/pkg/test"
	"github.com/knative/pkg/test/helpers"
)

type kubelogs struct {
	namespace string

	once sync.Once
	m    sync.RWMutex
	keys map[string]logger
	err  error
}

type logger func(string, ...interface{})

var _ streamer = (*kubelogs)(nil)

// timeFormat defines a simple timestamp with millisecond granularity
const timeFormat = "15:04:05.000"

func (k *kubelogs) init(t *testing.T) {
	k.keys = make(map[string]logger)

	kc, err := test.NewKubeClient(test.Flags.Kubeconfig, test.Flags.Cluster)
	if err != nil {
		t.Errorf("Error loading client config: %v", err)
	}

	// List the pods in the given namespace.
	pl, err := kc.Kube.CoreV1().Pods(k.namespace).List(metav1.ListOptions{})
	if err != nil {
		t.Errorf("Error listing pods: %v", err)
	}

	eg := errgroup.Group{}
	for _, pod := range pl.Items {
		// Grab data from all containers in the pods.  We need this in case
		// an envoy sidecar is injected for mesh installs.  This should be
		// equivalent to --all-containers.
		for _, container := range pod.Spec.Containers {
			// Required for capture below.
			pod, container := pod, container
			eg.Go(func() error {
				options := &corev1.PodLogOptions{
					Container: container.Name,
					// Follow directs the api server to continuously stream logs back.
					Follow: true,
					// Only return new logs (this value is being used for "epsilon").
					SinceSeconds: ptr.Int64(1),
				}

				req := kc.Kube.CoreV1().Pods(k.namespace).GetLogs(pod.Name, options)
				stream, err := req.Stream()
				if err != nil {
					return err
				}
				defer stream.Close()
				// Read this container's stream.
				for scanner := bufio.NewScanner(stream); scanner.Scan(); {
					k.handleLine(scanner.Text())
				}
				return fmt.Errorf("logstream completed prematurely for: %s/%s", pod.Name, container.Name)
			})
		}
	}

	// Monitor the error group in the background and surface an error on the kubelogs
	// in case anything had an active stream open.
	go func() {
		if err := eg.Wait(); err != nil {
			k.m.Lock()
			defer k.m.Unlock()
			k.err = err
		}
	}()
}

func (k *kubelogs) handleLine(l string) {
	// This holds the standard structure of our logs.
	var line struct {
		Level      string    `json:"level"`
		Timestamp  time.Time `json:"ts"`
		Controller string    `json:"knative.dev/controller"`
		Caller     string    `json:"caller"`
		Key        string    `json:"knative.dev/key"`
		Message    string    `json:"msg"`

		// TODO(mattmoor): Parse out more context.
	}
	if err := json.Unmarshal([]byte(l), &line); err != nil {
		// Ignore malformed lines.
		return
	}
	if line.Key == "" {
		return
	}

	k.m.RLock()
	defer k.m.RUnlock()

	for name, logf := range k.keys {
		// TODO(mattmoor): Do a slightly smarter match.
		if !strings.Contains(line.Key, name) {
			continue
		}
		// E 15:04:05.000 [route-controller] [default/testroute-xyz] this is my message
		logf("%s %s [%s] [%s] %s",
			strings.ToUpper(string(line.Level[0])),
			line.Timestamp.Format(timeFormat),
			line.Controller,
			line.Key,
			line.Message)
	}
}

// Start implements streamer
func (k *kubelogs) Start(t *testing.T) Canceler {
	k.once.Do(func() { k.init(t) })

	name := helpers.ObjectPrefixForTest(t)

	// Register a key
	k.m.Lock()
	defer k.m.Unlock()
	k.keys[name] = t.Logf

	// Return a function that unregisters that key.
	return func() {
		k.m.Lock()
		defer k.m.Unlock()
		delete(k.keys, name)

		if k.err != nil {
			t.Errorf("error during logstream: %v", k.err)
		}
	}
}
<commit_msg>Add 'error' field to extracted logs, if available. (#487)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logstream

import (
	"bufio"
	"encoding/json"
	"fmt"
	"strings"
	"sync"
	"testing"
	"time"

	"golang.org/x/sync/errgroup"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/knative/pkg/ptr"
	"github.com/knative/pkg/test"
	"github.com/knative/pkg/test/helpers"
)

type kubelogs struct {
	namespace string

	once sync.Once
	m    sync.RWMutex
	keys map[string]logger
	err  error
}

type logger func(string, ...interface{})

var _ streamer = (*kubelogs)(nil)

// timeFormat defines a simple timestamp with millisecond granularity
const timeFormat = "15:04:05.000"

func (k *kubelogs) init(t *testing.T) {
	k.keys = make(map[string]logger)

	kc, err := test.NewKubeClient(test.Flags.Kubeconfig, test.Flags.Cluster)
	if err != nil {
		t.Errorf("Error loading client config: %v", err)
	}

	// List the pods in the given namespace.
	pl, err := kc.Kube.CoreV1().Pods(k.namespace).List(metav1.ListOptions{})
	if err != nil {
		t.Errorf("Error listing pods: %v", err)
	}

	eg := errgroup.Group{}
	for _, pod := range pl.Items {
		// Grab data from all containers in the pods.  We need this in case
		// an envoy sidecar is injected for mesh installs.  This should be
		// equivalent to --all-containers.
		for _, container := range pod.Spec.Containers {
			// Required for capture below.
			pod, container := pod, container
			eg.Go(func() error {
				options := &corev1.PodLogOptions{
					Container: container.Name,
					// Follow directs the api server to continuously stream logs back.
					Follow: true,
					// Only return new logs (this value is being used for "epsilon").
					SinceSeconds: ptr.Int64(1),
				}

				req := kc.Kube.CoreV1().Pods(k.namespace).GetLogs(pod.Name, options)
				stream, err := req.Stream()
				if err != nil {
					return err
				}
				defer stream.Close()
				// Read this container's stream.
				for scanner := bufio.NewScanner(stream); scanner.Scan(); {
					k.handleLine(scanner.Text())
				}
				return fmt.Errorf("logstream completed prematurely for: %s/%s", pod.Name, container.Name)
			})
		}
	}

	// Monitor the error group in the background and surface an error on the kubelogs
	// in case anything had an active stream open.
	go func() {
		if err := eg.Wait(); err != nil {
			k.m.Lock()
			defer k.m.Unlock()
			k.err = err
		}
	}()
}

func (k *kubelogs) handleLine(l string) {
	// This holds the standard structure of our logs.
	var line struct {
		Level      string    `json:"level"`
		Timestamp  time.Time `json:"ts"`
		Controller string    `json:"knative.dev/controller"`
		Caller     string    `json:"caller"`
		Key        string    `json:"knative.dev/key"`
		Message    string    `json:"msg"`
		Error      string    `json:"error"`

		// TODO(mattmoor): Parse out more context.
	}
	if err := json.Unmarshal([]byte(l), &line); err != nil {
		// Ignore malformed lines.
		return
	}
	if line.Key == "" {
		return
	}

	k.m.RLock()
	defer k.m.RUnlock()

	for name, logf := range k.keys {
		// TODO(mattmoor): Do a slightly smarter match.
		if !strings.Contains(line.Key, name) {
			continue
		}

		// E 15:04:05.000 [route-controller] [default/testroute-xyz] this is my message
		msg := fmt.Sprintf("%s %s [%s] [%s] %s",
			strings.ToUpper(string(line.Level[0])),
			line.Timestamp.Format(timeFormat),
			line.Controller,
			line.Key,
			line.Message)

		if line.Error != "" {
			msg += " err=" + line.Error
		}

		logf(msg)
	}
}

// Start implements streamer
func (k *kubelogs) Start(t *testing.T) Canceler {
	k.once.Do(func() { k.init(t) })

	name := helpers.ObjectPrefixForTest(t)

	// Register a key
	k.m.Lock()
	defer k.m.Unlock()
	k.keys[name] = t.Logf

	// Return a function that unregisters that key.
	return func() {
		k.m.Lock()
		defer k.m.Unlock()
		delete(k.keys, name)

		if k.err != nil {
			t.Errorf("error during logstream: %v", k.err)
		}
	}
}
<|endoftext|>