<commit_before>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/summerwind/h2spec"
)

const VERSION = "v1.4.2"

type sections []string

func (s *sections) String() string {
	return fmt.Sprintf("%v", *s)
}

func (s *sections) Set(v string) error {
	*s = append(*s, v)
	return nil
}

func main() {
	port := flag.Int("p", 0, "Target port.")
	host := flag.String("h", "127.0.0.1", "Target host.")
	useTls := flag.Bool("t", false, "Connect over TLS.")
	insecureSkipVerify := flag.Bool("k", false, "Don't verify server's certificate.")
	timeout := flag.Int("o", 2, "Maximum time allowed for test.")
	strict := flag.Bool("S", false, "Strict mode.")
	junit := flag.String("j", "", "Create test report also in JUnit format.")
	version := flag.Bool("version", false, "Display version information and exit.")

	var sectionFlag sections
	flag.Var(&sectionFlag, "s", "Section number on which to run the test")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS]\n\n", os.Args[0])
		fmt.Println("Options:")
		fmt.Println("  -p:        Target port. (Default: 80 or 443)")
		fmt.Println("  -h:        Target host. (Default: 127.0.0.1)")
		fmt.Println("  -t:        Connect over TLS. (Default: false)")
		fmt.Println("  -k:        Don't verify server's certificate. (Default: false)")
		fmt.Println("  -o:        Maximum time allowed for test. (Default: 2)")
		fmt.Println("  -s:        Section number on which to run the test. (Example: -s 6.1 -s 6.2)")
		fmt.Println("  -S:        Run the test cases marked as \"strict\".")
		fmt.Println("  -j:        Creates report also in JUnit format into specified file.")
		fmt.Println("  --version: Display version information and exit.")
		fmt.Println("  --help:    Display this help and exit.")
		os.Exit(1)
	}

	flag.Parse()

	if *version {
		fmt.Fprintf(os.Stderr, "h2spec %s\n", VERSION)
		os.Exit(0)
	}

	if *port == 0 {
		if *useTls {
			*port = 443
		} else {
			*port = 80
		}
	}

	var ctx h2spec.Context
	ctx.Port = *port
	ctx.Host = *host
	ctx.Timeout = time.Duration(*timeout) * time.Second
	ctx.Strict = *strict
	ctx.Junit = *junit
	ctx.Tls = *useTls
	ctx.TlsConfig = &tls.Config{
		InsecureSkipVerify: *insecureSkipVerify,
	}

	if len(sectionFlag) > 0 {
		ctx.Sections = map[string]bool{}
		for _, sec := range sectionFlag {
			splitedSec := strings.Split(sec, ".")
			lastIndex := len(splitedSec) - 1

			num := []string{}
			for i, sec := range splitedSec {
				num = append(num, sec)
				if i != 0 {
					key := strings.Join(num, ".")

					runAll := false
					if i == lastIndex || ctx.Sections[key] {
						runAll = true
					}

					ctx.Sections[key] = runAll
				}
			}
		}
	}

	if !h2spec.Run(&ctx) {
		os.Exit(1)
	}
}
<commit_msg>Bump version<commit_after>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/summerwind/h2spec"
)

const VERSION = "v1.5.0"

type sections []string

func (s *sections) String() string {
	return fmt.Sprintf("%v", *s)
}

func (s *sections) Set(v string) error {
	*s = append(*s, v)
	return nil
}

func main() {
	port := flag.Int("p", 0, "Target port.")
	host := flag.String("h", "127.0.0.1", "Target host.")
	useTls := flag.Bool("t", false, "Connect over TLS.")
	insecureSkipVerify := flag.Bool("k", false, "Don't verify server's certificate.")
	timeout := flag.Int("o", 2, "Maximum time allowed for test.")
	strict := flag.Bool("S", false, "Strict mode.")
	junit := flag.String("j", "", "Create test report also in JUnit format.")
	version := flag.Bool("version", false, "Display version information and exit.")

	var sectionFlag sections
	flag.Var(&sectionFlag, "s", "Section number on which to run the test")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS]\n\n", os.Args[0])
		fmt.Println("Options:")
		fmt.Println("  -p:        Target port. (Default: 80 or 443)")
		fmt.Println("  -h:        Target host. (Default: 127.0.0.1)")
		fmt.Println("  -t:        Connect over TLS. (Default: false)")
		fmt.Println("  -k:        Don't verify server's certificate. (Default: false)")
		fmt.Println("  -o:        Maximum time allowed for test. (Default: 2)")
		fmt.Println("  -s:        Section number on which to run the test. (Example: -s 6.1 -s 6.2)")
		fmt.Println("  -S:        Run the test cases marked as \"strict\".")
		fmt.Println("  -j:        Creates report also in JUnit format into specified file.")
		fmt.Println("  --version: Display version information and exit.")
		fmt.Println("  --help:    Display this help and exit.")
		os.Exit(1)
	}

	flag.Parse()

	if *version {
		fmt.Fprintf(os.Stderr, "h2spec %s\n", VERSION)
		os.Exit(0)
	}

	if *port == 0 {
		if *useTls {
			*port = 443
		} else {
			*port = 80
		}
	}

	var ctx h2spec.Context
	ctx.Port = *port
	ctx.Host = *host
	ctx.Timeout = time.Duration(*timeout) * time.Second
	ctx.Strict = *strict
	ctx.Junit = *junit
	ctx.Tls = *useTls
	ctx.TlsConfig = &tls.Config{
		InsecureSkipVerify: *insecureSkipVerify,
	}

	if len(sectionFlag) > 0 {
		ctx.Sections = map[string]bool{}
		for _, sec := range sectionFlag {
			splitedSec := strings.Split(sec, ".")
			lastIndex := len(splitedSec) - 1

			num := []string{}
			for i, sec := range splitedSec {
				num = append(num, sec)
				if i != 0 {
					key := strings.Join(num, ".")

					runAll := false
					if i == lastIndex || ctx.Sections[key] {
						runAll = true
					}

					ctx.Sections[key] = runAll
				}
			}
		}
	}

	if !h2spec.Run(&ctx) {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"io"

	pb "github.com/qjcg/horeb/proto"

	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

func getRuneStream(client pb.HorebClient, rr *pb.RuneRequest) {
	grpclog.Printf("Getting rune stream via RuneRequest: %v", rr)

	stream, err := client.GetStream(context.Background(), rr)
	if err != nil {
		grpclog.Fatalf("%v.GetStream(_) = _, %v", client, err)
	}

	for {
		streamedRune, err := stream.Recv()
		if err == io.EOF {
			break
		}
		if err != nil {
			grpclog.Fatal("stream receive error: %v", err)
		}
		grpclog.Printf("%v", streamedRune.R)
	}
}

func main() {
	num := flag.Int("n", 100, "number of runes to request")
	flag.Parse()

	var opts []grpc.DialOption
	opts = append(opts, grpc.WithInsecure())
	conn, err := grpc.Dial(":9999", opts...)
	if err != nil {
		grpclog.Fatalf("fail to dial: %v", err)
	}
	defer conn.Close()
	client := pb.NewHorebClient(conn)

	getRuneStream(
		client,
		&pb.RuneRequest{Num: int32(*num)},
	)
}
<commit_msg>Tweak log messages<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"io"

	pb "github.com/qjcg/horeb/proto"

	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

func getRuneStream(client pb.HorebClient, rr *pb.RuneRequest) {
	grpclog.Printf("Sent: %#v", rr)

	stream, err := client.GetStream(context.Background(), rr)
	if err != nil {
		grpclog.Fatalf("%v.GetStream(_) = _, %v", client, err)
	}

	for {
		streamedRune, err := stream.Recv()
		if err == io.EOF {
			break
		}
		if err != nil {
			grpclog.Fatal("stream receive error: %v", err)
		}
		grpclog.Printf("Got: %#v", streamedRune)
	}
}

func main() {
	num := flag.Int("n", 100, "number of runes to request")
	flag.Parse()

	var opts []grpc.DialOption
	opts = append(opts, grpc.WithInsecure())
	conn, err := grpc.Dial(":9999", opts...)
	if err != nil {
		grpclog.Fatalf("fail to dial: %v", err)
	}
	defer conn.Close()
	client := pb.NewHorebClient(conn)

	getRuneStream(
		client,
		&pb.RuneRequest{Num: int32(*num)},
	)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"os"
	"os/signal"
	"syscall"
)

// ContextWithSignal create a context cancelled when SIGINT or SIGTERM are notified
func ContextWithSignal(ctx context.Context) context.Context {
	newCtx, cancel := context.WithCancel(ctx)
	signals := make(chan os.Signal)
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		select {
		case <-signals:
			cancel()
		}
	}()
	return newCtx
}
<commit_msg>Closing the channel<commit_after>package main

import (
	"context"
	"os"
	"os/signal"
	"syscall"
)

// ContextWithSignal create a context cancelled when SIGINT or SIGTERM are notified
func ContextWithSignal(ctx context.Context) context.Context {
	newCtx, cancel := context.WithCancel(ctx)
	signals := make(chan os.Signal)
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		select {
		case <-signals:
			cancel()
			close(signals)
		}
	}()
	return newCtx
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"

	"github.com/digitalocean/logtalez"
)

func main() {

	endpointsPtr := flag.String("endpoints", "", "comma delimited list of zeromq endpoints")
	hostsPtr := flag.String("hosts", "", "comma delimited list of hostnames to get logs from")
	programsPtr := flag.String("programs", "", "comma delimited list of programs to get logs from")
	serverCertPathPtr := flag.String("servercertpath", "", "path to server public cert")
	clientCertPathPtr := flag.String("clientcertpath", "", "path to client public cert")

	flag.Parse()

	if *endpointsPtr == "" {
		log.Fatal("--endpoints is mandatory")
	}

	if *serverCertPathPtr == "" {
		log.Fatal("--servercertpath is mandatory")
	}

	if *clientCertPathPtr == "" {
		log.Fatal("--clientcertpath is mandatory")
	}

	if _, err := os.Stat(*serverCertPathPtr); err != nil {
		log.Fatalf("error reading server certificate %q: %s", *serverCertPathPtr, err)
	}

	if _, err := os.Stat(*clientCertPathPtr); err != nil {
		log.Fatalf("error reading client certificate %q: %s", *clientCertPathPtr, err)
	}

	topicList := logtalez.MakeTopicList(*hostsPtr, *programsPtr)
	endpointList := logtalez.MakeEndpointList(*endpointsPtr)

	lt, err := logtalez.New(endpointList, topicList, *serverCertPathPtr, *clientCertPathPtr)
	if err != nil {
		log.Fatal(err)
	}
	defer lt.Destroy()

	buf := make([]byte, 65536)
	for {
		n, err := lt.Read(buf)
		if err != nil && err != io.EOF {
			panic(err)
		}
		fmt.Println(string(buf[:n]))
	}
}
<commit_msg>Adds -json flag which limits output to valid JSON.<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"github.com/digitalocean/logtalez"
)

func main() {

	endpointsPtr := flag.String("endpoints", "", "comma delimited list of zeromq endpoints")
	hostsPtr := flag.String("hosts", "", "comma delimited list of hostnames to get logs from")
	programsPtr := flag.String("programs", "", "comma delimited list of programs to get logs from")
	serverCertPathPtr := flag.String("servercertpath", "", "path to server public cert")
	clientCertPathPtr := flag.String("clientcertpath", "", "path to client public cert")
	jSONPtr := flag.Bool("json", false, "restrict output to valid JSON")

	flag.Parse()

	if *endpointsPtr == "" {
		log.Fatal("--endpoints is mandatory")
	}

	if *serverCertPathPtr == "" {
		log.Fatal("--servercertpath is mandatory")
	}

	if *clientCertPathPtr == "" {
		log.Fatal("--clientcertpath is mandatory")
	}

	if _, err := os.Stat(*serverCertPathPtr); err != nil {
		log.Fatalf("error reading server certificate %q: %s", *serverCertPathPtr, err)
	}

	if _, err := os.Stat(*clientCertPathPtr); err != nil {
		log.Fatalf("error reading client certificate %q: %s", *clientCertPathPtr, err)
	}

	topicList := logtalez.MakeTopicList(*hostsPtr, *programsPtr)
	endpointList := logtalez.MakeEndpointList(*endpointsPtr)

	lt, err := logtalez.New(endpointList, topicList, *serverCertPathPtr, *clientCertPathPtr)
	if err != nil {
		log.Fatal(err)
	}
	defer lt.Destroy()

	buf := make([]byte, 65536)
	for {
		n, err := lt.Read(buf)
		if err != nil && err != io.EOF {
			panic(err)
		}

		output := string(buf[:n])
		if *jSONPtr {
			output = strings.SplitAfterN(output, "@cee:", 2)[1]
		}

		fmt.Println(output)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"testing"
	"time"

	. "github.com/smartystreets/goconvey/convey"

	"go.polydawn.net/go-timeless-api/rio"
	"go.polydawn.net/rio/fs"
	"go.polydawn.net/rio/fs/osfs"
	"go.polydawn.net/rio/fsOp"
	"go.polydawn.net/rio/testutil"
)

func stdBuffers() (stdin, stdout, stderr *bytes.Buffer) {
	return &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}
}

type unpackTest struct {
	Name           string
	Args           []string
	ExpectedExit   int
	ExpectedStdout string
	ExpectedStderr string
}

func TestWithoutArgs(t *testing.T) {
	Convey("rio: usage printed to stderr", t, func() {
		args := []string{"rio"}
		stdin, stdout, stderr := stdBuffers()

		ctx := context.Background()
		exitCode := Main(ctx, args, stdin, stdout, stderr)
		So(string(stdout.Bytes()), ShouldBeBlank)
		So(string(stderr.Bytes()), ShouldNotBeBlank)
		firstLine, err := stderr.ReadString('\n')
		So(err, ShouldBeNil)
		So(string(firstLine), ShouldContainSubstring, "usage: rio [<flags>] <command> [<args> ...]")
		So(string(stderr.Bytes()), ShouldNotContainSubstring, "usage: rio [<flags>] <command> [<args> ...]")
		So(exitCode, ShouldEqual, rio.ExitCodeForCategory(rio.ErrUsage))
	})
}

func TestUnpackBogusFlag(t *testing.T) {
	Convey("rio: usage printed to stderr", t, func() {
		args := []string{"rio", "unpack", "--bogus"}
		stdin, stdout, stderr := stdBuffers()

		ctx := context.Background()
		exitCode := Main(ctx, args, stdin, stdout, stderr)
		So(string(stdout.Bytes()), ShouldBeBlank)
		So(string(stderr.Bytes()), ShouldResemble, "error parsing args: unknown long flag '--bogus'\n")
		So(exitCode, ShouldEqual, rio.ExitCodeForCategory(rio.ErrUsage))
	})
}

/*
	Tests against pre-generated, known fixtures of tar binary blobs.

	These tests allow us to cover compat with other tar impls, compression, etc.
*/
func TestTarFixtureUnpack(t *testing.T) {
	Convey("rio: unpacking of tar fixtures", t,
		testutil.Requires(testutil.RequiresCanManageOwnership, func() {
			testutil.WithTmpdir(func(tmpDir fs.AbsolutePath) {
				Convey("Unpack a fixture from gnu tar which includes a base dir", func() {
					ctx := context.Background()
					wareID := "tar:5y6NvK6GBPQ6CcuNyJyWtSrMAJQ4LVrAcZSoCRAzMSk5o53pkTYiieWyRivfvhZwhZ"
					source := "file://../../transmat/tar/fixtures/tar_withBase.tgz"
					for _, fixture := range []unpackTest{
						{"UnpackBasic",
							[]string{
								"rio",
								"unpack",
								tmpDir.String(),
								wareID,
								"--uid=keep", "--gid=keep",
								fmt.Sprintf("--placer=%s", rio.Placement_Direct),
								fmt.Sprintf("--source=%s", source),
							},
							0,
							wareID + "\n",
							"",
						},
						{"UnpackJsonFormat",
							[]string{
								"rio",
								"unpack",
								tmpDir.String(),
								wareID,
								"--uid=keep", "--gid=keep",
								fmt.Sprintf("--placer=%s", rio.Placement_Direct),
								fmt.Sprintf("--source=%s", source),
								fmt.Sprintf("--format=%s", format_Json),
							},
							0,
							fmt.Sprintf(`{"log":null,"prog":null,"result":{"wareID":"%s","error":null}}`, wareID) + "\n",
							"",
						},
					} {
						Convey(fmt.Sprintf("- test %q", fixture.Name), func() {
							stdin, stdout, stderr := stdBuffers()
							exitCode := Main(ctx, fixture.Args, stdin, stdout, stderr)
							So(string(stdout.Bytes()), ShouldEqual, fixture.ExpectedStdout)
							So(string(stderr.Bytes()), ShouldEqual, fixture.ExpectedStderr)
							So(exitCode, ShouldEqual, fixture.ExpectedExit)
							if fixture.ExpectedExit != 0 {
								Convey("The filesystem should not have things", func() {
									afs := osfs.New(tmpDir)
									_, err := afs.LStat(fs.MustRelPath("."))
									So(err, ShouldNotBeNil)
								})
							} else {
								Convey("The filesystem contains the correct unpacked fixture", func() {
									var err error
									fmeta, reader, err := fsOp.ScanFile(osfs.New(tmpDir), fs.MustRelPath("ab"))
									So(err, ShouldBeNil)
									So(fmeta.Name, ShouldResemble, fs.MustRelPath("ab"))
									So(fmeta.Type, ShouldResemble, fs.Type_File)
									So(fmeta.Uid, ShouldEqual, 7000)
									So(fmeta.Gid, ShouldEqual, 7000)
									So(fmeta.Mtime.UTC(), ShouldResemble, time.Date(2015, 05, 30, 19, 53, 35, 0, time.UTC))
									body, err := ioutil.ReadAll(reader)
									So(string(body), ShouldResemble, "")

									fmeta, reader, err = fsOp.ScanFile(osfs.New(tmpDir), fs.MustRelPath("bc"))
									So(err, ShouldBeNil)
									So(fmeta.Name, ShouldResemble, fs.MustRelPath("bc"))
									So(fmeta.Type, ShouldResemble, fs.Type_Dir)
									So(fmeta.Mtime.UTC(), ShouldResemble, time.Date(2015, 05, 30, 19, 53, 35, 0, time.UTC))
									So(reader, ShouldBeNil)

									fmeta, reader, err = fsOp.ScanFile(osfs.New(tmpDir), fs.MustRelPath("."))
									So(err, ShouldBeNil)
									So(fmeta.Name, ShouldResemble, fs.MustRelPath("."))
									So(fmeta.Type, ShouldResemble, fs.Type_Dir)
									So(fmeta.Mtime.UTC(), ShouldResemble, time.Date(2015, 05, 30, 19, 53, 35, 0, time.UTC))
									So(reader, ShouldBeNil)
								})
							}
						})
					}

				})
			})
		}),
	)
}
<commit_msg>un-fragilize main tests for log messages.<commit_after>package main

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"strings"
	"testing"
	"time"

	. "github.com/smartystreets/goconvey/convey"

	"go.polydawn.net/go-timeless-api/rio"
	"go.polydawn.net/rio/fs"
	"go.polydawn.net/rio/fs/osfs"
	"go.polydawn.net/rio/fsOp"
	"go.polydawn.net/rio/testutil"
)

func stdBuffers() (stdin, stdout, stderr *bytes.Buffer) {
	return &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}
}

type unpackTest struct {
	Name           string
	Args           []string
	ExpectedExit   int
	ExpectedStdout string
	ExpectedStderr string
}

func TestWithoutArgs(t *testing.T) {
	Convey("rio: usage printed to stderr", t, func() {
		args := []string{"rio"}
		stdin, stdout, stderr := stdBuffers()

		ctx := context.Background()
		exitCode := Main(ctx, args, stdin, stdout, stderr)
		So(string(stdout.Bytes()), ShouldBeBlank)
		So(string(stderr.Bytes()), ShouldNotBeBlank)
		firstLine, err := stderr.ReadString('\n')
		So(err, ShouldBeNil)
		So(string(firstLine), ShouldContainSubstring, "usage: rio [<flags>] <command> [<args> ...]")
		So(string(stderr.Bytes()), ShouldNotContainSubstring, "usage: rio [<flags>] <command> [<args> ...]")
		So(exitCode, ShouldEqual, rio.ExitCodeForCategory(rio.ErrUsage))
	})
}

func TestUnpackBogusFlag(t *testing.T) {
	Convey("rio: usage printed to stderr", t, func() {
		args := []string{"rio", "unpack", "--bogus"}
		stdin, stdout, stderr := stdBuffers()

		ctx := context.Background()
		exitCode := Main(ctx, args, stdin, stdout, stderr)
		So(string(stdout.Bytes()), ShouldBeBlank)
		So(string(stderr.Bytes()), ShouldResemble, "error parsing args: unknown long flag '--bogus'\n")
		So(exitCode, ShouldEqual, rio.ExitCodeForCategory(rio.ErrUsage))
	})
}

/*
	Tests against pre-generated, known fixtures of tar binary blobs.

	These tests allow us to cover compat with other tar impls, compression, etc.
*/
func TestTarFixtureUnpack(t *testing.T) {
	Convey("rio: unpacking of tar fixtures", t,
		testutil.Requires(testutil.RequiresCanManageOwnership, func() {
			testutil.WithTmpdir(func(tmpDir fs.AbsolutePath) {
				Convey("Unpack a fixture from gnu tar which includes a base dir", func() {
					ctx := context.Background()
					wareID := "tar:5y6NvK6GBPQ6CcuNyJyWtSrMAJQ4LVrAcZSoCRAzMSk5o53pkTYiieWyRivfvhZwhZ"
					source := "file://../../transmat/tar/fixtures/tar_withBase.tgz"
					for _, fixture := range []unpackTest{
						{"UnpackBasic",
							[]string{
								"rio",
								"unpack",
								tmpDir.String(),
								wareID,
								"--uid=keep", "--gid=keep",
								fmt.Sprintf("--placer=%s", rio.Placement_Direct),
								fmt.Sprintf("--source=%s", source),
							},
							0,
							wareID,
							fmt.Sprintf(`log: lvl=info msg=read for ware "%s" opened from warehouse "%s"`, wareID, source),
						},
						{"UnpackJsonFormat",
							[]string{
								"rio",
								"unpack",
								tmpDir.String(),
								wareID,
								"--uid=keep", "--gid=keep",
								fmt.Sprintf("--placer=%s", rio.Placement_Direct),
								fmt.Sprintf("--source=%s", source),
								fmt.Sprintf("--format=%s", format_Json),
							},
							0,
							fmt.Sprintf(`{"log":null,"prog":null,"result":{"wareID":"%s","error":null}}`, wareID),
							"",
						},
					} {
						Convey(fmt.Sprintf("- test %q", fixture.Name), func() {
							stdin, stdout, stderr := stdBuffers()
							exitCode := Main(ctx, fixture.Args, stdin, stdout, stderr)
							So(lastLine(string(stdout.Bytes())), ShouldEqual, fixture.ExpectedStdout)
							So(lastLine(string(stderr.Bytes())), ShouldEqual, fixture.ExpectedStderr)
							So(exitCode, ShouldEqual, fixture.ExpectedExit)
							if fixture.ExpectedExit != 0 {
								Convey("The filesystem should not have things", func() {
									afs := osfs.New(tmpDir)
									_, err := afs.LStat(fs.MustRelPath("."))
									So(err, ShouldNotBeNil)
								})
							} else {
								Convey("The filesystem contains the correct unpacked fixture", func() {
									var err error
									fmeta, reader, err := fsOp.ScanFile(osfs.New(tmpDir), fs.MustRelPath("ab"))
									So(err, ShouldBeNil)
									So(fmeta.Name, ShouldResemble, fs.MustRelPath("ab"))
									So(fmeta.Type, ShouldResemble, fs.Type_File)
									So(fmeta.Uid, ShouldEqual, 7000)
									So(fmeta.Gid, ShouldEqual, 7000)
									So(fmeta.Mtime.UTC(), ShouldResemble, time.Date(2015, 05, 30, 19, 53, 35, 0, time.UTC))
									body, err := ioutil.ReadAll(reader)
									So(string(body), ShouldResemble, "")

									fmeta, reader, err = fsOp.ScanFile(osfs.New(tmpDir), fs.MustRelPath("bc"))
									So(err, ShouldBeNil)
									So(fmeta.Name, ShouldResemble, fs.MustRelPath("bc"))
									So(fmeta.Type, ShouldResemble, fs.Type_Dir)
									So(fmeta.Mtime.UTC(), ShouldResemble, time.Date(2015, 05, 30, 19, 53, 35, 0, time.UTC))
									So(reader, ShouldBeNil)

									fmeta, reader, err = fsOp.ScanFile(osfs.New(tmpDir), fs.MustRelPath("."))
									So(err, ShouldBeNil)
									So(fmeta.Name, ShouldResemble, fs.MustRelPath("."))
									So(fmeta.Type, ShouldResemble, fs.Type_Dir)
									So(fmeta.Mtime.UTC(), ShouldResemble, time.Date(2015, 05, 30, 19, 53, 35, 0, time.UTC))
									So(reader, ShouldBeNil)
								})
							}
						})
					}

				})
			})
		}),
	)
}

func lastLine(str string) string {
	str = strings.TrimRight(str, "\n")
	ss := strings.Split(str, "\n")
	fmt.Printf("::goddamnit \n\t%#v\n", ss)
	return ss[len(ss)-1]
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/alecthomas/kingpin"
	"github.com/sirupsen/logrus"
	"github.com/versent/saml2aws/cmd/saml2aws/commands"
	"github.com/versent/saml2aws/pkg/flags"
)

var (
	// Version app version
	Version = "1.0.0"
)

// The `cmdLineList` type is used to make a `[]string` meet the requiements
// of the kingpin.Value interface
type cmdLineList []string

func (i *cmdLineList) Set(value string) error {
	*i = append(*i, value)

	return nil
}

func (i *cmdLineList) String() string {
	return ""
}

func (i *cmdLineList) IsCumulative() bool {
	return true
}

func buildCmdList(s kingpin.Settings) (target *[]string) {
	target = new([]string)
	s.SetValue((*cmdLineList)(target))
	return
}

func main() {

	app := kingpin.New("saml2aws", "A command line tool to help with SAML access to the AWS token service.")
	app.Version(Version)

	// Settings not related to commands
	verbose := app.Flag("verbose", "Enable verbose logging").Bool()
	provider := app.Flag("provider", "This flag it is obsolete see https://github.com/Versent/saml2aws#adding-idp-accounts.").Short('i').Enum("ADFS", "ADFS2", "Ping", "JumpCloud", "Okta", "OneLogin", "KeyCloak")

	// Common (to all commands) settings
	commonFlags := new(flags.CommonFlags)
	app.Flag("idp-account", "The name of the configured IDP account").Short('a').Default("default").StringVar(&commonFlags.IdpAccount)
	app.Flag("idp-provider", "The configured IDP provider").EnumVar(&commonFlags.IdpProvider, "ADFS", "ADFS2", "Ping", "JumpCloud", "Okta", "OneLogin", "KeyCloak")
	app.Flag("mfa", "The name of the mfa").EnumVar(&commonFlags.MFA, "Auto", "VIP")
	app.Flag("skip-verify", "Skip verification of server certificate.").Short('s').BoolVar(&commonFlags.SkipVerify)
	app.Flag("url", "The URL of the SAML IDP server used to login.").StringVar(&commonFlags.URL)
	app.Flag("username", "The username used to login.").Envar("SAML2AWS_USERNAME").StringVar(&commonFlags.Username)
	app.Flag("password", "The password used to login.").Envar("SAML2AWS_PASSWORD").StringVar(&commonFlags.Password)
	app.Flag("mfa-token", "The current MFA token (supported in Keycloak, ADFS).").Envar("SAML2AWS_MFA_TOKEN").StringVar(&commonFlags.MFAToken)
	app.Flag("role", "The ARN of the role to assume.").StringVar(&commonFlags.RoleArn)
	app.Flag("aws-urn", "The URN used by SAML when you login.").StringVar(&commonFlags.AmazonWebservicesURN)
	app.Flag("skip-prompt", "Skip prompting for parameters during login.").BoolVar(&commonFlags.SkipPrompt)
	app.Flag("session-duration", "The duration of your AWS Session.").IntVar(&commonFlags.SessionDuration)

	// `configure` command and settings
	cmdConfigure := app.Command("configure", "Configure a new IDP account.")
	cmdConfigure.Flag("app-id", "OneLogin app id required for SAML assertion.").Envar("ONELOGIN_APP_ID").StringVar(&commonFlags.AppID)
	cmdConfigure.Flag("client-id", "OneLogin client id, used to generate API access token.").Envar("ONELOGIN_CLIENT_ID").StringVar(&commonFlags.ClientID)
	cmdConfigure.Flag("client-secret", "OneLogin client secret, used to generate API access token.").Envar("ONELOGIN_CLIENT_SECRET").StringVar(&commonFlags.ClientSecret)
	cmdConfigure.Flag("subdomain", "OneLogin subdomain of your company account.").Envar("ONELOGIN_SUBDOMAIN").StringVar(&commonFlags.Subdomain)
	configFlags := commonFlags

	// `login` command and settings
	cmdLogin := app.Command("login", "Login to a SAML 2.0 IDP and convert the SAML assertion to an STS token.")
	loginFlags := new(flags.LoginExecFlags)
	loginFlags.CommonFlags = commonFlags
	cmdLogin.Flag("profile", "The AWS profile to save the temporary credentials").Short('p').StringVar(&commonFlags.Profile)

	// `exec` command and settings
	cmdExec := app.Command("exec", "Exec the supplied command with env vars from STS token.")
	execFlags := new(flags.LoginExecFlags)
	execFlags.CommonFlags = commonFlags
	cmdExec.Flag("profile", "The AWS profile to save the temporary credentials").Short('p').StringVar(&commonFlags.Profile)
	cmdLine := buildCmdList(cmdExec.Arg("command", "The command to execute."))

	// `list` command and settings
	cmdListRoles := app.Command("list-roles", "List available role ARNs.")
	listRolesFlags := new(flags.LoginExecFlags)
	listRolesFlags.CommonFlags = commonFlags

	// Trigger the parsing of the command line inputs via kingpin
	command := kingpin.MustParse(app.Parse(os.Args[1:]))

	// will leave this here for a while during upgrade process
	if *provider != "" {
		fmt.Println("The --provider flag has been replaced with a new configure command. See https://github.com/Versent/saml2aws#adding-idp-accounts")
		os.Exit(1)
	}

	if *verbose {
		logrus.SetLevel(logrus.DebugLevel)
	}

	logrus.WithField("command", command).Debug("Running")

	var err error
	switch command {
	case cmdLogin.FullCommand():
		err = commands.Login(loginFlags)
	case cmdExec.FullCommand():
		err = commands.Exec(execFlags, *cmdLine)
	case cmdListRoles.FullCommand():
		err = commands.ListRoles(listRolesFlags)
	case cmdConfigure.FullCommand():
		err = commands.Configure(configFlags)
	}

	if err != nil {
		fmt.Printf("%+v\n", err)
		os.Exit(1)
	}
}
<commit_msg>Display stack trace only when verbose flag is set<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/alecthomas/kingpin"
	"github.com/sirupsen/logrus"
	"github.com/versent/saml2aws/cmd/saml2aws/commands"
	"github.com/versent/saml2aws/pkg/flags"
)

var (
	// Version app version
	Version = "1.0.0"
)

// The `cmdLineList` type is used to make a `[]string` meet the requiements
// of the kingpin.Value interface
type cmdLineList []string

func (i *cmdLineList) Set(value string) error {
	*i = append(*i, value)

	return nil
}

func (i *cmdLineList) String() string {
	return ""
}

func (i *cmdLineList) IsCumulative() bool {
	return true
}

func buildCmdList(s kingpin.Settings) (target *[]string) {
	target = new([]string)
	s.SetValue((*cmdLineList)(target))
	return
}

func main() {

	app := kingpin.New("saml2aws", "A command line tool to help with SAML access to the AWS token service.")
	app.Version(Version)

	// Settings not related to commands
	verbose := app.Flag("verbose", "Enable verbose logging").Bool()
	provider := app.Flag("provider", "This flag it is obsolete see https://github.com/Versent/saml2aws#adding-idp-accounts.").Short('i').Enum("ADFS", "ADFS2", "Ping", "JumpCloud", "Okta", "OneLogin", "KeyCloak")

	// Common (to all commands) settings
	commonFlags := new(flags.CommonFlags)
	app.Flag("idp-account", "The name of the configured IDP account").Short('a').Default("default").StringVar(&commonFlags.IdpAccount)
	app.Flag("idp-provider", "The configured IDP provider").EnumVar(&commonFlags.IdpProvider, "ADFS", "ADFS2", "Ping", "JumpCloud", "Okta", "OneLogin", "KeyCloak")
	app.Flag("mfa", "The name of the mfa").EnumVar(&commonFlags.MFA, "Auto", "VIP")
	app.Flag("skip-verify", "Skip verification of server certificate.").Short('s').BoolVar(&commonFlags.SkipVerify)
	app.Flag("url", "The URL of the SAML IDP server used to login.").StringVar(&commonFlags.URL)
	app.Flag("username", "The username used to login.").Envar("SAML2AWS_USERNAME").StringVar(&commonFlags.Username)
	app.Flag("password", "The password used to login.").Envar("SAML2AWS_PASSWORD").StringVar(&commonFlags.Password)
	app.Flag("mfa-token", "The current MFA token (supported in Keycloak, ADFS).").Envar("SAML2AWS_MFA_TOKEN").StringVar(&commonFlags.MFAToken)
	app.Flag("role", "The ARN of the role to assume.").StringVar(&commonFlags.RoleArn)
	app.Flag("aws-urn", "The URN used by SAML when you login.").StringVar(&commonFlags.AmazonWebservicesURN)
	app.Flag("skip-prompt", "Skip prompting for parameters during login.").BoolVar(&commonFlags.SkipPrompt)
	app.Flag("session-duration", "The duration of your AWS Session.").IntVar(&commonFlags.SessionDuration)

	// `configure` command and settings
	cmdConfigure := app.Command("configure", "Configure a new IDP account.")
	cmdConfigure.Flag("app-id", "OneLogin app id required for SAML assertion.").Envar("ONELOGIN_APP_ID").StringVar(&commonFlags.AppID)
	cmdConfigure.Flag("client-id", "OneLogin client id, used to generate API access token.").Envar("ONELOGIN_CLIENT_ID").StringVar(&commonFlags.ClientID)
	cmdConfigure.Flag("client-secret", "OneLogin client secret, used to generate API access token.").Envar("ONELOGIN_CLIENT_SECRET").StringVar(&commonFlags.ClientSecret)
	cmdConfigure.Flag("subdomain", "OneLogin subdomain of your company account.").Envar("ONELOGIN_SUBDOMAIN").StringVar(&commonFlags.Subdomain)
	configFlags := commonFlags

	// `login` command and settings
	cmdLogin := app.Command("login", "Login to a SAML 2.0 IDP and convert the SAML assertion to an STS token.")
	loginFlags := new(flags.LoginExecFlags)
	loginFlags.CommonFlags = commonFlags
	cmdLogin.Flag("profile", "The AWS profile to save the temporary credentials").Short('p').StringVar(&commonFlags.Profile)

	// `exec` command and settings
	cmdExec := app.Command("exec", "Exec the supplied command with env vars from STS token.")
	execFlags := new(flags.LoginExecFlags)
	execFlags.CommonFlags = commonFlags
	cmdExec.Flag("profile", "The AWS profile to save the temporary credentials").Short('p').StringVar(&commonFlags.Profile)
	cmdLine := buildCmdList(cmdExec.Arg("command", "The command to execute."))

	// `list` command and settings
	cmdListRoles := app.Command("list-roles", "List available role ARNs.")
	listRolesFlags := new(flags.LoginExecFlags)
	listRolesFlags.CommonFlags = commonFlags

	// Trigger the parsing of the command line inputs via kingpin
	command := kingpin.MustParse(app.Parse(os.Args[1:]))

	// will leave this here for a while during upgrade process
	if *provider != "" {
		fmt.Println("The --provider flag has been replaced with a new configure command. See https://github.com/Versent/saml2aws#adding-idp-accounts")
		os.Exit(1)
	}

	errtpl := "%v\n"
	if *verbose {
		logrus.SetLevel(logrus.DebugLevel)
		errtpl = "%+v\n"
	}

	logrus.WithField("command", command).Debug("Running")

	var err error
	switch command {
	case cmdLogin.FullCommand():
		err = commands.Login(loginFlags)
	case cmdExec.FullCommand():
		err = commands.Exec(execFlags, *cmdLine)
	case cmdListRoles.FullCommand():
		err = commands.ListRoles(listRolesFlags)
	case cmdConfigure.FullCommand():
		err = commands.Configure(configFlags)
	}

	if err != nil {
		fmt.Printf(errtpl, err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

// Package utils contains internal helper functions for go-ethereum commands.
package utils

import (
	"fmt"
	"runtime"

	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/logger/glog"
	"github.com/ethereum/go-ethereum/params"
	"github.com/ethereum/go-ethereum/rlp"
)

const (
	VersionMajor = 1        // Major version component of the current release
	VersionMinor = 5        // Minor version component of the current release
	VersionPatch = 0        // Patch version component of the current release
	VersionMeta  = "stable" // Version metadata to append to the version string
)

// Version holds the textual version string.
var Version = func() string {
	v := fmt.Sprintf("%d.%d.%d", VersionMajor, VersionMinor, VersionPatch)
	if VersionMeta != "" {
		v += "-" + VersionMeta
	}
	return v
}()

// MakeDefaultExtraData returns the default Ethereum block extra data blob.
func MakeDefaultExtraData(clientIdentifier string) []byte {
	var clientInfo = struct {
		Version   uint
		Name      string
		GoVersion string
		Os        string
	}{uint(VersionMajor<<16 | VersionMinor<<8 | VersionPatch), clientIdentifier, runtime.Version(), runtime.GOOS}
	extra, err := rlp.EncodeToBytes(clientInfo)
	if err != nil {
		glog.V(logger.Warn).Infoln("error setting canonical miner information:", err)
	}
	if uint64(len(extra)) > params.MaximumExtraDataSize.Uint64() {
		glog.V(logger.Warn).Infoln("error setting canonical miner information: extra exceeds", params.MaximumExtraDataSize)
		glog.V(logger.Debug).Infof("extra: %x\n", extra)
		return nil
	}
	return extra
}
<commit_msg>utils: unstable 1.5.1<commit_after>// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

// Package utils contains internal helper functions for go-ethereum commands.
package utils

import (
	"fmt"
	"runtime"

	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/logger/glog"
	"github.com/ethereum/go-ethereum/params"
	"github.com/ethereum/go-ethereum/rlp"
)

const (
	VersionMajor = 1          // Major version component of the current release
	VersionMinor = 5          // Minor version component of the current release
	VersionPatch = 1          // Patch version component of the current release
	VersionMeta  = "unstable" // Version metadata to append to the version string
)

// Version holds the textual version string.
var Version = func() string {
	v := fmt.Sprintf("%d.%d.%d", VersionMajor, VersionMinor, VersionPatch)
	if VersionMeta != "" {
		v += "-" + VersionMeta
	}
	return v
}()

// MakeDefaultExtraData returns the default Ethereum block extra data blob.
func MakeDefaultExtraData(clientIdentifier string) []byte {
	var clientInfo = struct {
		Version   uint
		Name      string
		GoVersion string
		Os        string
	}{uint(VersionMajor<<16 | VersionMinor<<8 | VersionPatch), clientIdentifier, runtime.Version(), runtime.GOOS}
	extra, err := rlp.EncodeToBytes(clientInfo)
	if err != nil {
		glog.V(logger.Warn).Infoln("error setting canonical miner information:", err)
	}
	if uint64(len(extra)) > params.MaximumExtraDataSize.Uint64() {
		glog.V(logger.Warn).Infoln("error setting canonical miner information: extra exceeds", params.MaximumExtraDataSize)
		glog.V(logger.Debug).Infof("extra: %x\n", extra)
		return nil
	}
	return extra
}
<|endoftext|>


<commit_before>// Copyright © 2015 Steve Francia <spf@spf13.com>.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"text/template"
)

var projectPath string

var cmdDirs = [...]string{"cmd", "cmds", "command", "commands"}
var srcPaths []string

func init() {
	// Initialize goPaths and srcPaths
	envGoPath := os.Getenv("GOPATH")
	if envGoPath == "" {
		er("$GOPATH is not set")
	}

	goPaths := filepath.SplitList(envGoPath)
	srcPaths = make([]string, 0, len(goPaths))
	for _, goPath := range goPaths {
		srcPaths = append(srcPaths, filepath.Join(goPath, "src"))
	}
}

func er(msg interface{}) {
	fmt.Println("Error:", msg)
	os.Exit(1)
}

// isEmpty checks if a given path is empty.
func isEmpty(path string) bool {
	fi, err := os.Stat(path)
	if err != nil {
		er(err)
	}
	if fi.IsDir() {
		f, err := os.Open(path)
		if err != nil {
			er(err)
		}
		defer f.Close()
		dirs, err := f.Readdirnames(1)
		if err != nil && err != io.EOF {
			er(err)
		}
		return len(dirs) == 0
	}
	return fi.Size() == 0
}

// exists checks if a file or directory exists.
func exists(path string) bool {
	if path == "" {
		return false
	}
	_, err := os.Stat(path)
	if err == nil {
		return true
	}
	if !os.IsNotExist(err) {
		er(err)
	}
	return false
}

var tmpl = template.New("").Funcs(template.FuncMap{"comment": commentifyString})

func executeTemplate(tmplStr string, data interface{}) (string, error) {
	var err error
	tmpl, err = tmpl.Parse(tmplStr)
	if err != nil {
		return "", err
	}

	buf := new(bytes.Buffer)
	err = tmpl.Execute(buf, data)
	return buf.String(), err
}

func writeStringToFile(path string, s string) error {
	return writeToFile(path, strings.NewReader(s))
}

// writeToFile writes r to file with path only
// if file/directory on given path doesn't exist.
// If file/directory exists on given path, then
// it terminates app and prints an appropriate error.
func writeToFile(path string, r io.Reader) error {
	if exists(path) {
		return fmt.Errorf("%v already exists", path)
	}

	dir := filepath.Dir(path)
	if dir != "" {
		if err := os.MkdirAll(dir, 0777); err != nil {
			return err
		}
	}

	file, err := os.Create(path)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = io.Copy(file, r)
	return err
}

// commentfyString comments every line of in.
func commentifyString(in string) string {
	var newlines []string
	lines := strings.Split(in, "\n")
	for _, line := range lines {
		if strings.HasPrefix(line, "//") {
			newlines = append(newlines, line)
		} else {
			if line == "" {
				newlines = append(newlines, "//")
			} else {
				newlines = append(newlines, "// "+line)
			}
		}
	}
	return strings.Join(newlines, "\n")
}
<commit_msg>cmd: Make tmpl local variable<commit_after>// Copyright © 2015 Steve Francia <spf@spf13.com>.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"text/template"
)

var projectPath string

var cmdDirs = [...]string{"cmd", "cmds", "command", "commands"}
var srcPaths []string

func init() {
	// Initialize goPaths and srcPaths
	envGoPath := os.Getenv("GOPATH")
	if envGoPath == "" {
		er("$GOPATH is not set")
	}

	goPaths := filepath.SplitList(envGoPath)
	srcPaths = make([]string, 0, len(goPaths))
	for _, goPath := range goPaths {
		srcPaths = append(srcPaths, filepath.Join(goPath, "src"))
	}
}

func er(msg interface{}) {
	fmt.Println("Error:", msg)
	os.Exit(1)
}

// isEmpty checks if a given path is empty.
func isEmpty(path string) bool {
	fi, err := os.Stat(path)
	if err != nil {
		er(err)
	}
	if fi.IsDir() {
		f, err := os.Open(path)
		if err != nil {
			er(err)
		}
		defer f.Close()
		dirs, err := f.Readdirnames(1)
		if err != nil && err != io.EOF {
			er(err)
		}
		return len(dirs) == 0
	}
	return fi.Size() == 0
}

// exists checks if a file or directory exists.
func exists(path string) bool {
	if path == "" {
		return false
	}
	_, err := os.Stat(path)
	if err == nil {
		return true
	}
	if !os.IsNotExist(err) {
		er(err)
	}
	return false
}

func executeTemplate(tmplStr string, data interface{}) (string, error) {
	tmpl, err := template.New("").Funcs(template.FuncMap{"comment": commentifyString}).Parse(tmplStr)
	if err != nil {
		return "", err
	}

	buf := new(bytes.Buffer)
	err = tmpl.Execute(buf, data)
	return buf.String(), err
}

func writeStringToFile(path string, s string) error {
	return writeToFile(path, strings.NewReader(s))
}

// writeToFile writes r to file with path only
// if file/directory on given path doesn't exist.
// If file/directory exists on given path, then
// it terminates app and prints an appropriate error.
func writeToFile(path string, r io.Reader) error {
	if exists(path) {
		return fmt.Errorf("%v already exists", path)
	}

	dir := filepath.Dir(path)
	if dir != "" {
		if err := os.MkdirAll(dir, 0777); err != nil {
			return err
		}
	}

	file, err := os.Create(path)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = io.Copy(file, r)
	return err
}

// commentfyString comments every line of in.
func commentifyString(in string) string {
	var newlines []string
	lines := strings.Split(in, "\n")
	for _, line := range lines {
		if strings.HasPrefix(line, "//") {
			newlines = append(newlines, line)
		} else {
			if line == "" {
				newlines = append(newlines, "//")
			} else {
				newlines = append(newlines, "// "+line)
			}
		}
	}
	return strings.Join(newlines, "\n")
}
<|endoftext|>


<commit_before>package beater

import "sync"

type DummyRegistry struct {
	entries     map[string]*RegistryItem
	entriesLock *sync.RWMutex
}

func NewDummyRegistry() Registry {
	return &DummyRegistry{
		entries:     make(map[string]*RegistryItem),
		entriesLock: &sync.RWMutex{},
	}
}

func (registry *DummyRegistry) ReadStreamInfo(stream *Stream) error {
	key := generateKey(stream)
	registry.entriesLock.RLock()
	item, ok := registry.entries[key]
	registry.entriesLock.RUnlock()
	if ok {
		stream.Params.NextToken = &item.NextToken
		stream.Buffer.Reset()
		stream.Buffer.WriteString(item.Buffer)
	}
	return nil
}

func (registry *DummyRegistry) WriteStreamInfo(stream *Stream) error {
	item := RegistryItem{
		NextToken: *stream.Params.NextToken,
		Buffer:    stream.Buffer.String(),
	}
	panic("hello")
	key := generateKey(stream)
	registry.entriesLock.Lock()
	registry.entries[key] = &item
	registry.entriesLock.Unlock()
	return nil
}
<commit_msg>Removes panic statement from dummy registry<commit_after>package beater

import "sync"

type DummyRegistry struct {
	entries     map[string]*RegistryItem
	entriesLock *sync.RWMutex
}

func NewDummyRegistry() Registry {
	return &DummyRegistry{
		entries:     make(map[string]*RegistryItem),
		entriesLock: &sync.RWMutex{},
	}
}

func (registry *DummyRegistry) ReadStreamInfo(stream *Stream) error {
	key := generateKey(stream)
	registry.entriesLock.RLock()
	item, ok := registry.entries[key]
	registry.entriesLock.RUnlock()
	if ok {
		stream.Params.NextToken = &item.NextToken
		stream.Buffer.Reset()
		stream.Buffer.WriteString(item.Buffer)
	}
	return nil
}

func (registry *DummyRegistry) WriteStreamInfo(stream *Stream) error {
	item := RegistryItem{
		NextToken: *stream.Params.NextToken,
		Buffer:    stream.Buffer.String(),
	}
	key := generateKey(stream)
	registry.entriesLock.Lock()
	registry.entries[key] = &item
	registry.entriesLock.Unlock()
	return nil
}
<|endoftext|>


<commit_before>//+build linux

package driver

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/hashicorp/nomad/client/config"
	"github.com/hashicorp/nomad/client/fingerprint"
	"github.com/hashicorp/nomad/client/stats"
	"github.com/hashicorp/nomad/helper/fields"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/mitchellh/mapstructure"

	dstructs "github.com/hashicorp/nomad/client/driver/structs"
	cstructs "github.com/hashicorp/nomad/client/structs"
	lxc "gopkg.in/lxc/go-lxc.v2"
)

const (
	containerMonitorIntv = 2 * time.Second
)

var (
	LXCMeasuredCpuStats = []string{"System Mode", "User Mode", "Percent"}
	LXCMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage", "Kernel Usage", "Kernel Max Usage"}
)

// Add the lxc driver to the list of builtin drivers
func init() {
	BuiltinDrivers["lxc"] = NewLxcDriver
}

// LxcDriver allows users to run LXC Containers
type LxcDriver struct {
	DriverContext
	fingerprint.StaticFingerprinter
}

// LxcDriverConfig is the configuration of the LXC Container
type LxcDriverConfig struct {
	Template             string
	Distro               string
	Release              string
	Arch                 string
	ImageVariant         string   "mapstructure:`image_variant`"
	ImageServer          string   "mapstructure:`image_server`"
	GPGKeyID             string   "mapstructure:`gpg_key_id`"
	GPGKeyServer         string   "mapstructure:`gpg_key_server`"
	DisableGPGValidation bool     "mapstructure:`disable_gpg`"
	FlushCache           bool     "mapstructure:`flush_cache`"
	ForceCache           bool     "mapstructure:`force_cache`"
	TemplateArgs         []string "mapstructure:`template_args`"
}

// NewLxcDriver returns a new instance of the LXC driver
func NewLxcDriver(ctx *DriverContext) Driver {
	return &LxcDriver{DriverContext: *ctx}
}

// Validate validates the lxc driver configuration
func (d *LxcDriver) Validate(config map[string]interface{}) error {
	fd := &fields.FieldData{
		Raw: config,
		Schema: map[string]*fields.FieldSchema{
			"template": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: true,
			},
			"distro": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"release": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"arch": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"image_variant": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"image_server": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"gpg_key_id": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"gpg_key_server": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"disable_gpg": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"flush_cache": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"force_cache": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"template_args": &fields.FieldSchema{
				Type:     fields.TypeArray,
				Required: false,
			},
		},
	}

	if err := fd.Validate(); err != nil {
		return err
	}
	return nil
}

// Fingerprint fingerprints the lxc driver configuration
func (d *LxcDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
	version := lxc.Version()
	if version == "" {
		return false, nil
	}
	node.Attributes["driver.lxc.version"] = version
	node.Attributes["driver.lxc"] = "1"
	return true, nil
}

// Start starts the LXC Driver
func (d *LxcDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
	var driverConfig LxcDriverConfig
	if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
		return nil, err
	}
	lxcPath := lxc.DefaultConfigPath()
	if path := d.config.Read("lxc.path"); path != "" {
		lxcPath = path
	}

	containerName := fmt.Sprintf("%s-%s", task.Name, ctx.AllocID)
	c, err := lxc.NewContainer(containerName, lxcPath)
	if err != nil {
		return nil, fmt.Errorf("unable to create container: %v", err)
	}

	// TODO make them driver configuration
	c.SetVerbosity(lxc.Verbose)
	c.SetLogLevel(lxc.TRACE)

	logFile := filepath.Join(ctx.AllocDir.LogDir(), fmt.Sprintf("%v-lxc.log", task.Name))
	c.SetLogFile(logFile)

	options := lxc.TemplateOptions{
		Template:             driverConfig.Template,
		Distro:               driverConfig.Distro,
		Release:              driverConfig.Release,
		Arch:                 driverConfig.Arch,
		FlushCache:           driverConfig.FlushCache,
		DisableGPGValidation: driverConfig.DisableGPGValidation,
	}

	if err := c.Create(options); err != nil {
		return nil, fmt.Errorf("unable to create container: %v", err)
	}
	if err := c.Start(); err != nil {
		return nil, fmt.Errorf("unable to start container: %v", err)
	}
	// Set the resource limits
	if err := c.SetMemoryLimit(lxc.ByteSize(task.Resources.MemoryMB) * lxc.MB); err != nil {
		return nil, fmt.Errorf("unable to set memory limits: %v", err)
	}

	handle := lxcDriverHandle{
		container:      c,
		initPid:        c.InitPid(),
		lxcPath:        lxcPath,
		logger:         d.logger,
		killTimeout:    GetKillTimeout(task.KillTimeout, d.DriverContext.config.MaxKillTimeout),
		maxKillTimeout: d.DriverContext.config.MaxKillTimeout,
		totalCpuStats:  stats.NewCpuStats(),
		userCpuStats:   stats.NewCpuStats(),
		systemCpuStats: stats.NewCpuStats(),
		waitCh:         make(chan *dstructs.WaitResult, 1),
		doneCh:         make(chan bool, 1),
	}

	go handle.run()

	return &handle, nil
}

// Open opena the driver to monitor an existing LXC container
func (d *LxcDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
	pid := &lxcPID{}
	if err := json.Unmarshal([]byte(handleID), pid); err != nil {
		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
	}

	var container *lxc.Container
	containers := lxc.Containers(pid.LxcPath)
	for _, c := range containers {
		if c.Name() == pid.ContainerName {
			container = &c
			break
		}
	}

	if container == nil {
		return nil, fmt.Errorf("container %v not found", pid.ContainerName)
	}

	handle := lxcDriverHandle{
		container:      container,
		initPid:        container.InitPid(),
		lxcPath:        pid.LxcPath,
		logger:         d.logger,
		killTimeout:    pid.KillTimeout,
		maxKillTimeout: d.DriverContext.config.MaxKillTimeout,
		totalCpuStats:  stats.NewCpuStats(),
		userCpuStats:   stats.NewCpuStats(),
		systemCpuStats: stats.NewCpuStats(),
		waitCh:         make(chan *dstructs.WaitResult, 1),
		doneCh:         make(chan bool, 1),
	}
	go handle.run()

	return &handle, nil
}

// lxcDriverHandle allows controlling the lifecycle of an lxc container
type lxcDriverHandle struct {
	container *lxc.Container
	initPid   int
	lxcPath   string

	logger *log.Logger

	killTimeout    time.Duration
	maxKillTimeout time.Duration

	totalCpuStats  *stats.CpuStats
	userCpuStats   *stats.CpuStats
	systemCpuStats *stats.CpuStats

	waitCh chan *dstructs.WaitResult
	doneCh chan bool
}

type lxcPID struct {
	ContainerName string
	InitPid       int
	LxcPath       string
	KillTimeout   time.Duration
}

func (h *lxcDriverHandle) ID() string {
	pid := lxcPID{
		ContainerName: h.container.Name(),
		InitPid:       h.initPid,
		LxcPath:       h.lxcPath,
		KillTimeout:   h.killTimeout,
	}
	data, err := json.Marshal(pid)
	if err != nil {
		h.logger.Printf("[ERR] driver.lxc: failed to marshal lxc PID to JSON: %v", err)
	}
	return string(data)
}

func (h *lxcDriverHandle) WaitCh() chan *dstructs.WaitResult {
	return h.waitCh
}

func (h *lxcDriverHandle) Update(task *structs.Task) error {
	h.killTimeout = GetKillTimeout(task.KillTimeout, h.killTimeout)
	return nil
}

func (h *lxcDriverHandle) Kill() error {
	h.logger.Printf("[INFO] driver.lxc: shutting down container %q", h.container.Name())
	if err := h.container.Shutdown(h.killTimeout); err != nil {
		h.logger.Printf("[INFO] driver.lxc: shutting down container %q failed: %v", h.container.Name(), err)
		if err := h.container.Stop(); err != nil {
			h.logger.Printf("[ERR] driver.lxc: error stopping container %q: %v", h.container.Name(), err)
		}
	}
	close(h.doneCh)
	return nil
}

func (h *lxcDriverHandle) Stats() (*cstructs.TaskResourceUsage, error) {
	cpuStats, err := h.container.CPUStats()
	if err != nil {
		return nil, nil
	}
	total, err := h.container.CPUTime()
	if err != nil {
		return nil, nil
	}

	t := time.Now()

	// Get the cpu stats
	system := cpuStats["system"]
	user := cpuStats["user"]
	cs := &cstructs.CpuStats{
		SystemMode: h.systemCpuStats.Percent(float64(system)),
		UserMode:   h.systemCpuStats.Percent(float64(user)),
		Percent:    h.totalCpuStats.Percent(float64(total)),
		TotalTicks: float64(user + system),
		Measured:   LXCMeasuredCpuStats,
	}

	// Get the Memory Stats
	memData := map[string]uint64{
		"rss":   0,
		"cache": 0,
		"swap":  0,
	}
	rawMemStats := h.container.CgroupItem("memory.stat")
	for _, rawMemStat := range rawMemStats {
		key, val, err := keysToVal(rawMemStat)
		if err != nil {
			h.logger.Printf("[ERR] driver.lxc: error getting stat for key %q: %v", key, err)
			continue
		}
		if _, ok := memData[key]; ok {
			memData[key] = val

		}
	}
	ms := &cstructs.MemoryStats{
		RSS:      memData["rss"],
		Cache:    memData["cache"],
		Swap:     memData["swap"],
		Measured: LXCMeasuredMemStats,
	}

	mu := h.container.CgroupItem("memory.max_usage_in_bytes")
	for _, rawMemMaxUsage := range mu {
		val, err := strconv.ParseUint(rawMemMaxUsage, 10, 64)
		if err != nil {
			h.logger.Printf("[ERR] driver.lxc: unable to get max memory usage: %v", err)
			continue
		}
		ms.MaxUsage = val
	}
	ku := h.container.CgroupItem("memory.kmem.usage_in_bytes")
	for _, rawKernelUsage := range ku {
		val, err := strconv.ParseUint(rawKernelUsage, 10, 64)
		if err != nil {
			h.logger.Printf("[ERR] driver.lxc: unable to get kernel memory usage: %v", err)
			continue
		}
		ms.KernelUsage = val
	}

	mku := h.container.CgroupItem("memory.kmem.max_usage_in_bytes")
	for _, rawMaxKernelUsage := range mku {
		val, err := strconv.ParseUint(rawMaxKernelUsage, 10, 64)
		if err != nil {
			h.logger.Printf("[ERR] driver.lxc: unable to get max kernel memory usage: %v", err)
			continue
		}
		ms.KernelMaxUsage = val
	}

	taskResUsage := cstructs.TaskResourceUsage{
		ResourceUsage: &cstructs.ResourceUsage{
			CpuStats:    cs,
			MemoryStats: ms,
		},
		Timestamp: t.UTC().UnixNano(),
	}

	return &taskResUsage, nil
}

func (h *lxcDriverHandle) run() {
	defer close(h.waitCh)
	timer := time.NewTimer(containerMonitorIntv)
	for {
		select {
		case <-timer.C:
			process, err := os.FindProcess(h.initPid)
			if err != nil {
				h.waitCh <- &dstructs.WaitResult{Err: err}
				return
			}
			if err := process.Signal(syscall.Signal(0)); err != nil {
				h.waitCh <- &dstructs.WaitResult{}
				return
			}
			timer.Reset(containerMonitorIntv)
		case <-h.doneCh:
			h.waitCh <- &dstructs.WaitResult{}
			return
		}
	}
}

func keysToVal(line string) (string, uint64, error) {
	tokens := strings.Split(line, " ")
	key := tokens[0]
	val, err := strconv.ParseUint(tokens[1], 10, 64)
	return key, val, err
}
<commit_msg>Adding cpu resource limits<commit_after>//+build linux

package driver

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/hashicorp/nomad/client/config"
	"github.com/hashicorp/nomad/client/fingerprint"
	"github.com/hashicorp/nomad/client/stats"
	"github.com/hashicorp/nomad/helper/fields"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/mitchellh/mapstructure"

	dstructs "github.com/hashicorp/nomad/client/driver/structs"
	cstructs "github.com/hashicorp/nomad/client/structs"
	lxc "gopkg.in/lxc/go-lxc.v2"
)

const (
	containerMonitorIntv = 2 * time.Second
)

var (
	LXCMeasuredCpuStats = []string{"System Mode", "User Mode", "Percent"}
	LXCMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage", "Kernel Usage", "Kernel Max Usage"}
)

// Add the lxc driver to the list of builtin drivers
func init() {
	BuiltinDrivers["lxc"] = NewLxcDriver
}

// LxcDriver allows users to run LXC Containers
type LxcDriver struct {
	DriverContext
	fingerprint.StaticFingerprinter
}

// LxcDriverConfig is the configuration of the LXC Container
type LxcDriverConfig struct {
	Template             string
	Distro               string
	Release              string
	Arch                 string
	ImageVariant         string   "mapstructure:`image_variant`"
	ImageServer          string   "mapstructure:`image_server`"
	GPGKeyID             string   "mapstructure:`gpg_key_id`"
	GPGKeyServer         string   "mapstructure:`gpg_key_server`"
	DisableGPGValidation bool     "mapstructure:`disable_gpg`"
	FlushCache           bool     "mapstructure:`flush_cache`"
	ForceCache           bool     "mapstructure:`force_cache`"
	TemplateArgs         []string "mapstructure:`template_args`"
}

// NewLxcDriver returns a new instance of the LXC driver
func NewLxcDriver(ctx *DriverContext) Driver {
	return &LxcDriver{DriverContext: *ctx}
}

// Validate validates the lxc driver configuration
func (d *LxcDriver) Validate(config map[string]interface{}) error {
	fd := &fields.FieldData{
		Raw: config,
		Schema: map[string]*fields.FieldSchema{
			"template": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: true,
			},
			"distro": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"release": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"arch": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"image_variant": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"image_server": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"gpg_key_id": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"gpg_key_server": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"disable_gpg": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"flush_cache": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"force_cache": &fields.FieldSchema{
				Type:     fields.TypeString,
				Required: false,
			},
			"template_args": &fields.FieldSchema{
				Type:     fields.TypeArray,
				Required: false,
			},
		},
	}

	if err := fd.Validate(); err != nil {
		return err
	}
	return nil
}

// Fingerprint fingerprints the lxc driver configuration
func (d *LxcDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
	version := lxc.Version()
	if version == "" {
		return false, nil
	}
	node.Attributes["driver.lxc.version"] = version
	node.Attributes["driver.lxc"] = "1"
	return true, nil
}

// Start starts the LXC Driver
func (d *LxcDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
	var driverConfig LxcDriverConfig
	if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
		return nil, err
	}
	lxcPath := lxc.DefaultConfigPath()
	if path := d.config.Read("lxc.path"); path != "" {
		lxcPath = path
	}

	containerName := fmt.Sprintf("%s-%s", task.Name, ctx.AllocID)
	c, err := lxc.NewContainer(containerName, lxcPath)
	if err != nil {
		return nil, fmt.Errorf("unable to create container: %v", err)
	}

	// TODO make them driver configuration
	c.SetVerbosity(lxc.Verbose)
	c.SetLogLevel(lxc.TRACE)

	logFile := filepath.Join(ctx.AllocDir.LogDir(), fmt.Sprintf("%v-lxc.log", task.Name))
	c.SetLogFile(logFile)

	options := lxc.TemplateOptions{
		Template:             driverConfig.Template,
		Distro:               driverConfig.Distro,
		Release:              driverConfig.Release,
		Arch:                 driverConfig.Arch,
		FlushCache:           driverConfig.FlushCache,
		DisableGPGValidation: driverConfig.DisableGPGValidation,
	}

	if err := c.Create(options); err != nil {
		return nil, fmt.Errorf("unable to create container: %v", err)
	}
	if err := c.Start(); err != nil {
		return nil, fmt.Errorf("unable to start container: %v", err)
	}
	// Set the resource limits
	if err := c.SetMemoryLimit(lxc.ByteSize(task.Resources.MemoryMB) * lxc.MB); err != nil {
		return nil, fmt.Errorf("unable to set memory limits: %v", err)
	}
	if err := c.SetCgroupItem("cpu.shares", strconv.Itoa(task.Resources.CPU)); err != nil {
		return nil, fmt.Errorf("unable to set cpu shares: %v", err)
	}

	handle := lxcDriverHandle{
		container:      c,
		initPid:        c.InitPid(),
		lxcPath:        lxcPath,
		logger:         d.logger,
		killTimeout:    GetKillTimeout(task.KillTimeout, d.DriverContext.config.MaxKillTimeout),
		maxKillTimeout: d.DriverContext.config.MaxKillTimeout,
		totalCpuStats:  stats.NewCpuStats(),
		userCpuStats:   stats.NewCpuStats(),
		systemCpuStats: stats.NewCpuStats(),
		waitCh:         make(chan *dstructs.WaitResult, 1),
		doneCh:         make(chan bool, 1),
	}

	go handle.run()

	return &handle, nil
}

// Open opena the driver to monitor an existing LXC container
func (d *LxcDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
	pid := &lxcPID{}
	if err := json.Unmarshal([]byte(handleID), pid); err != nil {
		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
	}

	var container *lxc.Container
	containers := lxc.Containers(pid.LxcPath)
	for _, c := range containers {
		if c.Name() == pid.ContainerName {
			container = &c
			break
		}
	}

	if container == nil {
		return nil, fmt.Errorf("container %v not found", pid.ContainerName)
	}

	handle := lxcDriverHandle{
		container:      container,
		initPid:        container.InitPid(),
		lxcPath:        pid.LxcPath,
		logger:         d.logger,
		killTimeout:    pid.KillTimeout,
		maxKillTimeout: d.DriverContext.config.MaxKillTimeout,
		totalCpuStats:  stats.NewCpuStats(),
		userCpuStats:   stats.NewCpuStats(),
		systemCpuStats: stats.NewCpuStats(),
		waitCh:         make(chan *dstructs.WaitResult, 1),
		doneCh:         make(chan bool, 1),
	}
	go handle.run()

	return &handle, nil
}

// lxcDriverHandle allows controlling the lifecycle of an lxc container
type lxcDriverHandle struct {
	container *lxc.Container
	initPid   int
	lxcPath   string

	logger *log.Logger

	killTimeout    time.Duration
	maxKillTimeout time.Duration

	totalCpuStats  *stats.CpuStats
	userCpuStats   *stats.CpuStats
	systemCpuStats *stats.CpuStats

	waitCh chan *dstructs.WaitResult
	doneCh chan bool
}

type lxcPID struct {
	ContainerName string
	InitPid       int
	LxcPath       string
	KillTimeout   time.Duration
}

func (h *lxcDriverHandle) ID() string {
	pid := lxcPID{
		ContainerName: h.container.Name(),
		InitPid:       h.initPid,
		LxcPath:       h.lxcPath,
		KillTimeout:   h.killTimeout,
	}
	data, err := json.Marshal(pid)
	if err != nil {
		h.logger.Printf("[ERR] driver.lxc: failed to marshal lxc PID to JSON: %v", err)
	}
	return string(data)
}

func (h *lxcDriverHandle) WaitCh() chan *dstructs.WaitResult {
	return h.waitCh
}

func (h *lxcDriverHandle) Update(task *structs.Task) error {
	h.killTimeout = GetKillTimeout(task.KillTimeout, h.killTimeout)
	return nil
}

func (h *lxcDriverHandle) Kill() error {
	h.logger.Printf("[INFO] driver.lxc: shutting down container %q", h.container.Name())
	if err := h.container.Shutdown(h.killTimeout); err != nil {
		h.logger.Printf("[INFO] driver.lxc: shutting down container %q failed: %v", h.container.Name(), err)
		if err := h.container.Stop(); err != nil {
			h.logger.Printf("[ERR] driver.lxc: error stopping container %q: %v", h.container.Name(), err)
		}
	}
	close(h.doneCh)
	return nil
}

func (h *lxcDriverHandle) Stats() (*cstructs.TaskResourceUsage, error) {
	cpuStats, err := h.container.CPUStats()
	if err != nil {
		return nil, nil
	}
	total, err := h.container.CPUTime()
	if err != nil {
		return nil, nil
	}

	t := time.Now()

	// Get the cpu stats
	system := cpuStats["system"]
	user := cpuStats["user"]
	cs := &cstructs.CpuStats{
		SystemMode: h.systemCpuStats.Percent(float64(system)),
		UserMode:   h.systemCpuStats.Percent(float64(user)),
		Percent:    h.totalCpuStats.Percent(float64(total)),
		TotalTicks: float64(user + system),
		Measured:   LXCMeasuredCpuStats,
	}

	// Get the Memory Stats
	memData := map[string]uint64{
		"rss":   0,
		"cache": 0,
		"swap":  0,
	}
	rawMemStats := h.container.CgroupItem("memory.stat")
	for _, rawMemStat := range rawMemStats {
		key, val, err := keysToVal(rawMemStat)
		if err != nil {
			h.logger.Printf("[ERR] driver.lxc: error getting stat for key %q: %v", key, err)
			continue
		}
		if _, ok := memData[key]; ok {
			memData[key] = val

		}
	}
	ms := &cstructs.MemoryStats{
		RSS:      memData["rss"],
		Cache:    memData["cache"],
		Swap:     memData["swap"],
		Measured: LXCMeasuredMemStats,
	}

	mu := h.container.CgroupItem("memory.max_usage_in_bytes")
	for _, rawMemMaxUsage := range mu {
		val, err := strconv.ParseUint(rawMemMaxUsage, 10, 64)
		if err != nil {
			h.logger.Printf("[ERR] driver.lxc: unable to get max memory usage: %v", err)
			continue
		}
		ms.MaxUsage = val
	}
	ku := h.container.CgroupItem("memory.kmem.usage_in_bytes")
	for _, rawKernelUsage := range ku {
		val, err := strconv.ParseUint(rawKernelUsage, 10, 64)
		if err != nil {
			h.logger.Printf("[ERR] driver.lxc: unable to get kernel memory usage: %v", err)
			continue
		}
		ms.KernelUsage = val
	}

	mku := h.container.CgroupItem("memory.kmem.max_usage_in_bytes")
	for _, rawMaxKernelUsage := range mku {
		val, err := strconv.ParseUint(rawMaxKernelUsage, 10, 64)
		if err != nil {
			h.logger.Printf("[ERR] driver.lxc: unable to get max kernel memory usage: %v", err)
			continue
		}
		ms.KernelMaxUsage = val
	}

	taskResUsage := cstructs.TaskResourceUsage{
		ResourceUsage: &cstructs.ResourceUsage{
			CpuStats:    cs,
			MemoryStats: ms,
		},
		Timestamp: t.UTC().UnixNano(),
	}

	return &taskResUsage, nil
}

func (h *lxcDriverHandle) run() {
	defer close(h.waitCh)
	timer := time.NewTimer(containerMonitorIntv)
	for {
		select {
		case <-timer.C:
			process, err := os.FindProcess(h.initPid)
			if err != nil {
				h.waitCh <- &dstructs.WaitResult{Err: err}
				return
			}
			if err := process.Signal(syscall.Signal(0)); err != nil {
				h.waitCh <- &dstructs.WaitResult{}
				return
			}
			timer.Reset(containerMonitorIntv)
		case <-h.doneCh:
			h.waitCh <- &dstructs.WaitResult{}
			return
		}
	}
}

func keysToVal(line string) (string, uint64, error) {
	tokens := strings.Split(line, " ")
	key := tokens[0]
	val, err := strconv.ParseUint(tokens[1], 10, 64)
	return key, val, err
}
<|endoftext|>


<commit_before>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
// Author: arnej

package jvm

import (
	"os"

	"github.com/vespa-engine/vespa/client/go/prog"
	"github.com/vespa-engine/vespa/client/go/trace"
	"github.com/vespa-engine/vespa/client/go/util"
)

func RunApplicationContainer(extraArgs []string) int {
	trace.AdjustVerbosity(0)
	container := NewApplicationContainer(extraArgs)
	container.Exec()
	// unreachable:
	return 1
}

func NewApplicationContainer(extraArgs []string) Container {
	configId := os.Getenv(util.ENV_CONFIG_ID)
	serviceName := os.Getenv(util.ENV_SERVICE_NAME)
	a := ApplicationContainer{
		configId:    configId,
		serviceName: serviceName,
	}
	opts := NewOptions(&a)
	a.addJvmArgs(&opts)
	for _, x := range extraArgs {
		opts.AddOption(x)
	}
	return &a
}

func (a *ApplicationContainer) Exec() {
	argv := make([]string, 0, 100)
	argv = append(argv, "java")
	for _, x := range a.jvmArgs.Args() {
		argv = append(argv, x)
	}
	p := prog.NewSpec(argv)
	p.ConfigureNumaCtl()
	exportEnvSettings(a, p)
	err := p.Run()
	util.JustExitWith(err)
}
<commit_msg>use TRACE_JVM_STARTUP / DEBUG_JVM_STARTUP to trigger logging<commit_after>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
// Author: arnej

package jvm

import (
	"os"

	"github.com/vespa-engine/vespa/client/go/prog"
	"github.com/vespa-engine/vespa/client/go/trace"
	"github.com/vespa-engine/vespa/client/go/util"
)

func RunApplicationContainer(extraArgs []string) int {
	if doTrace := os.Getenv("TRACE_JVM_STARTUP"); doTrace != "" {
		trace.AdjustVerbosity(1)
	}
	if doDebug := os.Getenv("DEBUG_JVM_STARTUP"); doDebug != "" {
		trace.AdjustVerbosity(2)
	}
	container := NewApplicationContainer(extraArgs)
	container.Exec()
	// unreachable:
	return 1
}

func NewApplicationContainer(extraArgs []string) Container {
	configId := os.Getenv(util.ENV_CONFIG_ID)
	serviceName := os.Getenv(util.ENV_SERVICE_NAME)
	a := ApplicationContainer{
		configId:    configId,
		serviceName: serviceName,
	}
	opts := NewOptions(&a)
	a.addJvmArgs(&opts)
	for _, x := range extraArgs {
		opts.AddOption(x)
	}
	return &a
}

func (a *ApplicationContainer) Exec() {
	argv := make([]string, 0, 100)
	argv = append(argv, "java")
	for _, x := range a.jvmArgs.Args() {
		argv = append(argv, x)
	}
	p := prog.NewSpec(argv)
	p.ConfigureNumaCtl()
	exportEnvSettings(a, p)
	err := p.Run()
	util.JustExitWith(err)
}
<|endoftext|>


<commit_before>// Notes: the program does not care the JSON doc
// is from who and where.
// This program will be expanded to:
// (1) include more JSON specs/formats.
// (2) one JSON doc has multiple JSON structures
// (3) Log output to separate file per process using Go’s logging framework
// (4) watchdog timer for spawned processes
// (5) recovery in case of process crashes
// (6) deal with repeated failures of startup
//     due to resoure constraints or other reasons
//     like network connection failures, etc.
// (7) a REST API support to stop/suspend/kill/cleanup
//     the spawned processes manually by the admin
// (8) more stuff to be added
//
// For now, the workflow is simply as follows:
// (a) read the JSON doc
// (b) parse the JSON
// (c) spawn processes
// (d) logging the output
// (e) starts watchdog timer if needed for each process
// (f) restart the process if it exits with error
// (g) ...

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"time"
)

type Watchdog struct {
	resets   chan int64
	timeouts chan bool
}

//create a new watchdog
func newWatchdog() *Watchdog {
	wd := &Watchdog{
		resets: make(chan int64, 100),

		timeouts: make(chan bool, 2),
	}
    //init. as false to coordinate events
    wd.timeouts <- false
	go wd.loop()
	return wd
}

//check if there are new resets
func (wd *Watchdog) pump(t0 int64) (t1 int64) {
	t1 = t0
	for {
		select {
		case t := <-wd.resets:
			if t > t0 {
				t1 = t
			}
		default:
			return
		}
	}
	panic("unreachable")
}

//if there is no reset, it is in idle state
//if there is reset, decrement the timer
//till it times out and set the timeout
//channel value to true
func (wd *Watchdog) loop() {
	var t0 int64
idle:
	t0 = <-wd.resets
	t0 = wd.pump(t0)
loop:
	t0 = t0 - time.Now().UnixNano()
	time.Sleep(time.Second * time.Duration(t0))
	now := time.Now().UnixNano()
	t0 = wd.pump(now)
	if t0 == now {
		wd.timeouts <- true
		goto idle
	}
	goto loop
}

//check errors
func check(e error) {
	if e != nil {
		fmt.Println("error:", e)
		panic(e)
	}
}

//this is to set the timeout in number of NanoSeconds
//use absolute time value to avoid miscalculations
func (wd *Watchdog) reset(timeoutNanoSecs int64) {
	wd.resets <- timeoutNanoSecs + time.Now().UnixNano()
}

//check timeouts, TO CHANGE: may need to add additional
//channel to coordinate
func check_timeouts(wd *Watchdog, logFile *os.File) int {
    tmo:= <-wd.timeouts
    if (tmo == true) {
        fmt.Fprintf(logFile, "Timed out!")
        return 1
    }
    //no need to continue the timer
    wd.reset(0)
    return 0
}

func main() {

	type Process struct {
		Name    string   `json:"name"`
		Path    string   `json:"path"`
		Cmdargs string   `json:"cmdargs"`
		Env     []string `json:"env"`
	}

	type Processes struct {
		Processes []Process `json:"processes"`
	}

	//read the JSON doc
	jsonBlob, err := ioutil.ReadFile("JSONdoc.txt")
	check(err)
	fmt.Print(string(jsonBlob))

	//JSON parser
	var processes Processes
	var logFile *os.File

	err = json.Unmarshal(jsonBlob, &processes)
	check(err)
	fmt.Printf("%+v", processes)
	fmt.Println()

	//watchdog for each process
	var wdg []*Watchdog
	wdg = make([]*Watchdog, len(processes.Processes))

	//spawn processes
	for i := 0; i <= len(processes.Processes)-1; i++ {
		fileName := "logfile"
		//log file name is based on date/time
		cmd := exec.Command("date")
		cmdOut, err := cmd.Output()
		check(err)
		fileName += string(cmdOut) + "_" + string(48+i)
		logFile, err = os.Create(fileName)
		check(err)

        //TO CHANGE: use goroutines to do this
		cmd = exec.Command(processes.Processes[i].Name, processes.Processes[i].Cmdargs)
		cmdOut, err = cmd.Output()
		//TODO: if there is error, needs to restart for a given number of times
		check(err)

		//test only for the watchdogs
		wdg[i] = newWatchdog()
        //set the timeout as 50 NanoSeconds in the test, it should change later
        //based on the application scenarios
		wdg[i].reset(50)

        // use goroutines to check the timeout
        // and use another channel w/ buffer size to see if timeout occurs
        //before the process ends, if process already ends, reset(0)
        go check_timeouts(wdg[i], logFile)

		//fmt.Println(string(cmdOut))
		log.Println(string(cmdOut))
		fmt.Fprintf(logFile, string(cmdOut))
		defer logFile.Close()
	}
}
<commit_msg>added restart if the process errors out<commit_after>// Notes: the program does not care the JSON doc
// is from who and where.
// This program will be expanded to:
// (1) include more JSON specs/formats.
// (2) one JSON doc has multiple JSON structures
// (3) Log output to separate file per process using Go’s logging framework
// (4) watchdog timer for spawned processes
// (5) recovery in case of process crashes
// (6) deal with repeated failures of startup
//     due to resoure constraints or other reasons
//     like network connection failures, etc.
// (7) a REST API support to stop/suspend/kill/cleanup
//     the spawned processes manually by the admin
// (8) more stuff to be added
//
// For now, the workflow is simply as follows:
// (a) read the JSON doc
// (b) parse the JSON
// (c) spawn processes
// (d) logging the output
// (e) starts watchdog timer if needed for each process
// (f) restart the process if it exits with error
// (g) ...

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"time"
)

type Watchdog struct {
	resets   chan int64
	timeouts chan bool
}

//create a new watchdog
func newWatchdog() *Watchdog {
	wd := &Watchdog{
		resets: make(chan int64, 100),

		timeouts: make(chan bool, 2),
	}
	//init. as false to coordinate events
	wd.timeouts <- false
	go wd.loop()
	return wd
}

//check if there are new resets
func (wd *Watchdog) pump(t0 int64) (t1 int64) {
	t1 = t0
	for {
		select {
		case t := <-wd.resets:
			if t > t0 {
				t1 = t
			}
		default:
			return
		}
	}
	panic("unreachable")
}

//if there is no reset, it is in idle state
//if there is reset, decrement the timer
//till it times out and set the timeout
//channel value to true
func (wd *Watchdog) loop() {
	var t0 int64
idle:
	t0 = <-wd.resets
	t0 = wd.pump(t0)
loop:
	t0 = t0 - time.Now().UnixNano()
	time.Sleep(time.Second * time.Duration(t0))
	now := time.Now().UnixNano()
	t0 = wd.pump(now)
	if t0 == now {
		wd.timeouts <- true
		goto idle
	}
	goto loop
}

//check errors
func check(e error) {
	if e != nil {
		fmt.Println("error:", e)
		panic(e)
	}
}

//this is to set the timeout in Seconds
//use absolute time value to avoid miscalculations
func (wd *Watchdog) reset(timeoutSecs int64) {
	wd.resets <- timeoutSecs + time.Now().UnixNano()
}

//check timeouts, note that we may need to add additional
//channel to coordinate
func check_timeouts(wd *Watchdog,
	logFile *os.File, pre int) {
	tmo := <-wd.timeouts
	if tmo == true {
		fmt.Fprintf(logFile, "Timed out!")
		// TO ADD: we may need to terminate
		// the spawned process forcefully here
		// if it is still running
	}
	//no need to continue the timer if process already ends
	if pre == 1 {
		wd.reset(0)
	}
}

//run procs, if it fails, we need to retry
//with configuratable delay time and maximum
//number of tries
func run_procs(cmd *exec.Cmd, wd *Watchdog,
	logFile *os.File, delaySecs int64,
	numRetries int, tmo int64) {
	//check the timeout with a goroutine
	go check_timeouts(wd, logFile, 0)
restart:
	//run the process and returns the output
	cmdOut, err := cmd.Output()
	if err != nil {
		log.Fatal(err)
		//restart if it errors out
		numRetries--
		if numRetries >= 0 {
			//delay for a while
			time.Sleep(time.Second * time.Duration(delaySecs))
			//reset the watchdog timer
			wd.reset(tmo)
			//retry
			goto restart
		}
	}
	go check_timeouts(wd, logFile, 1)

	//deal with the output
	log.Println(string(cmdOut))
	fmt.Fprintf(logFile, string(cmdOut))
	//wait until operation on logFile is done, close it
	defer logFile.Close()
}

func main() {

	type Process struct {
		Name    string   `json:"name"`
		Path    string   `json:"path"`
		Cmdargs string   `json:"cmdargs"`
		Env     []string `json:"env"`
	}

	type Processes struct {
		Processes []Process `json:"processes"`
	}

	//read the JSON doc
	jsonBlob, err := ioutil.ReadFile("JSONdoc.txt")
	check(err)
	fmt.Print(string(jsonBlob))

	//JSON parser
	var processes Processes
	var logFile *os.File

	err = json.Unmarshal(jsonBlob, &processes)
	check(err)
	fmt.Printf("%+v", processes)
	fmt.Println()

	//watchdog for each process
	var tmo int64
	var delaySecs int64
	var numRetries int
	var wdg []*Watchdog
	wdg = make([]*Watchdog, len(processes.Processes))

	//spawning processes
	for i := 0; i <= len(processes.Processes)-1; i++ {
		fileName := "logfile"
		//log file name is based on date/time
		cmd := exec.Command("date")
		cmdOut, err := cmd.Output()
		check(err)
		fileName += string(cmdOut) + "_" + string(48+i)
		logFile, err = os.Create(fileName)
		check(err)

		//test only for the watchdogs
		wdg[i] = newWatchdog()
		//set the timeout as 5 Seconds in the test, it should change later
		//based on the application scenarios
		tmo = 5
		delaySecs = 1
		numRetries = 3
		wdg[i].reset(tmo)

        //spawn and run the processes
		cmd = exec.Command(processes.Processes[i].Name, processes.Processes[i].Cmdargs)
		run_procs(cmd, wdg[i], logFile, delaySecs, numRetries, tmo)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Workiva, LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package bitarray

func andSparseWithSparseBitArray(sba, other *sparseBitArray) BitArray {
	max := maxInt64(int64(len(sba.indices)), int64(len(other.indices)))
	indices := make(uintSlice, 0, max)
	blocks := make(blocks, 0, max)

	selfIndex := 0
	otherIndex := 0
	var resultBlock block

	// move through the array and compare the blocks if they happen to
	// intersect
	for {
		if selfIndex == len(sba.indices) || otherIndex == len(other.indices) {
			// One of the arrays has been exhausted. We don't need
			// to compare anything else for a bitwise and; the
			// operation is complete.
			break
		}

		selfValue := sba.indices[selfIndex]
		otherValue := other.indices[otherIndex]

		switch {
		case otherValue < selfValue:
			// The `sba` bitarray has a block with a index position
			// greater than us. We want to compare with that block
			// if possible, so move our `other` index closer to that
			// block's index.
			otherIndex++

		case otherValue > selfValue:
			// This is the exact logical inverse of the above case.
			selfIndex++

		default:
			// Here, our indices match for both `sba` and `other`.
			// Time to do the bitwise AND operation and add a block
			// to our result list if the block has values in it.
			resultBlock = sba.blocks[selfIndex].and(other.blocks[otherIndex])
			if resultBlock > 0 {
				indices = append(indices, selfValue)
				blocks = append(blocks, resultBlock)
			}
			selfIndex++
			otherIndex++
		}
	}

	return &sparseBitArray{
		indices: indices,
		blocks:  blocks,
	}
}

func andSparseWithDenseBitArray(sba *sparseBitArray, other *bitArray) BitArray {
	// Use a duplicate of the sparse array to store the results of the
	// bitwise and. More memory-efficient than allocating a new dense bit
	// array.
	//
	// NOTE: this could be faster if we didn't copy the values as well
	// (since they are overwritten), but I don't want this method to know
	// too much about the internals of sparseBitArray. The performance hit
	// should be minor anyway.
	ba := sba.copy()

	// Run through the sparse array and attempt comparisons wherever
	// possible against the dense bit array.
	for selfIndex, selfValue := range ba.indices {

		if selfValue >= uint64(len(other.blocks)) {
			// The dense bit array has been exhausted. This is the
			// annoying case because we have to trim the sparse
			// array to the size of the dense array.
			ba.blocks = ba.blocks[:selfIndex]
			ba.indices = ba.indices[:selfIndex]

			// once this is done, there are no more comparisons.
			// We're ready to return
			break
		}
		ba.blocks[selfIndex] = ba.blocks[selfIndex].and(
			other.blocks[selfValue])
	}

	return ba
}

func andDenseWithDenseBitArray(dba, other *bitArray) BitArray {
	min := minUint64(uint64(len(dba.blocks)), uint64(len(other.blocks)))

	ba := newBitArray(min * s)

	for i := uint64(0); i < min; i++ {
		ba.blocks[i] = dba.blocks[i].and(other.blocks[i])
	}

	ba.setLowest()
	ba.setHighest()

	return ba
}
<commit_msg>drop zero-block values in sparse vs dense AND<commit_after>/*
Copyright 2014 Workiva, LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package bitarray

func andSparseWithSparseBitArray(sba, other *sparseBitArray) BitArray {
	max := maxInt64(int64(len(sba.indices)), int64(len(other.indices)))
	indices := make(uintSlice, 0, max)
	blocks := make(blocks, 0, max)

	selfIndex := 0
	otherIndex := 0
	var resultBlock block

	// move through the array and compare the blocks if they happen to
	// intersect
	for {
		if selfIndex == len(sba.indices) || otherIndex == len(other.indices) {
			// One of the arrays has been exhausted. We don't need
			// to compare anything else for a bitwise and; the
			// operation is complete.
			break
		}

		selfValue := sba.indices[selfIndex]
		otherValue := other.indices[otherIndex]

		switch {
		case otherValue < selfValue:
			// The `sba` bitarray has a block with a index position
			// greater than us. We want to compare with that block
			// if possible, so move our `other` index closer to that
			// block's index.
			otherIndex++

		case otherValue > selfValue:
			// This is the exact logical inverse of the above case.
			selfIndex++

		default:
			// Here, our indices match for both `sba` and `other`.
			// Time to do the bitwise AND operation and add a block
			// to our result list if the block has values in it.
			resultBlock = sba.blocks[selfIndex].and(other.blocks[otherIndex])
			if resultBlock > 0 {
				indices = append(indices, selfValue)
				blocks = append(blocks, resultBlock)
			}
			selfIndex++
			otherIndex++
		}
	}

	return &sparseBitArray{
		indices: indices,
		blocks:  blocks,
	}
}

func andSparseWithDenseBitArray(sba *sparseBitArray, other *bitArray) BitArray {
	// Use a duplicate of the sparse array to store the results of the
	// bitwise and. More memory-efficient than allocating a new dense bit
	// array.
	//
	// NOTE: this could be faster if we didn't copy the values as well
	// (since they are overwritten), but I don't want this method to know
	// too much about the internals of sparseBitArray. The performance hit
	// should be minor anyway.
	ba := sba.copy()

	// Run through the sparse array and attempt comparisons wherever
	// possible against the dense bit array.
	for selfIndex, selfValue := range sba.indices {

		if selfValue >= uint64(len(other.blocks)) {
			// The dense bit array has been exhausted. This is the
			// annoying case because we have to trim the sparse
			// array to the size of the dense array.
			ba.blocks = ba.blocks[:selfIndex]
			ba.indices = ba.indices[:selfIndex]

			// once this is done, there are no more comparisons.
			// We're ready to return
			break
		}
		ba.blocks[selfIndex] = ba.blocks[selfIndex].and(
			other.blocks[selfValue])

		if ba.blocks[selfIndex] == 0 {
			ba.blocks.deleteAtIndex(int64(selfIndex))
			ba.indices.deleteAtIndex(int64(selfIndex))
		}
	}

	return ba
}

func andDenseWithDenseBitArray(dba, other *bitArray) BitArray {
	min := minUint64(uint64(len(dba.blocks)), uint64(len(other.blocks)))

	ba := newBitArray(min * s)

	for i := uint64(0); i < min; i++ {
		ba.blocks[i] = dba.blocks[i].and(other.blocks[i])
	}

	ba.setLowest()
	ba.setHighest()

	return ba
}
<|endoftext|>