<commit_before>package log_sync

import (
	"crypto/md5"
	"errors"
	"fmt"
	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/s3"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

type SyncT struct {
	Dir     string
	Suffix  string
	Bucket  *s3.Bucket
	Prefix  string
	Threads uint
	Auth    aws.Auth
	Dry     bool
	NoAws   bool
}

type empty struct{}

type transfer struct {
	Src  string
	Dest string
}

func (s *SyncT) validateDest() error {
	testPath := fmt.Sprintf("%s/sync_test", s.Prefix)
	err := s.Bucket.Put(testPath, []byte("test"), "text", s3.Private, s3.Options{})
	if err != nil {
		return err
	}
	err = s.Bucket.Del(testPath)
	return err
}

func (s *SyncT) validateSrc() error {
	_, err := os.Stat(s.Dir)
	return err
}

func (s *SyncT) validate() error {
	if s.Threads == 0 {
		return errors.New("Number of threads specified must be non-zero")
	}

	err := s.validateSrc()
	if err != nil {
		return err
	}

	if !s.NoAws {
		err = s.validateDest()
		if err != nil {
			return err
		}
	}

	return nil
}

func relativePath(path string, logPath string) string {
	if path == "." {
		return strings.TrimLeft(logPath, "/")
	} else {
		return strings.TrimLeft(strings.TrimPrefix(logPath, path), "/")
	}
}

func (s *SyncT) loadSrc() map[string]string {
	logs := map[string]string{}
	filepath.Walk(s.Dir, func(logPath string, info os.FileInfo, err error) error {
		if !info.IsDir() && strings.HasSuffix(info.Name(), s.Suffix) {
			path := relativePath(s.Dir, logPath)

			buf, err := ioutil.ReadFile(logPath)
			if err != nil {
				// Something went wrong reading a log, so print and panic
				return err
			}

			md5Hash := md5.New()
			md5Hash.Write(buf)
			md5sum := fmt.Sprintf("%x", md5Hash.Sum(nil))
			logs[path] = md5sum
		}
		return nil
	})
	return logs
}

func (s *SyncT) loadDest() (map[string]string, error) {
	logs := map[string]string{}
	data, err := s.Bucket.List(s.Prefix, "", "", 0)
	if err != nil {
		return nil, err
	}
	if data.IsTruncated == true {
		msg := "Results from S3 truncated and I don't yet know how to download next set of results, so we will exit to avoid invalidating results."
		return nil, errors.New(msg)
	}
	for i := range data.Contents {
		md5sum := strings.Trim(data.Contents[i].ETag, "\"")
		path := relativePath(s.Prefix, data.Contents[i].Key)
		logs[path] = md5sum
	}
	return logs, nil
}

func putLog(log transfer, bucket *s3.Bucket, dry bool) {
	data, err := ioutil.ReadFile(log.Src)
	if err != nil {
		// Error reading log
		fmt.Printf("Error reading source file %s:\n", log.Src)
		panic(err.Error())
	}

	contType := "binary/octet-stream"
	perm := s3.ACL("private")

	if dry {
		fmt.Printf("Starting sync of %s to bucket path %s...\n", log.Src, log.Dest)
	} else {
		fmt.Printf("Starting sync of %s to s3://%s/%s...\n", log.Src, bucket.Name, log.Dest)
		err = bucket.Put(log.Dest, data, contType, perm, s3.Options{})
		if err != nil {
			// Error uploading log to s3
			fmt.Printf("Sync of %s to s3://%s/%s failed:\n", log.Src, bucket.Name, log.Dest)
			panic(err.Error())
		}
	}
}

func syncFile(log transfer, bucket *s3.Bucket, workerChan chan empty, dry bool) {
	putLog(log, bucket, dry)
	<-workerChan
}

func workerSpawner(bucket *s3.Bucket, fileChan chan transfer, workerChan chan empty, dieChan chan empty, dry bool) {
	for {
		select {
		case file := <-fileChan:
			go syncFile(file, bucket, workerChan, dry)
		case <-dieChan:
			return
		}
	}
}

func (s *SyncT) syncLogs(src, dest map[string]string) error {
	fileChan := make(chan transfer)
	workerChan := make(chan empty, s.Threads)
	dieChan := make(chan empty)
	go workerSpawner(s.Bucket, fileChan, workerChan, dieChan, s.Dry)

	for log, _ := range src {
		if dest[log] != src[log] {
			srcPath := strings.Join([]string{s.Dir, log}, "/")
			destPath := strings.Join([]string{s.Prefix, log}, "/")
			workerChan <- empty{}
			fileChan <- transfer{srcPath, destPath}
		}
	}
	dieChan <- empty{}
	return nil
}

func (s *SyncT) Sync() error {
	err := s.validate()
	if err != nil {
		return err
	}

	var destLogs map[string]string
	if !s.NoAws {
		destLogs, err = s.loadDest()
		if err != nil {
			return err
		}
	}
	srcLogs := s.loadSrc()

	return s.syncLogs(srcLogs, destLogs)
}

<commit_msg>change package name to logsync<commit_after>package logsync

import (
	"crypto/md5"
	"errors"
	"fmt"
	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/s3"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

type SyncT struct {
	Dir     string
	Suffix  string
	Bucket  *s3.Bucket
	Prefix  string
	Threads uint
	Auth    aws.Auth
	Dry     bool
	NoAws   bool
}

type empty struct{}

type transfer struct {
	Src  string
	Dest string
}

func (s *SyncT) validateDest() error {
	testPath := fmt.Sprintf("%s/sync_test", s.Prefix)
	err := s.Bucket.Put(testPath, []byte("test"), "text", s3.Private, s3.Options{})
	if err != nil {
		return err
	}
	err = s.Bucket.Del(testPath)
	return err
}

func (s *SyncT) validateSrc() error {
	_, err := os.Stat(s.Dir)
	return err
}

func (s *SyncT) validate() error {
	if s.Threads == 0 {
		return errors.New("Number of threads specified must be non-zero")
	}

	err := s.validateSrc()
	if err != nil {
		return err
	}

	if !s.NoAws {
		err = s.validateDest()
		if err != nil {
			return err
		}
	}

	return nil
}

func relativePath(path string, logPath string) string {
	if path == "." {
		return strings.TrimLeft(logPath, "/")
	} else {
		return strings.TrimLeft(strings.TrimPrefix(logPath, path), "/")
	}
}

func (s *SyncT) loadSrc() map[string]string {
	logs := map[string]string{}
	filepath.Walk(s.Dir, func(logPath string, info os.FileInfo, err error) error {
		if !info.IsDir() && strings.HasSuffix(info.Name(), s.Suffix) {
			path := relativePath(s.Dir, logPath)

			buf, err := ioutil.ReadFile(logPath)
			if err != nil {
				// Something went wrong reading a log, so print and panic
				return err
			}

			md5Hash := md5.New()
			md5Hash.Write(buf)
			md5sum := fmt.Sprintf("%x", md5Hash.Sum(nil))
			logs[path] = md5sum
		}
		return nil
	})
	return logs
}

func (s *SyncT) loadDest() (map[string]string, error) {
	logs := map[string]string{}
	data, err := s.Bucket.List(s.Prefix, "", "", 0)
	if err != nil {
		return nil, err
	}
	if data.IsTruncated == true {
		msg := "Results from S3 truncated and I don't yet know how to download next set of results, so we will exit to avoid invalidating results."
		return nil, errors.New(msg)
	}
	for i := range data.Contents {
		md5sum := strings.Trim(data.Contents[i].ETag, "\"")
		path := relativePath(s.Prefix, data.Contents[i].Key)
		logs[path] = md5sum
	}
	return logs, nil
}

func putLog(log transfer, bucket *s3.Bucket, dry bool) {
	data, err := ioutil.ReadFile(log.Src)
	if err != nil {
		// Error reading log
		fmt.Printf("Error reading source file %s:\n", log.Src)
		panic(err.Error())
	}

	contType := "binary/octet-stream"
	perm := s3.ACL("private")

	if dry {
		fmt.Printf("Starting sync of %s to bucket path %s...\n", log.Src, log.Dest)
	} else {
		fmt.Printf("Starting sync of %s to s3://%s/%s...\n", log.Src, bucket.Name, log.Dest)
		err = bucket.Put(log.Dest, data, contType, perm, s3.Options{})
		if err != nil {
			// Error uploading log to s3
			fmt.Printf("Sync of %s to s3://%s/%s failed:\n", log.Src, bucket.Name, log.Dest)
			panic(err.Error())
		}
	}
}

func syncFile(log transfer, bucket *s3.Bucket, workerChan chan empty, dry bool) {
	putLog(log, bucket, dry)
	<-workerChan
}

func workerSpawner(bucket *s3.Bucket, fileChan chan transfer, workerChan chan empty, dieChan chan empty, dry bool) {
	for {
		select {
		case file := <-fileChan:
			go syncFile(file, bucket, workerChan, dry)
		case <-dieChan:
			return
		}
	}
}

func (s *SyncT) syncLogs(src, dest map[string]string) error {
	fileChan := make(chan transfer)
	workerChan := make(chan empty, s.Threads)
	dieChan := make(chan empty)
	go workerSpawner(s.Bucket, fileChan, workerChan, dieChan, s.Dry)

	for log, _ := range src {
		if dest[log] != src[log] {
			srcPath := strings.Join([]string{s.Dir, log}, "/")
			destPath := strings.Join([]string{s.Prefix, log}, "/")
			workerChan <- empty{}
			fileChan <- transfer{srcPath, destPath}
		}
	}
	dieChan <- empty{}
	return nil
}

func (s *SyncT) Sync() error {
	err := s.validate()
	if err != nil {
		return err
	}

	var destLogs map[string]string
	if !s.NoAws {
		destLogs, err = s.loadDest()
		if err != nil {
			return err
		}
	}
	srcLogs := s.loadSrc()

	return s.syncLogs(srcLogs, destLogs)
}

<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"os"
)

func initRIO() *RObject {
	obj := &RObject{}
	obj.name = "RIO"
	obj.ivars = make(map[string]Object)
	obj.class = nil
	obj.methods = make(map[string]*RMethod)

	// RString method initialization
	obj.methods["readlines"] = &RMethod{gofunc: RIO_readlines}

	return obj
}

// IO.readlines(filename)
// [RString]
func RIO_readlines(vm *GobiesVM, receiver Object, v []Object) Object {
	filename := v[0].(*RObject).val.str

	obj := RArray_new(vm, receiver, nil)

	input, _ := os.Open(filename)
	defer input.Close()

	reader := bufio.NewReader(input)

	line, err := reader.ReadString('\n')
	for ; err == nil; line, err = reader.ReadString('\n') {
		dummy_obj := make([]Object, 1, 1)
		dummy_obj[0] = line
		rstr := RString_new(vm, receiver, dummy_obj)
		dummy_obj[0] = rstr
		RArray_append(vm, obj, dummy_obj)
	}

	return obj
}
<commit_msg>[IO] further improve readlines() performance against large file by reading entire file into memory<commit_after>package main

import (
	"io/ioutil"
	"strings"
)

func initRIO() *RObject {
	obj := &RObject{}
	obj.name = "RIO"
	obj.ivars = make(map[string]Object)
	obj.class = nil
	obj.methods = make(map[string]*RMethod)

	// RString method initialization
	obj.methods["readlines"] = &RMethod{gofunc: RIO_readlines}

	return obj
}

// IO.readlines(filename)
// [RString]
func RIO_readlines(vm *GobiesVM, receiver Object, v []Object) Object {
	filename := v[0].(*RObject).val.str

	obj := RArray_new(vm, receiver, nil)

	content, _ := ioutil.ReadFile(filename)
	str := string(content[:])

	lines := strings.Split(str, "\n")
	for _, line := range lines {
		dummy_obj := make([]Object, 1, 1)
		dummy_obj[0] = line
		rstr := RString_new(vm, receiver, dummy_obj)
		dummy_obj[0] = rstr
		RArray_append(vm, obj, dummy_obj)
	}

	return obj
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"strings"
)

func initRIO() *RObject {
	obj := &RObject{}
	obj.name = "RIO"
	obj.ivars = make(map[string]Object)
	obj.class = nil
	obj.methods = make(map[string]*RMethod)

	// RString method initialization
	obj.methods["readlines"] = &RMethod{gofunc: RIO_readlines}

	return obj
}

// IO.readlines(filename)
// [RString]
func RIO_readlines(vm *GobiesVM, receiver Object, v []Object) Object {
	filename := v[0].(*RObject).val.str

	obj := RArray_new(vm, receiver, nil)

	content, _ := ioutil.ReadFile(filename)
	str := string(content[:])

	lines := strings.Split(str, "\n")
	for _, line := range lines {
		dummy_obj := make([]Object, 1, 1)
		dummy_obj[0] = line
		rstr := RString_new(vm, receiver, dummy_obj)
		dummy_obj[0] = rstr
		RArray_append(vm, obj, dummy_obj)
	}

	return obj
}
<commit_msg>[IO] revert the problem which optimization in last commit caused different results<commit_after>package main

import (
	"io/ioutil"
	"strings"
)

func initRIO() *RObject {
	obj := &RObject{}
	obj.name = "RIO"
	obj.ivars = make(map[string]Object)
	obj.class = nil
	obj.methods = make(map[string]*RMethod)

	// RString method initialization
	obj.methods["readlines"] = &RMethod{gofunc: RIO_readlines}

	return obj
}

// IO.readlines(filename)
// [RString]
func RIO_readlines(vm *GobiesVM, receiver Object, v []Object) Object {
	filename := v[0].(*RObject).val.str

	obj := RArray_new(vm, receiver, nil)

	content, _ := ioutil.ReadFile(filename)
	str := string(content[:])

	lines := strings.SplitAfter(str, "\n")
	for _, line := range lines {
		dummy_obj := make([]Object, 1, 1)
		dummy_obj[0] = line
		rstr := RString_new(vm, receiver, dummy_obj)
		dummy_obj[0] = rstr
		RArray_append(vm, obj, dummy_obj)
	}

	return obj
}
<|endoftext|>


<commit_before>// Copyright 2020 Martin Holst Swende. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be found
// in the COPYING file.
//

package uint256

import (
	"bytes"
	"fmt"
	"math/big"
	"testing"
)

var (
	_ fmt.Formatter = &Int{} // Test if Int supports Formatter interface.
)

func TestFromBig(t *testing.T) {
	a := new(big.Int)
	b, o := FromBig(a)
	if o {
		t.Fatalf("conversion overflowed! big.Int %x", a.Bytes())
	}
	if exp, got := a.Bytes(), b.Bytes(); !bytes.Equal(got, exp) {
		t.Fatalf("got %x exp %x", got, exp)
	}

	a = big.NewInt(1)
	b, o = FromBig(a)
	if o {
		t.Fatalf("conversion overflowed! big.Int %x", a.Bytes())
	}
	if exp, got := a.Bytes(), b.Bytes(); !bytes.Equal(got, exp) {
		t.Fatalf("got %x exp %x", got, exp)
	}

	a = big.NewInt(0x1000000000000000)
	b, o = FromBig(a)
	if o {
		t.Fatalf("conversion overflowed! big.Int %x", a.Bytes())
	}
	if exp, got := a.Bytes(), b.Bytes(); !bytes.Equal(got, exp) {
		t.Fatalf("got %x exp %x", got, exp)
	}

	a = big.NewInt(0x1234)
	b, o = FromBig(a)
	if o {
		t.Fatalf("conversion overflowed! big.Int %x", a.Bytes())
	}
	if exp, got := a.Bytes(), b.Bytes(); !bytes.Equal(got, exp) {
		t.Fatalf("got %x exp %x", got, exp)
	}

	a = big.NewInt(1)
	a.Lsh(a, 256)

	b, o = FromBig(a)
	if !o {
		t.Fatalf("expected overflow")
	}
	if !b.Eq(new(Int)) {
		t.Fatalf("got %x exp 0", b.Bytes())
	}

	a.Sub(a, big.NewInt(1))
	b, o = FromBig(a)
	if o {
		t.Fatalf("conversion overflowed! big.Int %x", a.Bytes())
	}
	if exp, got := a.Bytes(), b.Bytes(); !bytes.Equal(got, exp) {
		t.Fatalf("got %x exp %x", got, exp)
	}
}

func TestFromBigOverflow(t *testing.T) {
	_, o := FromBig(new(big.Int).SetBytes(hex2Bytes("ababee444444444444ffcc333333333333ddaa222222222222bb8811111111111199")))
	if !o {
		t.Errorf("expected overflow, got %v", o)
	}
	_, o = FromBig(new(big.Int).SetBytes(hex2Bytes("ee444444444444ffcc333333333333ddaa222222222222bb8811111111111199")))
	if o {
		t.Errorf("expected no overflow, got %v", o)
	}
	b := new(big.Int).SetBytes(hex2Bytes("ee444444444444ffcc333333333333ddaa222222222222bb8811111111111199"))
	_, o = FromBig(b.Neg(b))
	if o {
		t.Errorf("expected no overflow, got %v", o)
	}
}

func TestToBig(t *testing.T) {

	if bigZero := new(Int).ToBig(); bigZero.Cmp(new(big.Int)) != 0 {
		t.Errorf("expected big.Int 0, got %x", bigZero)
	}

	for i := uint(0); i < 256; i++ {
		f := new(Int).SetUint64(1)
		f.Lsh(f, i)
		b := f.ToBig()
		expected := big.NewInt(1)
		expected.Lsh(expected, i)
		if b.Cmp(expected) != 0 {
			t.Fatalf("expected %x, got %x", expected, b)
		}
	}
}

func benchmarkSetFromBig(bench *testing.B, b *big.Int) Int {
	var f Int
	for i := 0; i < bench.N; i++ {
		f.SetFromBig(b)
	}
	return f
}

func BenchmarkSetFromBig(bench *testing.B) {
	param1 := big.NewInt(0xff)
	bench.Run("1word", func(bench *testing.B) { benchmarkSetFromBig(bench, param1) })

	param2 := new(big.Int).Lsh(param1, 64)
	bench.Run("2words", func(bench *testing.B) { benchmarkSetFromBig(bench, param2) })

	param3 := new(big.Int).Lsh(param2, 64)
	bench.Run("3words", func(bench *testing.B) { benchmarkSetFromBig(bench, param3) })

	param4 := new(big.Int).Lsh(param3, 64)
	bench.Run("4words", func(bench *testing.B) { benchmarkSetFromBig(bench, param4) })

	param5 := new(big.Int).Lsh(param4, 64)
	bench.Run("overflow", func(bench *testing.B) { benchmarkSetFromBig(bench, param5) })
}

func benchmarkToBig(bench *testing.B, f *Int) *big.Int {
	var b *big.Int
	for i := 0; i < bench.N; i++ {
		b = f.ToBig()
	}
	return b
}

func BenchmarkToBig(bench *testing.B) {
	param1 := new(Int).SetUint64(0xff)
	bench.Run("1word", func(bench *testing.B) { benchmarkToBig(bench, param1) })

	param2 := new(Int).Lsh(param1, 64)
	bench.Run("2words", func(bench *testing.B) { benchmarkToBig(bench, param2) })

	param3 := new(Int).Lsh(param2, 64)
	bench.Run("3words", func(bench *testing.B) { benchmarkToBig(bench, param3) })

	param4 := new(Int).Lsh(param3, 64)
	bench.Run("4words", func(bench *testing.B) { benchmarkToBig(bench, param4) })
}
<commit_msg>test: Add unit tests for (*Int).Format()<commit_after>// Copyright 2020 Martin Holst Swende. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be found
// in the COPYING file.
//

package uint256

import (
	"bytes"
	"fmt"
	"math/big"
	"testing"
)

var (
	_ fmt.Formatter = &Int{} // Test if Int supports Formatter interface.
)

func TestFromBig(t *testing.T) {
	a := new(big.Int)
	b, o := FromBig(a)
	if o {
		t.Fatalf("conversion overflowed! big.Int %x", a.Bytes())
	}
	if exp, got := a.Bytes(), b.Bytes(); !bytes.Equal(got, exp) {
		t.Fatalf("got %x exp %x", got, exp)
	}

	a = big.NewInt(1)
	b, o = FromBig(a)
	if o {
		t.Fatalf("conversion overflowed! big.Int %x", a.Bytes())
	}
	if exp, got := a.Bytes(), b.Bytes(); !bytes.Equal(got, exp) {
		t.Fatalf("got %x exp %x", got, exp)
	}

	a = big.NewInt(0x1000000000000000)
	b, o = FromBig(a)
	if o {
		t.Fatalf("conversion overflowed! big.Int %x", a.Bytes())
	}
	if exp, got := a.Bytes(), b.Bytes(); !bytes.Equal(got, exp) {
		t.Fatalf("got %x exp %x", got, exp)
	}

	a = big.NewInt(0x1234)
	b, o = FromBig(a)
	if o {
		t.Fatalf("conversion overflowed! big.Int %x", a.Bytes())
	}
	if exp, got := a.Bytes(), b.Bytes(); !bytes.Equal(got, exp) {
		t.Fatalf("got %x exp %x", got, exp)
	}

	a = big.NewInt(1)
	a.Lsh(a, 256)

	b, o = FromBig(a)
	if !o {
		t.Fatalf("expected overflow")
	}
	if !b.Eq(new(Int)) {
		t.Fatalf("got %x exp 0", b.Bytes())
	}

	a.Sub(a, big.NewInt(1))
	b, o = FromBig(a)
	if o {
		t.Fatalf("conversion overflowed! big.Int %x", a.Bytes())
	}
	if exp, got := a.Bytes(), b.Bytes(); !bytes.Equal(got, exp) {
		t.Fatalf("got %x exp %x", got, exp)
	}
}

func TestFromBigOverflow(t *testing.T) {
	_, o := FromBig(new(big.Int).SetBytes(hex2Bytes("ababee444444444444ffcc333333333333ddaa222222222222bb8811111111111199")))
	if !o {
		t.Errorf("expected overflow, got %v", o)
	}
	_, o = FromBig(new(big.Int).SetBytes(hex2Bytes("ee444444444444ffcc333333333333ddaa222222222222bb8811111111111199")))
	if o {
		t.Errorf("expected no overflow, got %v", o)
	}
	b := new(big.Int).SetBytes(hex2Bytes("ee444444444444ffcc333333333333ddaa222222222222bb8811111111111199"))
	_, o = FromBig(b.Neg(b))
	if o {
		t.Errorf("expected no overflow, got %v", o)
	}
}

func TestToBig(t *testing.T) {

	if bigZero := new(Int).ToBig(); bigZero.Cmp(new(big.Int)) != 0 {
		t.Errorf("expected big.Int 0, got %x", bigZero)
	}

	for i := uint(0); i < 256; i++ {
		f := new(Int).SetUint64(1)
		f.Lsh(f, i)
		b := f.ToBig()
		expected := big.NewInt(1)
		expected.Lsh(expected, i)
		if b.Cmp(expected) != 0 {
			t.Fatalf("expected %x, got %x", expected, b)
		}
	}
}

func benchmarkSetFromBig(bench *testing.B, b *big.Int) Int {
	var f Int
	for i := 0; i < bench.N; i++ {
		f.SetFromBig(b)
	}
	return f
}

func BenchmarkSetFromBig(bench *testing.B) {
	param1 := big.NewInt(0xff)
	bench.Run("1word", func(bench *testing.B) { benchmarkSetFromBig(bench, param1) })

	param2 := new(big.Int).Lsh(param1, 64)
	bench.Run("2words", func(bench *testing.B) { benchmarkSetFromBig(bench, param2) })

	param3 := new(big.Int).Lsh(param2, 64)
	bench.Run("3words", func(bench *testing.B) { benchmarkSetFromBig(bench, param3) })

	param4 := new(big.Int).Lsh(param3, 64)
	bench.Run("4words", func(bench *testing.B) { benchmarkSetFromBig(bench, param4) })

	param5 := new(big.Int).Lsh(param4, 64)
	bench.Run("overflow", func(bench *testing.B) { benchmarkSetFromBig(bench, param5) })
}

func benchmarkToBig(bench *testing.B, f *Int) *big.Int {
	var b *big.Int
	for i := 0; i < bench.N; i++ {
		b = f.ToBig()
	}
	return b
}

func BenchmarkToBig(bench *testing.B) {
	param1 := new(Int).SetUint64(0xff)
	bench.Run("1word", func(bench *testing.B) { benchmarkToBig(bench, param1) })

	param2 := new(Int).Lsh(param1, 64)
	bench.Run("2words", func(bench *testing.B) { benchmarkToBig(bench, param2) })

	param3 := new(Int).Lsh(param2, 64)
	bench.Run("3words", func(bench *testing.B) { benchmarkToBig(bench, param3) })

	param4 := new(Int).Lsh(param3, 64)
	bench.Run("4words", func(bench *testing.B) { benchmarkToBig(bench, param4) })
}

func TestFormat(t *testing.T) {
	testCases := []string{
		"0",
		"1",
		"ffeeddccbbaa99887766554433221100ffeeddccbbaa99887766554433221100",
	}

	for i := 0; i < len(testCases); i++ {
		expected := testCases[i]
		b, _ := new(big.Int).SetString(expected, 16)
		f, o := FromBig(b)
		if o {
			t.Fatalf("too big test case %s", expected)
		}
		s := fmt.Sprintf("%x", f)
		if s != expected {
			t.Errorf("Invalid format conversion to hex: %s, expected %s", s, expected)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"reflect"
	"testing"
)

type testCase struct {
	in  []string
	out []string
}

func testCases() map[string]testCase {
	m := map[string]testCase{
		"short": {
			in:  short(),
			out: []string{".hi", "My name is Omar", "\"123\""},
		},
		"long": {
			in:  long(),
			out: []string{".hi I'm the real Slim ShadyMy name is Omar", "hello", "world", "\"123\"", "a"},
		},
	}
	return m
}

func testFunc(t *testing.T, fname string, fn func([]string) []string) {
	for name, tc := range testCases() {
		name = fmt.Sprintf("%s(%s)", fname, name)
		t.Run(name, func(t *testing.T) {
			got := fn(tc.in)
			want := tc.out
			if !reflect.DeepEqual(got, want) {
				t.Errorf("got %v; want %v", got, want)
			}
		})
	}
}

func TestOriginal(t *testing.T) {
	testFunc(t, "original", original)
}
<commit_msg>fromquotes: add TestBetter<commit_after>package main

import (
	"fmt"
	"reflect"
	"testing"
)

type testCase struct {
	in  []string
	out []string
}

func testCases() map[string]testCase {
	m := map[string]testCase{
		"short": {
			in:  short(),
			out: []string{".hi", "My name is Omar", "\"123\""},
		},
		"long": {
			in:  long(),
			out: []string{".hi I'm the real Slim ShadyMy name is Omar", "hello", "world", "\"123\"", "a"},
		},
	}
	return m
}

func testFunc(t *testing.T, fname string, fn func([]string) []string) {
	for name, tc := range testCases() {
		name = fmt.Sprintf("%s(%s)", fname, name)
		t.Run(name, func(t *testing.T) {
			got := fn(tc.in)
			want := tc.out
			if !reflect.DeepEqual(got, want) {
				t.Errorf("got %v; want %v", got, want)
			}
		})
	}
}

func TestOriginal(t *testing.T) {
	testFunc(t, "original", original)
}

func TestBetter(t *testing.T) {
	testFunc(t, "better", better)
}
<|endoftext|>


<commit_before>// Package thread implements simple routines for forcing tasks to be executed
// on certain threads. It is useful for functions that must be run on the same
// thread, or code that must be run on the main thread (e.g. GUI code under
// OS X).
package thread

import (
    "runtime"
)

// Thread is a handle for executing code on an OS thread.
type Thread struct {
    tasks chan func()
    done  chan struct{}
}

// New creates a new Thread. The thread will not process tasks until Run is
// called.
func New() *Thread {
    return &Thread{
        // Buffering tasks should reduce the required goroutine switches for
        // single-goroutine-threaded programs.
        make(chan func(), 1),
        make(chan struct{}),
    }
}

// Run causes the current thread to execute all functions passed to Do. The
// thread might still be used randomly by other goroutines. This calls
// runtime.LockOSThread but does not unlock it again.
func (thread *Thread) Run() {
    runtime.LockOSThread()
    for task := range thread.tasks {
        task()
        thread.done <- struct{}{}
    }
}

// Do causes the task to be executed on the Thread and waits for the task to
// finish. Do panics, if
func (thread *Thread) Do(task func()) {
    thread.tasks <- task
    <-thread.done
}

// Stop makes the Thread stop after the next task has completed. It does not
// wait for the queue to empty. To do this, you can call Do with thread.Stop.
func (thread *Thread) Stop() {
    close(thread.tasks)
}
<commit_msg>Completed sentence from previous commit in documentation.<commit_after>// Package thread implements simple routines for forcing tasks to be executed
// on certain threads. It is useful for functions that must be run on the same
// thread, or code that must be run on the main thread (e.g. GUI code under
// OS X).
package thread

import (
    "runtime"
)

// Thread is a handle for executing code on an OS thread.
type Thread struct {
    tasks chan func()
    done  chan struct{}
}

// New creates a new Thread. The thread will not process tasks until Run is
// called.
func New() *Thread {
    return &Thread{
        // Buffering tasks should reduce the required goroutine switches for
        // single-goroutine-threaded programs.
        make(chan func(), 1),
        make(chan struct{}),
    }
}

// Run causes the current thread to execute all functions passed to Do. The
// thread might still be used randomly by other goroutines. This calls
// runtime.LockOSThread but does not unlock it again.
func (thread *Thread) Run() {
    runtime.LockOSThread()
    for task := range thread.tasks {
        task()
        thread.done <- struct{}{}
    }
}

// Do causes the task to be executed on the Thread and waits for the task to
// finish. Do panics, if the thread has been stopped with Stop.
func (thread *Thread) Do(task func()) {
    thread.tasks <- task
    <-thread.done
}

// Stop makes the Thread stop after the next task has completed. It does not
// wait for the queue to empty. To do this, you can call Do with thread.Stop.
func (thread *Thread) Stop() {
    close(thread.tasks)
}
<|endoftext|>


<commit_before>package atomas
import (
	"net/http"
	"strings"
	"fmt"
)

func CreateGetGameHandler(games map[string]GameDTO) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		path := strings.Split(r.URL.Path, "/")
		gameId := path[len(path) - 1]
		fmt.Fprint(w, ToJsonString(games[gameId]))
	}
}<commit_msg>not found for invalid id<commit_after>package atomas
import (
	"net/http"
	"strings"
	"fmt"
)

func CreateGetGameHandler(games map[string]GameDTO) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		path := strings.Split(r.URL.Path, "/")
		gameId := path[len(path) - 1]
		game := games[gameId]
		if (game.Id == gameId) {
			fmt.Fprint(w, ToJsonString(game))
		}else {
			http.NotFound(w, r)
		}
	}
}<|endoftext|>


<commit_before>package audio

import (
	"code.google.com/p/portaudio-go/portaudio"
	"encoding/binary"
	"os"
	"time"
)

type Recording struct {
	path       string
	streamInfo *portaudio.StreamParameters
	stream     *portaudio.Stream
	startedAt  time.Time
	file       *os.File
	err        error
	channels   int
	sampleSize int
	buffer     portaudio.Buffer
}

const (
	aiffFORMSize       = 4
	aiffCOMMSize       = 8 + 18
	aiffSSNDHeaderSize = 16
	paBufferSize       = 128
)

func (r *Recording) Start() error {
	r.file, err = os.Create(r.path)
	f := r.file
	if err != nil {
		return err
	}
	// Form Chunk
	_, err = f.WriteString("FORM")
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.BigEndian, int32(0))
	if err != nil {
		return err
	}
	_, err = f.WriteString("AIFF")
	if err != nil {
		return err
	}
	// Common Chunk
	_, err = f.WriteString("COMM")
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.BigEndian, int32(18))
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.BigEndian, int16(r.streamInfo.Input.Channels))
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.BigEndian, int32(0))
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.BigEndian, int16(32))
	if err != nil {
		return err
	}
	_, err = f.Write([]byte{0x40, 0x0e, 0xac, 0x44, 0, 0, 0, 0, 0, 0})
	if err != nil {
		return err
	}
	// Sound Data Chunk
	_, err = f.WriteString("SSND")
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.BigEndian, int32(0))
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.BigEndian, int32(0))
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.BigEndian, int32(0))
	if err != nil {
		return err
	}
	r.startedAt = time.Now()
	switch sampleSize {
	case 32:
		r.buffer = make([][]int32, r.channels)
		for c := 0; c < r.channels; c++ {
			r.buffer[c] = make([]int32, paBufferSize)
		}
	case 24:
		r.buffer = make([][]Int24, r.channels)
		for _, c := range(r.buffer) {
			c = make([]Int24, paBufferSize)
		}
	case 16:
		r.buffer = make([][]int16, r.channels)
		for _, c := range(r.buffer) {

		}
	}
	go r.run()
	return nil
}

func (r *Recording) run() {
	frameCount := 0
	f := r.file
	defer func() {
		if r.err != nil {
			return
		}
		bytesPerSample = r.sampleSize / 8
		audioSize = framecount * r.channels * bytesPerSample
		totalSize = aiffCOMMSize + aiffSSNDHeaderSize + audioSize + aiffFORMSize
		_, r.err = f.Seek(4, 0)
		if r.err != nil {
			return
		}
		r.err = binary.Write(f, binary.BigEndian, int32(totalSize))
		if r.err != nil {
			return
		}
		_, r.err = f.Seek(22, 0)
		if r.err != nil {
			return
		}
		r.err = binary.Write(f, binary.BigEndian, int32(frameCount))
		if r.err != nil {
			return
		}
		_, r.err = f.Seek(42, 0)
		if r.err != nil {
			return
		}
		r.err = binary.Write(f, binary.BigEndian, int32(audioSize+8))
		if r.err != nil {
			return
		}
		r.err = f.Close()
	}()

}
<commit_msg>Changes to recording system<commit_after>package audio

import (
	"code.google.com/p/portaudio-go/portaudio"
	"encoding/binary"
	"os"
	"time"
)

type Recording struct {
	path       string
	streamInfo *portaudio.StreamParameters
	stream     *portaudio.Stream
	startedAt  time.Time
	file       *os.File
	err        error
	channels   int
	sampleSize int
	buffer     portaudio.Buffer
}

const (
	aiffFORMSize       = 4
	aiffCOMMSize       = 8 + 18
	aiffSSNDHeaderSize = 16
	paBufferSize       = 128
)

func (r *Recording) Start() error {
	r.file, err = os.Create(r.path)
	f := r.file
	if err != nil {
		return err
	}
	// Form Chunk
	_, err = f.WriteString("FORM")
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.BigEndian, int32(0))
	if err != nil {
		return err
	}
	_, err = f.WriteString("AIFF")
	if err != nil {
		return err
	}
	// Common Chunk
	_, err = f.WriteString("COMM")
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.BigEndian, int32(18))
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.BigEndian, int16(r.streamInfo.Input.Channels))
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.BigEndian, int32(0))
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.BigEndian, int16(32))
	if err != nil {
		return err
	}
	_, err = f.Write([]byte{0x40, 0x0e, 0xac, 0x44, 0, 0, 0, 0, 0, 0})
	if err != nil {
		return err
	}
	// Sound Data Chunk
	_, err = f.WriteString("SSND")
y	if err != nil {
		return err
	}
	err = binary.Write(f, binary.BigEndian, int32(0))
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.BigEndian, int32(0))
	if err != nil {
		return err
	}
	err = binary.Write(f, binary.BigEndian, int32(0))
	if err != nil {
		return err
	}
	r.startedAt = time.Now()
	switch sampleSize {
	case 32:
		r.buffer = make([][]int32, r.channels)
		for c := 0; c < r.channels; c++ {
			r.buffer[c] = make([]int32, paBufferSize)
		}
	case 24:
		r.buffer = make([][]Int24, r.channels)
		for _, c := range(r.buffer) {
			c = make([]Int24, paBufferSize)
		}
	case 16:
		r.buffer = make([][]int16, r.channels)
		for _, c := range(r.buffer) {
			c = make([]int16, paBufferSize)
		}
	case 8:
		r.buffer = make([][]int8, r.channels)
		for _, c := range(r.buffer) {
			c = make([]int8, paBufferSize)
		}
	default:
		r.err = error("Invalid sample size")
		return r.err
	}
	go r.run()
	return nil
}

func (r *Recording) run() {
	frameCount := 0
	f := r.file
	defer func() {
		if r.err != nil {
			return
		}
		bytesPerSample = r.sampleSize / 8
		audioSize = framecount * r.channels * bytesPerSample
		totalSize = aiffCOMMSize + aiffSSNDHeaderSize + audioSize + aiffFORMSize
		_, r.err = f.Seek(4, 0)
		if r.err != nil {
			return
		}
		r.err = binary.Write(f, binary.BigEndian, int32(totalSize))
		if r.err != nil {
			return
		}
		_, r.err = f.Seek(22, 0)
		if r.err != nil {
			return
		}
		r.err = binary.Write(f, binary.BigEndian, int32(frameCount))
		if r.err != nil {
			return
		}
		_, r.err = f.Seek(42, 0)
		if r.err != nil {
			return
		}
		r.err = binary.Write(f, binary.BigEndian, int32(audioSize+8))
		if r.err != nil {
			return
		}
		r.err = f.Close()
	}()

}
<|endoftext|>


<commit_before>// Copyright 2016 CloudByte, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vsm

import (
	"github.com/openebs/openebs/api/server/router"
)

type validationError struct {
	error
}

func (validationError) IsValidationError() bool {
	return true
}

// vsmRouter is a router to talk with the server logic of VSM
type vsmRouter struct {
	backend Backend
	routes  []router.Route
}

// This initializes the routes in vsm router
func (r *vsmRouter) initRoutes() {
	r.routes = []router.Route{
		// HEAD
		//router.NewHeadRoute("/vsm/{name:.*}/archive", r.headVsmArchive),
		// GET
		router.NewGetRoute("/vsm/lsjson", r.getVsmLsJSON),
		// POST
		//router.NewPostRoute("/vsm/create", r.postVsmCreate),
		// PUT
		//router.NewPutRoute("/vsm/{name:.*}/archive", r.putVsmArchive),
		// DELETE
		//router.NewDeleteRoute("/vsm/{name:.*}", r.deleteVsm),
	}
}

// This initializes a new vsm router
func NewRouter(b Backend) router.Router {
	r := &vsmRouter{
		backend: b,
	}
	r.initRoutes()
	return r
}

// This returns the available routes to the vsm controller
func (r *vsmRouter) Routes() []router.Route {
	return r.routes
}
<commit_msg>Uncommented the required ^CM create command<commit_after>// Copyright 2016 CloudByte, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vsm

import (
	"github.com/openebs/openebs/api/server/router"
)

type validationError struct {
	error
}

func (validationError) IsValidationError() bool {
	return true
}

// vsmRouter is a router to talk with the server logic of VSM
type vsmRouter struct {
	backend Backend
	routes  []router.Route
}

// This initializes the routes in vsm router
func (r *vsmRouter) initRoutes() {
	r.routes = []router.Route{
		// HEAD
		//router.NewHeadRoute("/vsm/{name:.*}/archive", r.headVsmArchive),
		// GET
		router.NewGetRoute("/vsm/lsjson", r.getVsmLsJSON),
		// POST
		router.NewPostRoute("/vsm/create", r.postVsmCreate),
		// PUT
		//router.NewPutRoute("/vsm/{name:.*}/archive", r.putVsmArchive),
		// DELETE
		//router.NewDeleteRoute("/vsm/{name:.*}", r.deleteVsm),
	}
}

// This initializes a new vsm router
func NewRouter(b Backend) router.Router {
	r := &vsmRouter{
		backend: b,
	}
	r.initRoutes()
	return r
}

// This returns the available routes to the vsm controller
func (r *vsmRouter) Routes() []router.Route {
	return r.routes
}
<|endoftext|>


<commit_before>package util

import (
	"bytes"
	"io/ioutil"
	"os"
	"os/exec"
	"syscall"

	"github.com/fatih/color"
)

const defaultFailedCode = 1

// Set up the output streams (and colors) to stream command output if verbose is configured
func StreamCommand(cmd *exec.Cmd) error {
	return RunCommand(cmd, false)
}

// Set up the output streams (and colors) to stream command output regardless of verbosity
func ForceStreamCommand(cmd *exec.Cmd) error {
	return RunCommand(cmd, true)
}

// Run the command
func RunCommand(cmd *exec.Cmd, forceOutput bool) error {
	cmd.Stderr = os.Stderr
	if Logger().IsVerbose || forceOutput {
		cmd.Stdout = os.Stdout
	} else {
		cmd.Stdout = ioutil.Discard
	}

	color.Set(color.FgCyan)
	err := cmd.Run()
	color.Unset()
	return err
}

// This is similar to ForceStreamCommand in that it will issue all output
// regardless of verbose mode. Further, this version of the command captures the
// exit status of any executed command. This function is intended to simulate
// native execution of the command passed to it.
//
// @todo streaming the output instead of buffering until completion.
func PassthruCommand(cmd *exec.Cmd) (stdout string, stderr string, exitCode int) {
	var outbuf, errbuf bytes.Buffer

	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout

	err := cmd.Run()
	stdout = outbuf.String()
	stderr = errbuf.String()

	if err != nil {
		// Try to get the exit code.
		if exitError, ok := err.(*exec.ExitError); ok {
			ws := exitError.Sys().(syscall.WaitStatus)
			exitCode = ws.ExitStatus()
		} else {
			// This will happen (in OSX) if `name` is not available in $PATH,
			// in this situation, exit code could not be get, and stderr will be
			// empty string very likely, so we use the default fail code, and format err
			// to string and set to stderr
			exitCode = defaultFailedCode
			if stderr == "" {
				stderr = err.Error()
			}
		}
	} else {
		// Success, exitCode should be 0.
		ws := cmd.ProcessState.Sys().(syscall.WaitStatus)
		exitCode = ws.ExitStatus()
	}

	return
}
<commit_msg>Enable running interactive commands such as cli container.<commit_after>package util

import (
	"bytes"
	"io/ioutil"
	"os"
	"os/exec"
	"syscall"

	"github.com/fatih/color"
)

const defaultFailedCode = 1

// Set up the output streams (and colors) to stream command output if verbose is configured
func StreamCommand(cmd *exec.Cmd) error {
	return RunCommand(cmd, false)
}

// Set up the output streams (and colors) to stream command output regardless of verbosity
func ForceStreamCommand(cmd *exec.Cmd) error {
	return RunCommand(cmd, true)
}

// Run the command
func RunCommand(cmd *exec.Cmd, forceOutput bool) error {
	cmd.Stderr = os.Stderr
	if Logger().IsVerbose || forceOutput {
		cmd.Stdout = os.Stdout
	} else {
		cmd.Stdout = ioutil.Discard
	}

	color.Set(color.FgCyan)
	err := cmd.Run()
	color.Unset()
	return err
}

// This is similar to ForceStreamCommand in that it will issue all output
// regardless of verbose mode. Further, this version of the command captures the
// exit status of any executed command. This function is intended to simulate
// native execution of the command passed to it.
//
// @todo streaming the output instead of buffering until completion.
func PassthruCommand(cmd *exec.Cmd) (stdout string, stderr string, exitCode int) {
	var outbuf, errbuf bytes.Buffer

	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
        cmd.Stdin = os.Stdin

	err := cmd.Run()
	stdout = outbuf.String()
	stderr = errbuf.String()

	if err != nil {
		// Try to get the exit code.
		if exitError, ok := err.(*exec.ExitError); ok {
			ws := exitError.Sys().(syscall.WaitStatus)
			exitCode = ws.ExitStatus()
		} else {
			// This will happen (in OSX) if `name` is not available in $PATH,
			// in this situation, exit code could not be get, and stderr will be
			// empty string very likely, so we use the default fail code, and format err
			// to string and set to stderr
			exitCode = defaultFailedCode
			if stderr == "" {
				stderr = err.Error()
			}
		}
	} else {
		// Success, exitCode should be 0.
		ws := cmd.ProcessState.Sys().(syscall.WaitStatus)
		exitCode = ws.ExitStatus()
	}

	return
}
<|endoftext|>


<commit_before>// Copyright © 2016-2021 Wei Shen <shenwei356@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"bufio"
	"fmt"
	"sort"
	"strconv"
	"strings"

	"github.com/shenwei356/xopen"
	"github.com/spf13/cobra"
)

// camiFilterCmd represents the fx2tab command
var camiFilterCmd = &cobra.Command{
	Use:   "cami-filter",
	Short: "Remove taxa of given TaxIds and their descendants in CAMI metagenomic profile table",
	Long: `Remove taxa of given TaxIds and their descendants in CAMI metagenomic profile table

`,
	Run: func(cmd *cobra.Command, args []string) {
		config := getConfigs(cmd)

		taxidsStr := getFlagStringSlice(cmd, "taxids")
		// if len(taxidsStr) == 0 {
		// 	checkError(fmt.Errorf("flag --taxids needed"))
		// }

		filter := make(map[string]interface{}, len(taxidsStr))
		for _, t := range taxidsStr {
			filter[t] = struct{}{}
		}

		fieldTaxid := getFlagPositiveInt(cmd, "field-taxid") - 1
		fieldRank := getFlagPositiveInt(cmd, "field-rank") - 1
		fieldTaxpath := getFlagPositiveInt(cmd, "field-taxpath") - 1
		fieldTaxpathSN := getFlagPositiveInt(cmd, "field-taxpathsn") - 1
		fieldPercentage := getFlagPositiveInt(cmd, "field-percentage") - 1

		taxidSep := getFlagString(cmd, "taxid-sep")
		if taxidSep == "" {
			checkError(fmt.Errorf("flag --taxid-sep needed and should not be empty"))
		}

		files := getFileList(args)

		if len(files) > 1 {
			checkError(fmt.Errorf("only one input file allowed"))
		}

		if len(files) == 1 && isStdin(files[0]) && !xopen.IsStdin() {
			checkError(fmt.Errorf("stdin not detected"))
		}

		showRanks := getFlagStringSlice(cmd, "show-rank")

		showRanksMap := make(map[string]interface{}, 128)
		for _, _rank := range showRanks {
			showRanksMap[_rank] = struct{}{}
		}
		rankOrder := make(map[string]int, len(showRanks))
		for _i, _r := range showRanks {
			rankOrder[_r] = _i
		}

		leavesRanks := getFlagStringSlice(cmd, "leave-ranks")
		leavesRanksMap := make(map[string]interface{}, len(leavesRanks))
		for _, r := range leavesRanks {
			leavesRanksMap[r] = struct{}{}
		}

		// ----------------------------------------------------------------

		idx := []int{fieldTaxid, fieldRank, fieldTaxpath, fieldTaxpathSN, fieldPercentage}

		maxField := fieldTaxid
		for _, i := range idx[1:] {
			if i > maxField {
				maxField = i
			}
		}
		maxField++

		n := maxField + 1

		file := files[0]

		fh, err := xopen.Ropen(file)
		checkError(err)

		outfh, err := xopen.Wopen(config.OutFile)
		checkError(err)
		defer outfh.Close()

		scanner := bufio.NewScanner(fh)

		items := make([]string, n)
		var line string
		var _taxid int
		var taxid uint32
		var taxids []string
		var taxidsUint []uint32
		var percenage float64
		var rank, taxpath, taxpathsn string
		var hasData bool

		var taxidS string
		var ok bool
		var skipThis bool

		rankMap := make(map[uint32]string, 1024)
		meta := make([]string, 0, 8)

		targets := make([]*Target, 0, 512)

		for scanner.Scan() {
			line = scanner.Text()
			if line == "" {
				continue
			}

			// meta date
			if line[0] == '#' || line[0] == '@' {
				if hasData { // new record, need to summarize and output
					targets1 := filterLeaves(rankMap, leavesRanksMap, targets)

					profile := generateProfile2(targets, targets1)

					nodes := make([]*ProfileNode, 0, len(profile))
					for _, node := range profile {
						nodes = append(nodes, node)
					}

					sort.Slice(nodes, func(i, j int) bool {
						if rankOrder[nodes[i].Rank] < rankOrder[nodes[j].Rank] {
							return true
						}
						if rankOrder[nodes[i].Rank] == rankOrder[nodes[j].Rank] {
							return nodes[i].Abundance > nodes[j].Abundance
						}
						return false
					})

					for _, line = range meta {
						outfh.WriteString(line + "\n")
					}
					for _, node := range nodes {
						taxids = taxids[:0]
						for _, taxid = range node.LineageTaxids {
							taxids = append(taxids, strconv.Itoa(int(taxid)))
						}

						fmt.Fprintf(outfh, "%d\t%s\t%s\t%s\t%.15f\n",
							node.Taxid,
							node.Rank,
							strings.Join(taxids, taxidSep),
							strings.Join(node.LineageNames, taxidSep),
							node.Abundance,
						)
					}

					rankMap = make(map[uint32]string, 1024)
					meta = meta[:0]
					targets = targets[:0]
					hasData = false
					continue
				}

				meta = append(meta, line)
				continue
			}

			stringSplitN(line, "\t", n, &items)
			if len(items) < maxField {
				continue
			}

			percenage, err = strconv.ParseFloat(items[fieldPercentage], 64)
			if err != nil {
				checkError(fmt.Errorf("failed to parse abundance: %s", items[fieldPercentage]))
			}

			if percenage == 0 {
				continue
			}

			hasData = true

			_taxid, err = strconv.Atoi(items[fieldTaxid])
			if err != nil {
				checkError(fmt.Errorf("failed to parse taxid: %s", items[fieldTaxid]))
			}
			taxid = uint32(_taxid)

			rank = items[fieldRank]
			rankMap[taxid] = rank
			taxpath = items[fieldTaxpath]
			taxpathsn = items[fieldTaxpathSN]

			taxids = strings.Split(taxpath, taxidSep)
			skipThis = false
			for _, taxidS = range taxids {
				if _, ok = filter[taxidS]; ok {
					skipThis = true
					break
				}
			}
			if skipThis {
				continue
			}

			taxidsUint = make([]uint32, 0, len(taxids))
			for _, taxidS = range taxids {
				_taxid, err = strconv.Atoi(taxidS)
				if err != nil {
					checkError(fmt.Errorf("failed to parse taxid: %s", taxidS))
				}
				taxidsUint = append(taxidsUint, uint32(_taxid))
			}

			targets = append(targets, &Target{
				Taxid:     taxid,
				Abundance: percenage,

				Rank:          rank,
				TaxonName:     "",
				LineageNames:  strings.Split(taxpathsn, taxidSep),
				LineageTaxids: taxids,

				CompleteLineageTaxids: taxidsUint,
			})

		}

		if err := scanner.Err(); err != nil {
			checkError(err)
		}
		checkError(fh.Close())

		if hasData { // new record, need to summarize and output
			targets1 := filterLeaves(rankMap, leavesRanksMap, targets)

			profile := generateProfile2(targets, targets1)

			nodes := make([]*ProfileNode, 0, len(profile))
			for _, node := range profile {
				nodes = append(nodes, node)
			}

			sort.Slice(nodes, func(i, j int) bool {
				if rankOrder[nodes[i].Rank] < rankOrder[nodes[j].Rank] {
					return true
				}
				if rankOrder[nodes[i].Rank] == rankOrder[nodes[j].Rank] {
					return nodes[i].Abundance > nodes[j].Abundance
				}
				return false
			})

			for _, line = range meta {
				outfh.WriteString(line + "\n")
			}
			for _, node := range nodes {
				taxids = taxids[:0]
				for _, taxid = range node.LineageTaxids {
					taxids = append(taxids, strconv.Itoa(int(taxid)))
				}

				fmt.Fprintf(outfh, "%d\t%s\t%s\t%s\t%.15f\n",
					node.Taxid,
					node.Rank,
					strings.Join(taxids, taxidSep),
					strings.Join(node.LineageNames, taxidSep),
					node.Abundance,
				)
			}
		}

	},
}

func generateProfile2(targets0, targets []*Target) map[uint32]*ProfileNode {

	targetsMap := make(map[uint32]*Target, len(targets0))
	for _, target := range targets0 {
		targetsMap[target.Taxid] = target
	}

	profile := make(map[uint32]*ProfileNode, len(targets))

	var target0 *Target
	for _, target := range targets {
		for _, taxid := range target.CompleteLineageTaxids {
			if node, ok := profile[taxid]; !ok {
				target0 = targetsMap[taxid]

				profile[taxid] = &ProfileNode{
					Taxid:         taxid,
					Rank:          target0.Rank,
					TaxonName:     target0.TaxonName,
					LineageNames:  target0.LineageNames,
					LineageTaxids: target0.CompleteLineageTaxids,

					Abundance: target.Abundance,
				}
			} else {
				node.Abundance += target.Abundance
			}
		}
	}

	return profile
}

func filterLeaves(rankMap map[uint32]string, leavesRanksMap map[string]interface{}, targets []*Target) []*Target {

	targetsMap := make(map[uint32]*Target, len(targets))
	tree := make(map[uint32]map[uint32]uint32, 1024)

	var i int
	var taxidP, taxid uint32
	var ok bool
	for _, target := range targets {
		targetsMap[target.Taxid] = target

		for i, taxid = range target.CompleteLineageTaxids {
			if i == 0 {
				taxidP = 1
			} else {
				taxidP = target.CompleteLineageTaxids[i-1]
			}

			if _, ok = tree[taxidP]; !ok {
				tree[taxidP] = make(map[uint32]uint32, 8)
			}
			tree[taxidP][taxid] = target.Taxid
		}
	}
	// leaves := make([]uint32, 0, 1024)
	leaves := make([]*Target, 0, 1024)
	for _, m := range tree {
		for taxid = range m {
			if _, ok = tree[taxid]; !ok {
				if _, ok = leavesRanksMap[rankMap[taxid]]; ok {
					// leaves = append(leaves, taxid)
					leaves = append(leaves, targetsMap[m[taxid]])
				}
			}
		}
	}

	// recompute abundance
	var sum float64
	for _, target := range leaves {
		sum += target.Abundance
	}

	for _, target := range leaves {
		target.Abundance = target.Abundance / sum * 100
	}

	return leaves
}

func init() {
	RootCmd.AddCommand(camiFilterCmd)

	camiFilterCmd.Flags().IntP("field-taxid", "", 1, "field index of taxid")
	camiFilterCmd.Flags().IntP("field-rank", "", 2, "field index of taxid")
	camiFilterCmd.Flags().IntP("field-taxpath", "", 3, "field index of TAXPATH")
	camiFilterCmd.Flags().IntP("field-taxpathsn", "", 4, "field index of TAXPATHSN")
	camiFilterCmd.Flags().IntP("field-percentage", "", 5, "field index of PERCENTAGE")

	camiFilterCmd.Flags().StringP("taxid-sep", "", "|", "separator of taxid in TAXPATH and TAXPATHSN")

	camiFilterCmd.Flags().StringSliceP("taxids", "t", []string{}, "the parent taxid(s) to filter out")

	camiFilterCmd.Flags().StringSliceP("show-rank", "", []string{"superkingdom", "phylum", "class", "order", "family", "genus", "species", "strain"}, "only show TaxIds and names of these ranks")
	camiFilterCmd.Flags().StringSliceP("leave-ranks", "", []string{"species", "strain"}, "only consider leaves at these ranks")
}
<commit_msg>cami-filter: fix missing sample id<commit_after>// Copyright © 2016-2021 Wei Shen <shenwei356@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"bufio"
	"fmt"
	"sort"
	"strconv"
	"strings"

	"github.com/shenwei356/xopen"
	"github.com/spf13/cobra"
)

// camiFilterCmd represents the fx2tab command
var camiFilterCmd = &cobra.Command{
	Use:   "cami-filter",
	Short: "Remove taxa of given TaxIds and their descendants in CAMI metagenomic profile table",
	Long: `Remove taxa of given TaxIds and their descendants in CAMI metagenomic profile table

`,
	Run: func(cmd *cobra.Command, args []string) {
		config := getConfigs(cmd)

		taxidsStr := getFlagStringSlice(cmd, "taxids")
		// if len(taxidsStr) == 0 {
		// 	checkError(fmt.Errorf("flag --taxids needed"))
		// }

		filter := make(map[string]interface{}, len(taxidsStr))
		for _, t := range taxidsStr {
			filter[t] = struct{}{}
		}

		fieldTaxid := getFlagPositiveInt(cmd, "field-taxid") - 1
		fieldRank := getFlagPositiveInt(cmd, "field-rank") - 1
		fieldTaxpath := getFlagPositiveInt(cmd, "field-taxpath") - 1
		fieldTaxpathSN := getFlagPositiveInt(cmd, "field-taxpathsn") - 1
		fieldPercentage := getFlagPositiveInt(cmd, "field-percentage") - 1

		taxidSep := getFlagString(cmd, "taxid-sep")
		if taxidSep == "" {
			checkError(fmt.Errorf("flag --taxid-sep needed and should not be empty"))
		}

		files := getFileList(args)

		if len(files) > 1 {
			checkError(fmt.Errorf("only one input file allowed"))
		}

		if len(files) == 1 && isStdin(files[0]) && !xopen.IsStdin() {
			checkError(fmt.Errorf("stdin not detected"))
		}

		showRanks := getFlagStringSlice(cmd, "show-rank")

		showRanksMap := make(map[string]interface{}, 128)
		for _, _rank := range showRanks {
			showRanksMap[_rank] = struct{}{}
		}
		rankOrder := make(map[string]int, len(showRanks))
		for _i, _r := range showRanks {
			rankOrder[_r] = _i
		}

		leavesRanks := getFlagStringSlice(cmd, "leave-ranks")
		leavesRanksMap := make(map[string]interface{}, len(leavesRanks))
		for _, r := range leavesRanks {
			leavesRanksMap[r] = struct{}{}
		}

		// ----------------------------------------------------------------

		idx := []int{fieldTaxid, fieldRank, fieldTaxpath, fieldTaxpathSN, fieldPercentage}

		maxField := fieldTaxid
		for _, i := range idx[1:] {
			if i > maxField {
				maxField = i
			}
		}
		maxField++

		n := maxField + 1

		file := files[0]

		fh, err := xopen.Ropen(file)
		checkError(err)

		outfh, err := xopen.Wopen(config.OutFile)
		checkError(err)
		defer outfh.Close()

		scanner := bufio.NewScanner(fh)

		items := make([]string, n)
		var _line, line string
		var _taxid int
		var taxid uint32
		var taxids []string
		var taxidsUint []uint32
		var percenage float64
		var rank, taxpath, taxpathsn string
		var hasData bool

		var taxidS string
		var ok bool
		var skipThis bool

		rankMap := make(map[uint32]string, 1024)
		meta := make([]string, 0, 8)

		targets := make([]*Target, 0, 512)

		for scanner.Scan() {
			line = scanner.Text()
			if line == "" {
				continue
			}

			// meta date
			if line[0] == '#' || line[0] == '@' {
				if hasData { // new record, need to summarize and output
					targets1 := filterLeaves(rankMap, leavesRanksMap, targets)

					profile := generateProfile2(targets, targets1)

					nodes := make([]*ProfileNode, 0, len(profile))
					for _, node := range profile {
						nodes = append(nodes, node)
					}

					sort.Slice(nodes, func(i, j int) bool {
						if rankOrder[nodes[i].Rank] < rankOrder[nodes[j].Rank] {
							return true
						}
						if rankOrder[nodes[i].Rank] == rankOrder[nodes[j].Rank] {
							return nodes[i].Abundance > nodes[j].Abundance
						}
						return false
					})

					for _, _line = range meta {
						outfh.WriteString(_line + "\n")
					}
					for _, node := range nodes {
						taxids = taxids[:0]
						for _, taxid = range node.LineageTaxids {
							taxids = append(taxids, strconv.Itoa(int(taxid)))
						}

						fmt.Fprintf(outfh, "%d\t%s\t%s\t%s\t%.15f\n",
							node.Taxid,
							node.Rank,
							strings.Join(taxids, taxidSep),
							strings.Join(node.LineageNames, taxidSep),
							node.Abundance,
						)
					}

					rankMap = make(map[uint32]string, 1024)
					meta = meta[:0]
					meta = append(meta, line)
					targets = targets[:0]
					hasData = false
					continue
				}

				meta = append(meta, line)
				continue
			}

			stringSplitN(line, "\t", n, &items)
			if len(items) < maxField {
				continue
			}

			percenage, err = strconv.ParseFloat(items[fieldPercentage], 64)
			if err != nil {
				checkError(fmt.Errorf("failed to parse abundance: %s", items[fieldPercentage]))
			}

			if percenage == 0 {
				continue
			}

			hasData = true

			_taxid, err = strconv.Atoi(items[fieldTaxid])
			if err != nil {
				checkError(fmt.Errorf("failed to parse taxid: %s", items[fieldTaxid]))
			}
			taxid = uint32(_taxid)

			rank = items[fieldRank]
			rankMap[taxid] = rank
			taxpath = items[fieldTaxpath]
			taxpathsn = items[fieldTaxpathSN]

			taxids = strings.Split(taxpath, taxidSep)
			skipThis = false
			for _, taxidS = range taxids {
				if _, ok = filter[taxidS]; ok {
					skipThis = true
					break
				}
			}
			if skipThis {
				continue
			}

			taxidsUint = make([]uint32, 0, len(taxids))
			for _, taxidS = range taxids {
				_taxid, err = strconv.Atoi(taxidS)
				if err != nil {
					checkError(fmt.Errorf("failed to parse taxid: %s", taxidS))
				}
				taxidsUint = append(taxidsUint, uint32(_taxid))
			}

			targets = append(targets, &Target{
				Taxid:     taxid,
				Abundance: percenage,

				Rank:          rank,
				TaxonName:     "",
				LineageNames:  strings.Split(taxpathsn, taxidSep),
				LineageTaxids: taxids,

				CompleteLineageTaxids: taxidsUint,
			})

		}

		if err := scanner.Err(); err != nil {
			checkError(err)
		}
		checkError(fh.Close())

		if hasData { // new record, need to summarize and output
			targets1 := filterLeaves(rankMap, leavesRanksMap, targets)

			profile := generateProfile2(targets, targets1)

			nodes := make([]*ProfileNode, 0, len(profile))
			for _, node := range profile {
				nodes = append(nodes, node)
			}

			sort.Slice(nodes, func(i, j int) bool {
				if rankOrder[nodes[i].Rank] < rankOrder[nodes[j].Rank] {
					return true
				}
				if rankOrder[nodes[i].Rank] == rankOrder[nodes[j].Rank] {
					return nodes[i].Abundance > nodes[j].Abundance
				}
				return false
			})

			for _, _line = range meta {
				outfh.WriteString(_line + "\n")
			}
			for _, node := range nodes {
				taxids = taxids[:0]
				for _, taxid = range node.LineageTaxids {
					taxids = append(taxids, strconv.Itoa(int(taxid)))
				}

				fmt.Fprintf(outfh, "%d\t%s\t%s\t%s\t%.15f\n",
					node.Taxid,
					node.Rank,
					strings.Join(taxids, taxidSep),
					strings.Join(node.LineageNames, taxidSep),
					node.Abundance,
				)
			}
		}

	},
}

func generateProfile2(targets0, targets []*Target) map[uint32]*ProfileNode {

	targetsMap := make(map[uint32]*Target, len(targets0))
	for _, target := range targets0 {
		targetsMap[target.Taxid] = target
	}

	profile := make(map[uint32]*ProfileNode, len(targets))

	var target0 *Target
	for _, target := range targets {
		for _, taxid := range target.CompleteLineageTaxids {
			if node, ok := profile[taxid]; !ok {
				target0 = targetsMap[taxid]

				profile[taxid] = &ProfileNode{
					Taxid:         taxid,
					Rank:          target0.Rank,
					TaxonName:     target0.TaxonName,
					LineageNames:  target0.LineageNames,
					LineageTaxids: target0.CompleteLineageTaxids,

					Abundance: target.Abundance,
				}
			} else {
				node.Abundance += target.Abundance
			}
		}
	}

	return profile
}

func filterLeaves(rankMap map[uint32]string, leavesRanksMap map[string]interface{}, targets []*Target) []*Target {

	targetsMap := make(map[uint32]*Target, len(targets))
	tree := make(map[uint32]map[uint32]uint32, 1024)

	var i int
	var taxidP, taxid uint32
	var ok bool
	for _, target := range targets {
		targetsMap[target.Taxid] = target

		for i, taxid = range target.CompleteLineageTaxids {
			if i == 0 {
				taxidP = 1
			} else {
				taxidP = target.CompleteLineageTaxids[i-1]
			}

			if _, ok = tree[taxidP]; !ok {
				tree[taxidP] = make(map[uint32]uint32, 8)
			}
			tree[taxidP][taxid] = target.Taxid
		}
	}
	// leaves := make([]uint32, 0, 1024)
	leaves := make([]*Target, 0, 1024)
	for _, m := range tree {
		for taxid = range m {
			if _, ok = tree[taxid]; !ok {
				if _, ok = leavesRanksMap[rankMap[taxid]]; ok {
					// leaves = append(leaves, taxid)
					leaves = append(leaves, targetsMap[m[taxid]])
				}
			}
		}
	}

	// recompute abundance
	var sum float64
	for _, target := range leaves {
		sum += target.Abundance
	}

	for _, target := range leaves {
		target.Abundance = target.Abundance / sum * 100
	}

	return leaves
}

func init() {
	RootCmd.AddCommand(camiFilterCmd)

	camiFilterCmd.Flags().IntP("field-taxid", "", 1, "field index of taxid")
	camiFilterCmd.Flags().IntP("field-rank", "", 2, "field index of taxid")
	camiFilterCmd.Flags().IntP("field-taxpath", "", 3, "field index of TAXPATH")
	camiFilterCmd.Flags().IntP("field-taxpathsn", "", 4, "field index of TAXPATHSN")
	camiFilterCmd.Flags().IntP("field-percentage", "", 5, "field index of PERCENTAGE")

	camiFilterCmd.Flags().StringP("taxid-sep", "", "|", "separator of taxid in TAXPATH and TAXPATHSN")

	camiFilterCmd.Flags().StringSliceP("taxids", "t", []string{}, "the parent taxid(s) to filter out")

	camiFilterCmd.Flags().StringSliceP("show-rank", "", []string{"superkingdom", "phylum", "class", "order", "family", "genus", "species", "strain"}, "only show TaxIds and names of these ranks")
	camiFilterCmd.Flags().StringSliceP("leave-ranks", "", []string{"species", "strain"}, "only consider leaves at these ranks")
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/gorilla/mux"
	"github.com/hashicorp/consul/api"
)

var (
	client            *api.Client
	dataDir           = "./data"
	Namespace         = "tuggle"
	Port              = 8080
	MaxFetchMultiplex = 3
	fetcherCh         = make(chan fetchRequest, 10)
)

const (
	defaultContentType = "application/octet-stream"
	InternalHeader     = "X-Tuggle-Internal"
)

type Object struct {
	ID          string    `json:"id"`
	Name        string    `json:"name"`
	ContentType string    `json:"content_type"`
	Size        int64     `json:"size"`
	CreatedAt   time.Time `json:"created_at"`
}

func md5Hex(b []byte) string {
	h := md5.New()
	h.Write(b)
	return fmt.Sprintf("%x", h.Sum(nil))
}

func NewObject(name string) *Object {
	return &Object{
		ID:        md5Hex([]byte(name)),
		Name:      name,
		CreatedAt: time.Now(),
	}
}

func init() {
	var err error
	client, err = api.NewClient(api.DefaultConfig())
	if err != nil {
		panic(err)
	}
}

func init() {
	rand.Seed(time.Now().UnixNano())
}

func main() {
	go fileFetcher()
	go eventWatcher()

	flag.StringVar(&dataDir, "data-dir", dataDir, "data directory")
	flag.IntVar(&Port, "port", Port, "listen port")
	flag.StringVar(&Namespace, "namespace", Namespace, "namespace")
	flag.Parse()

	m := mux.NewRouter()
	m.HandleFunc("/", indexHandler).Methods("GET")
	m.HandleFunc("/{name:[^/]+}", putHandler).Methods("PUT")
	m.HandleFunc("/{name:[^/]+}", fileHandler).Methods("GET", "HEAD", "DELETE")

	log.Printf(
		"starting tuggle data-dir:%s port:%d namespace:%s\n",
		dataDir,
		Port,
		Namespace,
	)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", Port), m))
}

func manageService(obj *Object, register bool) error {
	var mu sync.Mutex
	mu.Lock()
	defer mu.Unlock()

	var tags []string
	sv, err := client.Agent().Services()
	if err != nil {
		return err
	}
	if service := sv[Namespace]; service != nil {
		for _, tag := range service.Tags {
			if tag != obj.ID {
				tags = append(tags, tag)
			}
		}
	}
	if register {
		tags = append(tags, obj.ID)
	}

	reg := &api.AgentServiceRegistration{
		ID:   Namespace,
		Name: Namespace,
		Tags: tags,
		Port: Port,
	}
	return client.Agent().ServiceRegister(reg)
}

func registerService(obj *Object) error {
	return manageService(obj, true)
}

func deregisterService(obj *Object) error {
	return manageService(obj, false)
}

func eventWatcher() {
	var lastIndex uint64
	processedEvents := make(map[string]bool)
WATCH:
	for {
		events, qm, err := client.Event().List(
			Namespace, // eventName
			&api.QueryOptions{
				WaitIndex: lastIndex,
				WaitTime:  time.Second * 10,
			},
		)
		if err != nil {
			log.Println(err)
			time.Sleep(time.Second)
			continue WATCH
		}
	EVENT:
		for _, ev := range events {
			if processedEvents[ev.ID] {
				continue EVENT
			}
			processedEvents[ev.ID] = true
			if lastIndex == 0 {
				// at first time, ignore all stucked events
				continue EVENT
			}
			err := processEvent(string(ev.Payload))
			if err != nil {
				log.Println(err)
			}
		}
		lastIndex = qm.LastIndex
	}
}

func processEvent(payload string) error {
	p := strings.SplitN(payload, ":", 2)
	if len(p) != 2 {
		return fmt.Errorf("invalid payload %s", payload)
	}
	method := p[0]
	name := p[1]
	switch method {
	case "PUT":
		log.Println("fetching", name)
		f, err := fetch(name)
		if err != nil {
			return err
		}
		f.Close()
	case "DELETE":
		log.Println("deleting", name)
		obj := NewObject(name)
		if err := deregisterService(obj); err != nil {
			return err
		}
		if err := purgeFile(name); err != nil {
			return err
		}
	default:
		return fmt.Errorf("unknown payload %s", payload)
	}
	return nil
}

func storeObject(obj *Object) error {
	kvp := &api.KVPair{
		Key: path.Join(Namespace, obj.Name),
	}
	kvp.Value, _ = json.Marshal(obj)

	wm, err := client.KV().Put(kvp, nil)
	if err != nil {
		return err
	}
	log.Printf("%s metadata stored to consul kv in %s", obj.Name, wm.RequestTime)
	return nil
}

func purgeObject(obj *Object) error {
	_, err := client.KV().Delete(
		path.Join(Namespace, obj.Name),
		nil,
	)
	return err
}

func purgeFile(name string) error {
	return os.Remove(filepath.Join(dataDir, name))
}

func storeFile(name string, r io.ReadCloser) (*Object, error) {
	defer r.Close()
	i := rand.Int()
	filename := filepath.Join(dataDir, name)
	tmp := filename + "." + strconv.Itoa(i)
	f, err := os.Create(tmp)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	obj := NewObject(name)

	if n, err := io.Copy(f, r); err != nil {
		return nil, err
	} else {
		obj.Size = n
	}
	if err := os.Rename(tmp, filename); err != nil {
		return nil, err
	}
	return obj, nil
}

func putHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	name := vars["name"]

	obj, err := loadObject(name)
	if err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if obj != nil {
		http.Error(w, "exists", http.StatusMethodNotAllowed)
		return
	}

	obj, err = storeFile(name, r.Body)
	if err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if t := r.Header.Get("Content-Type"); t != "" {
		obj.ContentType = t
	} else {
		obj.ContentType = defaultContentType
	}

	if err := storeObject(obj); err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if err := registerService(obj); err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	ev := &api.UserEvent{
		Name:    Namespace,
		Payload: []byte("PUT:" + name),
	}
	eventID, _, err := client.Event().Fire(ev, nil)
	if err != nil {
		log.Println(err)
	} else {
		log.Printf("event PUT:%s fired ID:%s", name, eventID)
	}

	w.Header().Set("Content-Type", "text/plain")
	w.WriteHeader(http.StatusCreated)
}

func lockFetchMultiplex(key string) (*api.Semaphore, error) {
	prefix := path.Join(Namespace+".lock", key)
	sem, _ := client.SemaphoreOpts(&api.SemaphoreOptions{
		Prefix:            prefix,
		Limit:             MaxFetchMultiplex,
		SemaphoreWaitTime: time.Second,
		SemaphoreTryOnce:  true,
	})
	ch, err := sem.Acquire(nil)
	if err != nil {
		return nil, err
	}
	if ch == nil {
		sem.Release()
		return nil, nil
	}
	return sem, nil
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	kv := client.KV()
	kvps, _, err := kv.List(Namespace+"/", nil)
	if err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	objects := make([]Object, 0, len(kvps))
	for _, kvp := range kvps {
		var o Object
		if err := json.Unmarshal(kvp.Value, &o); err != nil {
			log.Println(err)
			continue
		}
		objects = append(objects, o)
	}
	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(objects)
}

func loadObject(name string) (*Object, error) {
	kv := client.KV()
	kvp, _, err := kv.Get(path.Join(Namespace, name), nil)
	if err != nil {
		return nil, err
	}
	if kvp == nil {
		return nil, nil
	}
	var obj Object
	if err := json.Unmarshal(kvp.Value, &obj); err != nil {
		return nil, err
	}
	return &obj, nil
}

func loadFile(name string) (io.ReadCloser, error) {
	return os.Open(filepath.Join(dataDir, name))
}

type fetchResponse struct {
	Reader io.ReadCloser
	Err    error
}

type fetchRequest struct {
	Name string
	Ch   chan fetchResponse
}

func fetch(name string) (io.ReadCloser, error) {
	ch := make(chan fetchResponse, 1)
	fetcherCh <- fetchRequest{
		Name: name,
		Ch:   ch,
	}
	res := <-ch
	return res.Reader, res.Err
}

func fileFetcher() {
	for req := range fetcherCh {
		r, err := loadFileOrRemote(req.Name)
		req.Ch <- fetchResponse{
			Reader: r,
			Err:    err,
		}
	}
}

func loadFileOrRemote(name string) (io.ReadCloser, error) {
	f, err := loadFile(name)
	if err == nil && f != nil {
		log.Println("hit local file")
		return f, nil
	}

	tries := 10
	for tries >= 0 {
		tries--
		// lookup service catlog
		catalogServices, _, err := client.Catalog().Service(
			Namespace,            // service name
			md5Hex([]byte(name)), // tag is Object.ID
			&api.QueryOptions{
				RequireConsistent: true,
				Near:              "_agent",
			},
		)
		if err != nil {
			log.Println(err)
			continue
		}
		nodes := len(catalogServices)
		if nodes == 0 {
			return nil, errors.New("Not Found in this cluster")
		}

		//pick up randomly from nearest 3 nodes
		n := 3
		if nodes < n {
			n = nodes
		}
		cs := catalogServices[rand.Intn(n)]

		sem, err := lockFetchMultiplex(cs.Address)
		if err != nil || sem == nil {
			log.Println("failed to get semaphore for", cs.Address)
			continue
		}

		err = loadRemoteAndStore(
			fmt.Sprintf("%s:%d", cs.Address, cs.ServicePort),
			name,
		)
		sem.Release()
		if err != nil {
			log.Println(err)
			continue
		}
		self, _ := client.Agent().NodeName()
		log.Printf(`[dot] "%s" -> "%s";`, cs.Node, self)

		return loadFile(name)
	}
	return nil, fmt.Errorf("failed to get %s in this cluster", name)
}

func loadRemoteAndStore(addr, name string) error {
	u := fmt.Sprintf("http://%s/%s", addr, name)
	log.Printf("loading remote %s", u)
	req, _ := http.NewRequest("GET", u, nil)
	req.Header.Set(InternalHeader, "True")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusOK {
		resp.Body.Close()
		return err
	}

	obj, err := storeFile(name, resp.Body)
	if err != nil {
		return err
	}
	if err := registerService(obj); err != nil {
		return err
	}
	return nil
}

func fileHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	name := vars["name"]

	obj, err := loadObject(name)
	if err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if obj == nil {
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	if r.Method == http.MethodDelete {
		err := purgeObject(obj)
		if err != nil {
			log.Println(err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}

		ev := &api.UserEvent{
			Name:    Namespace,
			Payload: []byte("DELETE:" + obj.Name),
		}
		eventID, _, err := client.Event().Fire(ev, nil)
		if err != nil {
			log.Println(err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		log.Printf("event DELETE:%s fired ID:%s", obj.Name, eventID)
		return
	}

	w.Header().Set("Content-Type", obj.ContentType)
	w.Header().Set("Content-Length", strconv.FormatInt(obj.Size, 10))
	w.Header().Set("Last-Modified", obj.CreatedAt.Format(time.RFC1123))
	w.Header().Set("X-Tuggle-Object-ID", obj.ID)

	if r.Method == http.MethodHead {
		return
	}

	// default action
	action := fetch

	// internal access won't fallback to remote cluster
	if r.Header.Get(InternalHeader) != "" {
		action = loadFile
	}

	f, err := action(name)
	if err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer f.Close()

	io.Copy(w, f)
}
<commit_msg>port 8900<commit_after>package main

import (
	"crypto/md5"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/gorilla/mux"
	"github.com/hashicorp/consul/api"
)

var (
	client            *api.Client
	dataDir           = "./data"
	Namespace         = "tuggle"
	Port              = 8900
	MaxFetchMultiplex = 3
	fetcherCh         = make(chan fetchRequest, 10)
)

const (
	defaultContentType = "application/octet-stream"
	InternalHeader     = "X-Tuggle-Internal"
)

type Object struct {
	ID          string    `json:"id"`
	Name        string    `json:"name"`
	ContentType string    `json:"content_type"`
	Size        int64     `json:"size"`
	CreatedAt   time.Time `json:"created_at"`
}

func md5Hex(b []byte) string {
	h := md5.New()
	h.Write(b)
	return fmt.Sprintf("%x", h.Sum(nil))
}

func NewObject(name string) *Object {
	return &Object{
		ID:        md5Hex([]byte(name)),
		Name:      name,
		CreatedAt: time.Now(),
	}
}

func init() {
	var err error
	client, err = api.NewClient(api.DefaultConfig())
	if err != nil {
		panic(err)
	}
}

func init() {
	rand.Seed(time.Now().UnixNano())
}

func main() {
	go fileFetcher()
	go eventWatcher()

	flag.StringVar(&dataDir, "data-dir", dataDir, "data directory")
	flag.IntVar(&Port, "port", Port, "listen port")
	flag.StringVar(&Namespace, "namespace", Namespace, "namespace")
	flag.Parse()

	m := mux.NewRouter()
	m.HandleFunc("/", indexHandler).Methods("GET")
	m.HandleFunc("/{name:[^/]+}", putHandler).Methods("PUT")
	m.HandleFunc("/{name:[^/]+}", fileHandler).Methods("GET", "HEAD", "DELETE")

	log.Printf(
		"starting tuggle data-dir:%s port:%d namespace:%s\n",
		dataDir,
		Port,
		Namespace,
	)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", Port), m))
}

func manageService(obj *Object, register bool) error {
	var mu sync.Mutex
	mu.Lock()
	defer mu.Unlock()

	var tags []string
	sv, err := client.Agent().Services()
	if err != nil {
		return err
	}
	if service := sv[Namespace]; service != nil {
		for _, tag := range service.Tags {
			if tag != obj.ID {
				tags = append(tags, tag)
			}
		}
	}
	if register {
		tags = append(tags, obj.ID)
	}

	reg := &api.AgentServiceRegistration{
		ID:   Namespace,
		Name: Namespace,
		Tags: tags,
		Port: Port,
	}
	return client.Agent().ServiceRegister(reg)
}

func registerService(obj *Object) error {
	return manageService(obj, true)
}

func deregisterService(obj *Object) error {
	return manageService(obj, false)
}

func eventWatcher() {
	var lastIndex uint64
	processedEvents := make(map[string]bool)
WATCH:
	for {
		events, qm, err := client.Event().List(
			Namespace, // eventName
			&api.QueryOptions{
				WaitIndex: lastIndex,
				WaitTime:  time.Second * 10,
			},
		)
		if err != nil {
			log.Println(err)
			time.Sleep(time.Second)
			continue WATCH
		}
	EVENT:
		for _, ev := range events {
			if processedEvents[ev.ID] {
				continue EVENT
			}
			processedEvents[ev.ID] = true
			if lastIndex == 0 {
				// at first time, ignore all stucked events
				continue EVENT
			}
			err := processEvent(string(ev.Payload))
			if err != nil {
				log.Println(err)
			}
		}
		lastIndex = qm.LastIndex
	}
}

func processEvent(payload string) error {
	p := strings.SplitN(payload, ":", 2)
	if len(p) != 2 {
		return fmt.Errorf("invalid payload %s", payload)
	}
	method := p[0]
	name := p[1]
	switch method {
	case "PUT":
		log.Println("fetching", name)
		f, err := fetch(name)
		if err != nil {
			return err
		}
		f.Close()
	case "DELETE":
		log.Println("deleting", name)
		obj := NewObject(name)
		if err := deregisterService(obj); err != nil {
			return err
		}
		if err := purgeFile(name); err != nil {
			return err
		}
	default:
		return fmt.Errorf("unknown payload %s", payload)
	}
	return nil
}

func storeObject(obj *Object) error {
	kvp := &api.KVPair{
		Key: path.Join(Namespace, obj.Name),
	}
	kvp.Value, _ = json.Marshal(obj)

	wm, err := client.KV().Put(kvp, nil)
	if err != nil {
		return err
	}
	log.Printf("%s metadata stored to consul kv in %s", obj.Name, wm.RequestTime)
	return nil
}

func purgeObject(obj *Object) error {
	_, err := client.KV().Delete(
		path.Join(Namespace, obj.Name),
		nil,
	)
	return err
}

func purgeFile(name string) error {
	return os.Remove(filepath.Join(dataDir, name))
}

func storeFile(name string, r io.ReadCloser) (*Object, error) {
	defer r.Close()
	i := rand.Int()
	filename := filepath.Join(dataDir, name)
	tmp := filename + "." + strconv.Itoa(i)
	f, err := os.Create(tmp)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	obj := NewObject(name)

	if n, err := io.Copy(f, r); err != nil {
		return nil, err
	} else {
		obj.Size = n
	}
	if err := os.Rename(tmp, filename); err != nil {
		return nil, err
	}
	return obj, nil
}

func putHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	name := vars["name"]

	obj, err := loadObject(name)
	if err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if obj != nil {
		http.Error(w, "exists", http.StatusMethodNotAllowed)
		return
	}

	obj, err = storeFile(name, r.Body)
	if err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if t := r.Header.Get("Content-Type"); t != "" {
		obj.ContentType = t
	} else {
		obj.ContentType = defaultContentType
	}

	if err := storeObject(obj); err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if err := registerService(obj); err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	ev := &api.UserEvent{
		Name:    Namespace,
		Payload: []byte("PUT:" + name),
	}
	eventID, _, err := client.Event().Fire(ev, nil)
	if err != nil {
		log.Println(err)
	} else {
		log.Printf("event PUT:%s fired ID:%s", name, eventID)
	}

	w.Header().Set("Content-Type", "text/plain")
	w.WriteHeader(http.StatusCreated)
}

func lockFetchMultiplex(key string) (*api.Semaphore, error) {
	prefix := path.Join(Namespace+".lock", key)
	sem, _ := client.SemaphoreOpts(&api.SemaphoreOptions{
		Prefix:            prefix,
		Limit:             MaxFetchMultiplex,
		SemaphoreWaitTime: time.Second,
		SemaphoreTryOnce:  true,
	})
	ch, err := sem.Acquire(nil)
	if err != nil {
		return nil, err
	}
	if ch == nil {
		sem.Release()
		return nil, nil
	}
	return sem, nil
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	kv := client.KV()
	kvps, _, err := kv.List(Namespace+"/", nil)
	if err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	objects := make([]Object, 0, len(kvps))
	for _, kvp := range kvps {
		var o Object
		if err := json.Unmarshal(kvp.Value, &o); err != nil {
			log.Println(err)
			continue
		}
		objects = append(objects, o)
	}
	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(objects)
}

func loadObject(name string) (*Object, error) {
	kv := client.KV()
	kvp, _, err := kv.Get(path.Join(Namespace, name), nil)
	if err != nil {
		return nil, err
	}
	if kvp == nil {
		return nil, nil
	}
	var obj Object
	if err := json.Unmarshal(kvp.Value, &obj); err != nil {
		return nil, err
	}
	return &obj, nil
}

func loadFile(name string) (io.ReadCloser, error) {
	return os.Open(filepath.Join(dataDir, name))
}

type fetchResponse struct {
	Reader io.ReadCloser
	Err    error
}

type fetchRequest struct {
	Name string
	Ch   chan fetchResponse
}

func fetch(name string) (io.ReadCloser, error) {
	ch := make(chan fetchResponse, 1)
	fetcherCh <- fetchRequest{
		Name: name,
		Ch:   ch,
	}
	res := <-ch
	return res.Reader, res.Err
}

func fileFetcher() {
	for req := range fetcherCh {
		r, err := loadFileOrRemote(req.Name)
		req.Ch <- fetchResponse{
			Reader: r,
			Err:    err,
		}
	}
}

func loadFileOrRemote(name string) (io.ReadCloser, error) {
	f, err := loadFile(name)
	if err == nil && f != nil {
		log.Println("hit local file")
		return f, nil
	}

	tries := 10
	for tries >= 0 {
		tries--
		// lookup service catlog
		catalogServices, _, err := client.Catalog().Service(
			Namespace,            // service name
			md5Hex([]byte(name)), // tag is Object.ID
			&api.QueryOptions{
				RequireConsistent: true,
				Near:              "_agent",
			},
		)
		if err != nil {
			log.Println(err)
			continue
		}
		nodes := len(catalogServices)
		if nodes == 0 {
			return nil, errors.New("Not Found in this cluster")
		}

		//pick up randomly from nearest 3 nodes
		n := 3
		if nodes < n {
			n = nodes
		}
		cs := catalogServices[rand.Intn(n)]

		sem, err := lockFetchMultiplex(cs.Address)
		if err != nil || sem == nil {
			log.Println("failed to get semaphore for", cs.Address)
			continue
		}

		err = loadRemoteAndStore(
			fmt.Sprintf("%s:%d", cs.Address, cs.ServicePort),
			name,
		)
		sem.Release()
		if err != nil {
			log.Println(err)
			continue
		}
		self, _ := client.Agent().NodeName()
		log.Printf(`[dot] "%s" -> "%s";`, cs.Node, self)

		return loadFile(name)
	}
	return nil, fmt.Errorf("failed to get %s in this cluster", name)
}

func loadRemoteAndStore(addr, name string) error {
	u := fmt.Sprintf("http://%s/%s", addr, name)
	log.Printf("loading remote %s", u)
	req, _ := http.NewRequest("GET", u, nil)
	req.Header.Set(InternalHeader, "True")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusOK {
		resp.Body.Close()
		return err
	}

	obj, err := storeFile(name, resp.Body)
	if err != nil {
		return err
	}
	if err := registerService(obj); err != nil {
		return err
	}
	return nil
}

func fileHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	name := vars["name"]

	obj, err := loadObject(name)
	if err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if obj == nil {
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	if r.Method == http.MethodDelete {
		err := purgeObject(obj)
		if err != nil {
			log.Println(err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}

		ev := &api.UserEvent{
			Name:    Namespace,
			Payload: []byte("DELETE:" + obj.Name),
		}
		eventID, _, err := client.Event().Fire(ev, nil)
		if err != nil {
			log.Println(err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		log.Printf("event DELETE:%s fired ID:%s", obj.Name, eventID)
		return
	}

	w.Header().Set("Content-Type", obj.ContentType)
	w.Header().Set("Content-Length", strconv.FormatInt(obj.Size, 10))
	w.Header().Set("Last-Modified", obj.CreatedAt.Format(time.RFC1123))
	w.Header().Set("X-Tuggle-Object-ID", obj.ID)

	if r.Method == http.MethodHead {
		return
	}

	// default action
	action := fetch

	// internal access won't fallback to remote cluster
	if r.Header.Get(InternalHeader) != "" {
		action = loadFile
	}

	f, err := action(name)
	if err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer f.Close()

	io.Copy(w, f)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"encoding/json"
	"fmt"
	_ "github.com/joho/godotenv/autoload"
	"github.com/MariaTerzieva/gotumblr"
	"os"
	"strconv"
)

const (
	tumblrURL  = "http://api.tumblr.com"
	blogName   = "devopsreactions.tumblr.com"
	blogTypes  = "text"
	postsLimit = 20
)

func main() {
	posts := getPosts()
	writePostsToCSV(posts)
}

func writePostsToCSV(posts []gotumblr.TextPost) {
	var row []string
	file, err := os.Create("data.csv")
	defer file.Close()
	if err != nil {
		fmt.Println(err)
		return
	}
	writer := csv.NewWriter(file)
	for _, post := range posts {
		row = getRow(post)
		writer.Write(row)
		if err := writer.Error(); err != nil {
			fmt.Println(err)
		}
	}
	writer.Flush()
}

func getRow(post gotumblr.TextPost) (row []string) {
	row = []string{
		strconv.FormatInt(post.Id, 10),
		post.Title,
		post.Body,
		post.Post_url,
	}
	return row
}

func getPosts() []gotumblr.TextPost {
	var posts, newPosts []gotumblr.TextPost
	offset := 0
	client := getTumblrClient()
	for len(newPosts) == postsLimit || offset == 0 {
		options := getTumblrOptions(offset)
		postsResponse := client.Posts(blogName, blogTypes, options)
		newPosts = parsePosts(postsResponse)
		posts = append(posts, newPosts...)
		offset += postsLimit
	}
	return posts
}

func getTumblrClient() *gotumblr.TumblrRestClient {
	client := gotumblr.NewTumblrRestClient(
		os.Getenv("CONSUMER_KEY"),
		os.Getenv("CONSUMER_SECRET"),
		os.Getenv("TOKEN"),
		os.Getenv("TOKEN_SECRET"),
		"https://www.albertyw.com/",
		tumblrURL,
	)
	return client
}

func getTumblrOptions(offset int) map[string]string {
	options := map[string]string{}
	options["offset"] = strconv.Itoa(offset)
	options["limit"] = strconv.Itoa(postsLimit)
	return options
}

func parsePosts(postsResponse gotumblr.PostsResponse) []gotumblr.TextPost {
	var posts []gotumblr.TextPost
	var post gotumblr.TextPost
	for _, element := range postsResponse.Posts {
		err := json.Unmarshal(element, &post)
		if err != nil {
			fmt.Println(err)
		} else {
			posts = append(posts, post)
		}
	}
	return posts
}
<commit_msg>No need to explicitly return variable<commit_after>package main

import (
	"encoding/csv"
	"encoding/json"
	"fmt"
	_ "github.com/joho/godotenv/autoload"
	"github.com/MariaTerzieva/gotumblr"
	"os"
	"strconv"
)

const (
	tumblrURL  = "http://api.tumblr.com"
	blogName   = "devopsreactions.tumblr.com"
	blogTypes  = "text"
	postsLimit = 20
)

func main() {
	posts := getPosts()
	writePostsToCSV(posts)
}

func writePostsToCSV(posts []gotumblr.TextPost) {
	var row []string
	file, err := os.Create("data.csv")
	defer file.Close()
	if err != nil {
		fmt.Println(err)
		return
	}
	writer := csv.NewWriter(file)
	for _, post := range posts {
		row = getRow(post)
		writer.Write(row)
		if err := writer.Error(); err != nil {
			fmt.Println(err)
		}
	}
	writer.Flush()
}

func getRow(post gotumblr.TextPost) (row []string) {
	row = []string{
		strconv.FormatInt(post.Id, 10),
		post.Title,
		post.Body,
		post.Post_url,
	}
	return
}

func getPosts() []gotumblr.TextPost {
	var posts, newPosts []gotumblr.TextPost
	offset := 0
	client := getTumblrClient()
	for len(newPosts) == postsLimit || offset == 0 {
		options := getTumblrOptions(offset)
		postsResponse := client.Posts(blogName, blogTypes, options)
		newPosts = parsePosts(postsResponse)
		posts = append(posts, newPosts...)
		offset += postsLimit
	}
	return posts
}

func getTumblrClient() *gotumblr.TumblrRestClient {
	client := gotumblr.NewTumblrRestClient(
		os.Getenv("CONSUMER_KEY"),
		os.Getenv("CONSUMER_SECRET"),
		os.Getenv("TOKEN"),
		os.Getenv("TOKEN_SECRET"),
		"https://www.albertyw.com/",
		tumblrURL,
	)
	return client
}

func getTumblrOptions(offset int) map[string]string {
	options := map[string]string{}
	options["offset"] = strconv.Itoa(offset)
	options["limit"] = strconv.Itoa(postsLimit)
	return options
}

func parsePosts(postsResponse gotumblr.PostsResponse) []gotumblr.TextPost {
	var posts []gotumblr.TextPost
	var post gotumblr.TextPost
	for _, element := range postsResponse.Posts {
		err := json.Unmarshal(element, &post)
		if err != nil {
			fmt.Println(err)
		} else {
			posts = append(posts, post)
		}
	}
	return posts
}
<|endoftext|>


<commit_before>package app

import (
	"github.com/gorilla/mux"
	"net/http"
	"github.com/Zenika/MARCEL/backend/medias"
	"github.com/rs/cors"
	"log"
	"os"
	//"github.com/Zenika/MARCEL/backend/plugins"
	"github.com/Zenika/MARCEL/backend/apidoc"
)

//current version of the API
const MARCEL_API_VERSION = "1"
var logFileName string = os.Getenv("MARCEL_LOG_FILE")
var logFile *os.File

type App struct {
	Router http.Handler
}

func (a *App) Initialize() {

	a.initializeLog()

	a.initializeData()

	a.initializeRoutes()
}

func (a *App) Run(addr string) {
	log.Fatal(http.ListenAndServe(addr, a.Router))
	log.Printf("Server is started and listening on port %v", addr)

	defer logFile.Close()

	select {}
}

func (a *App) initializeRoutes() {

	c := cors.New(cors.Options{
		AllowedOrigins: []string{"*"},
		// AllowedOrigins:   []string{"http://localhost:*"},
		AllowedMethods:   []string{"GET", "POST", "DELETE", "OPTION", "PUT"},
		AllowCredentials: true,
	})

	r := mux.NewRouter()
	s := r.PathPrefix("/api/v" + MARCEL_API_VERSION).Subrouter()
	s.HandleFunc("/medias", medias.GetAllHandler).Methods("GET")
	s.HandleFunc("/medias/{idMedia:[0-9]*}", medias.GetHandler).Methods("GET")
	s.HandleFunc("/medias/{idMedia:[0-9]*}", medias.PostHandler).Methods("POST")
	s.HandleFunc("/medias/create", medias.CreateHandler).Methods("GET")
	r.HandleFunc("/swagger.json", apidoc.GetConfigHandler).Methods("GET")

	a.Router = c.Handler(r)
}

func (a* App) initializeLog() {
	if len(logFileName) == 0 {
		logFileName = "marcel.log"
	}
	var err error = nil
	logFile, err = os.OpenFile(logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		log.Fatal(err)
	}
	log.SetOutput(logFile)
}

func (a* App) initializeData() {

	//Load plugins list from DB
	//plugins.LoadPluginsCatalog()

	//Load Medias configuration from DB
	medias.LoadMedias()
}

<commit_msg>[Add] Config endpoint<commit_after>package app

import (
	"github.com/gorilla/mux"
	"net/http"
	"github.com/Zenika/MARCEL/backend/medias"
	"github.com/rs/cors"
	"log"
	"os"
	//"github.com/Zenika/MARCEL/backend/plugins"
	"github.com/Zenika/MARCEL/backend/apidoc"
)

//current version of the API
const MARCEL_API_VERSION = "1"
var logFileName string = os.Getenv("MARCEL_LOG_FILE")
var logFile *os.File

type App struct {
	Router http.Handler
}

func (a *App) Initialize() {

	a.initializeLog()

	a.initializeData()

	a.initializeRoutes()
}

func (a *App) Run(addr string) {
	log.Fatal(http.ListenAndServe(addr, a.Router))
	log.Printf("Server is started and listening on port %v", addr)

	defer logFile.Close()

	select {}
}

func (a *App) initializeRoutes() {

	c := cors.New(cors.Options{
		AllowedOrigins: []string{"*"},
		// AllowedOrigins:   []string{"http://localhost:*"},
		AllowedMethods:   []string{"GET", "POST", "DELETE", "OPTION", "PUT"},
		AllowCredentials: true,
	})

	r := mux.NewRouter()
	s := r.PathPrefix("/api/v" + MARCEL_API_VERSION).Subrouter()
	s.HandleFunc("/medias", medias.GetAllHandler).Methods("GET")
	s.HandleFunc("/medias/config", medias.GetConfigHandler).Methods("GET")
	s.HandleFunc("/medias/{idMedia:[0-9]*}", medias.GetHandler).Methods("GET")
	s.HandleFunc("/medias/{idMedia:[0-9]*}", medias.PostHandler).Methods("POST")
	s.HandleFunc("/medias/create", medias.CreateHandler).Methods("GET")
	r.HandleFunc("/swagger.json", apidoc.GetConfigHandler).Methods("GET")

	a.Router = c.Handler(r)
}

func (a* App) initializeLog() {
	if len(logFileName) == 0 {
		logFileName = "marcel.log"
	}
	var err error = nil
	logFile, err = os.OpenFile(logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		log.Fatal(err)
	}
	log.SetOutput(logFile)
}

func (a* App) initializeData() {

	//Load plugins list from DB
	//plugins.LoadPluginsCatalog()

	//Load Medias configuration from DB
	medias.LoadMedias()
}

<|endoftext|>