<commit_before>/*
Package eventstore provides mongo implementation of domain event store
*/
package eventstore

import (
	"context"
	"errors"
	"fmt"

	"github.com/google/uuid"
	"go.mongodb.org/mongo-driver/bson"
	"go.mongodb.org/mongo-driver/bson/primitive"
	"go.mongodb.org/mongo-driver/mongo"
	"go.mongodb.org/mongo-driver/mongo/options"

	"github.com/vardius/go-api-boilerplate/pkg/domain"
	apperrors "github.com/vardius/go-api-boilerplate/pkg/errors"
	baseeventstore "github.com/vardius/go-api-boilerplate/pkg/eventstore"
)

type eventStore struct {
	collection *mongo.Collection
}

// New creates new mongo event store
func New(collectionName string, mongoDB *mongo.Database) baseeventstore.EventStore {
	if collectionName == "" {
		collectionName = "events"
	}

	return &eventStore{
		collection: mongoDB.Collection(collectionName),
	}
}

func (s *eventStore) Store(ctx context.Context, events []domain.Event) error {
	if len(events) == 0 {
		return nil
	}

	var buffer []mongo.WriteModel
	for _, e := range events {
		upsert := mongo.NewInsertOneModel()
		upsert.SetDocument(bson.M{"$set": e})

		buffer = append(buffer, upsert)
	}

	opts := options.BulkWrite()
	opts.SetOrdered(true)

	const chunkSize = 500

	for i := 0; i < len(buffer); i += chunkSize {
		end := i + chunkSize

		if end > len(buffer) {
			end = len(buffer)
		}

		if _, err := s.collection.BulkWrite(ctx, buffer[i:end], opts); err != nil {
			return err
		}
	}

	return nil
}

func (s *eventStore) Get(ctx context.Context, id uuid.UUID) (domain.Event, error) {
	filter := bson.M{
		"event_id": id.String(),
	}

	var result domain.Event
	if err := s.collection.FindOne(ctx, filter).Decode(&result); err != nil {
		if errors.Is(err, mongo.ErrNoDocuments) {
			return domain.NullEvent, apperrors.Wrap(fmt.Errorf("%s: %w", err, baseeventstore.ErrEventNotFound))
		}

		return domain.NullEvent, apperrors.Wrap(err)
	}

	return result, nil
}

func (s *eventStore) FindAll(ctx context.Context) ([]domain.Event, error) {
	filter := bson.M{}
	findOptions := options.FindOptions{
		Sort: bson.D{
			primitive.E{Key: "occurred_at", Value: 1},
		},
	}

	cur, err := s.collection.Find(ctx, filter, &findOptions)
	if err != nil {
		return nil, fmt.Errorf("failed to query events: %w", err)
	}
	defer cur.Close(ctx)

	var result []domain.Event
	for cur.Next(ctx) {
		var event domain.Event
		if err := cur.Decode(&event); err != nil {
			return nil, fmt.Errorf("failed to decode event: %w", err)
		}

		result = append(result, event)
	}

	return result, nil
}

func (s *eventStore) GetStream(ctx context.Context, streamID uuid.UUID, streamName string) ([]domain.Event, error) {
	filter := bson.M{
		"stream_id":   streamID.String(),
		"stream_name": streamName,
	}
	findOptions := options.FindOptions{
		Sort: bson.D{
			primitive.E{Key: "occurred_at", Value: 1},
		},
	}

	cur, err := s.collection.Find(ctx, filter, &findOptions)
	if err != nil {
		return nil, fmt.Errorf("failed to query events: %w", err)
	}
	defer cur.Close(ctx)

	var result []domain.Event
	for cur.Next(ctx) {
		var event domain.Event
		if err := cur.Decode(&event); err != nil {
			return nil, fmt.Errorf("failed to decode event: %w", err)
		}

		result = append(result, event)
	}

	return result, nil
}
<commit_msg>Update mongo implementation<commit_after>/*
Package eventstore provides mongo implementation of domain event store
*/
package eventstore

import (
	"context"
	"errors"
	"fmt"

	"github.com/google/uuid"
	"go.mongodb.org/mongo-driver/bson"
	"go.mongodb.org/mongo-driver/bson/primitive"
	"go.mongodb.org/mongo-driver/mongo"
	"go.mongodb.org/mongo-driver/mongo/options"

	"github.com/vardius/go-api-boilerplate/pkg/domain"
	apperrors "github.com/vardius/go-api-boilerplate/pkg/errors"
	baseeventstore "github.com/vardius/go-api-boilerplate/pkg/eventstore"
)

type eventStore struct {
	collection *mongo.Collection
}

// New creates new mongo event store
func New(collectionName string, mongoDB *mongo.Database) baseeventstore.EventStore {
	if collectionName == "" {
		collectionName = "events"
	}

	return &eventStore{
		collection: mongoDB.Collection(collectionName),
	}
}

func (s *eventStore) Store(ctx context.Context, events []domain.Event) error {
	if len(events) == 0 {
		return nil
	}

	var buffer []mongo.WriteModel
	for _, e := range events {
		upsert := mongo.NewInsertOneModel()
		upsert.SetDocument(bson.M{"$set": e})

		buffer = append(buffer, upsert)
	}

	opts := options.BulkWrite()
	opts.SetOrdered(true)

	const chunkSize = 500

	for i := 0; i < len(buffer); i += chunkSize {
		end := i + chunkSize

		if end > len(buffer) {
			end = len(buffer)
		}

		if _, err := s.collection.BulkWrite(ctx, buffer[i:end], opts); err != nil {
			return err
		}
	}

	return nil
}

func (s *eventStore) Get(ctx context.Context, id uuid.UUID) (domain.Event, error) {
	filter := bson.M{
		"event_id": id.String(),
	}

	var result domain.Event
	if err := s.collection.FindOne(ctx, filter).Decode(&result); err != nil {
		if errors.Is(err, mongo.ErrNoDocuments) {
			return domain.NullEvent, apperrors.Wrap(fmt.Errorf("%s: %w", err, baseeventstore.ErrEventNotFound))
		}

		return domain.NullEvent, apperrors.Wrap(err)
	}

	return result, nil
}

func (s *eventStore) FindAll(ctx context.Context) ([]domain.Event, error) {
	filter := bson.M{}
	findOptions := options.FindOptions{
		Sort: bson.D{
			primitive.E{Key: "occurred_at", Value: 1},
		},
	}

	cur, err := s.collection.Find(ctx, filter, &findOptions)
	if err != nil {
		return nil, fmt.Errorf("failed to query events: %w", err)
	}
	defer cur.Close(ctx)

	var result []domain.Event
	for cur.Next(ctx) {
		var event domain.Event
		if err := cur.Decode(&event); err != nil {
			return nil, fmt.Errorf("failed to decode event: %w", err)
		}

		result = append(result, event)
	}

	return result, nil
}

func (s *eventStore) GetStream(ctx context.Context, streamID uuid.UUID, streamName string) ([]domain.Event, error) {
	filter := bson.M{
		"stream_id":   streamID.String(),
		"stream_name": streamName,
	}
	findOptions := options.FindOptions{
		Sort: bson.D{
			primitive.E{Key: "occurred_at", Value: 1},
		},
	}

	cur, err := s.collection.Find(ctx, filter, &findOptions)
	if err != nil {
		return nil, fmt.Errorf("failed to query events: %w", err)
	}
	defer cur.Close(ctx)

	var result []domain.Event
	for cur.Next(ctx) {
		var event domain.Event
		if err := cur.Decode(&event); err != nil {
			return nil, fmt.Errorf("failed to decode event: %w", err)
		}

		result = append(result, event)
	}

	return result, nil
}

func (s *eventStore) GetStreamEventsByType(ctx context.Context, streamID uuid.UUID, streamName, eventType string) ([]domain.Event, error) {
	filter := bson.M{
		"stream_id":   streamID.String(),
		"stream_name": streamName,
		"type":        eventType,
	}
	findOptions := options.FindOptions{
		Sort: bson.D{
			primitive.E{Key: "occurred_at", Value: 1},
		},
	}

	cur, err := s.collection.Find(ctx, filter, &findOptions)
	if err != nil {
		return nil, fmt.Errorf("failed to query events: %w", err)
	}
	defer cur.Close(ctx)

	var result []domain.Event
	for cur.Next(ctx) {
		var event domain.Event
		if err := cur.Decode(&event); err != nil {
			return nil, fmt.Errorf("failed to decode event: %w", err)
		}

		result = append(result, event)
	}

	return result, nil
}
<|endoftext|>


<commit_before>package uct

import "flag"

var JavascriptReserved = []string{
	"and",       "del",       "from",      "not",       "while",
	"as",        "elif",      "global",    "or",        "with",
	"assert",    "else",      "if",        "pass",      "yield",
	"break",     "except",    "import",    "print",     "len",
	"class",     "exec",      "in",        "raise", 	"open",
	"continue",  "finally",   "is",        "return",    "bool",
	"def",       "for",       "lambda",    "try",		"copy",

}

//This is the Java compiler for uct.
var Javascript bool

func init() {
	flag.BoolVar(&Javascript, "js", false, "Target Javascript")

	JavascriptAssembly["INBOX"] = Instruction{
		Args: 0,
		Data: " ",
		Function: func(args []string) (result string) {

			result = "_unwind = setInterval(function() { if (stack.inbox <= 0) { return; } clearInterval(_unwind); stack.share(stack.inbox.shift())\n"

			JavascriptAssembly["js_Unwind"] = Instruction {
				Args: JavascriptAssembly["js_Unwind"].Args+1,
			}

			return result
		},
	}

	JavascriptAssembly["RETURN"] = Instruction{
		Indented:    1,
		Data:        "}\n",
		Indent:      -1,
		Else: &Instruction{
			Data: "return",
		},
		Function: func(args []string) (result string) {

			var times int = JavascriptAssembly["js_Unwind"].Args

			JavascriptAssembly["js_Unwind"] = Instruction{}

			for i:=0; i < times; i++ {
				result += `}, 100)`
			}

			return result+"}\n"
		},
	}

	JavascriptAssembly["EXIT"] =  Instruction{
		Indented:    1,
		Data:        "}\n",
		Indent:      -1,
		Indentation: -1,
		Else: &Instruction{
			Data: "return;",
		},
		Function: func(args []string) (result string) {

			var times int = JavascriptAssembly["js_Unwind"].Args

			JavascriptAssembly["js_Unwind"] = Instruction{}

			println(times)

			for i:=0; i < times; i++ {
				result += `}, 100)`
			}

			return result+"}\n"
		},
	}

	RegisterAssembler(JavascriptAssembly, &Javascript, "js", "//")

	for _, word := range JavascriptReserved {
		JavascriptAssembly[word] = Reserved()
	}
}

var JavascriptAssembly = Assemblable{
	//Special commands.
	"HEADER": Instruction{
		Data:   `
var bigInt = bigInt
if (typeof require != 'undefined') { require('./stack.js'); bigInt = global.bigInt }
//Helper functions
String.prototype.getBytes = function () {
var bytes = [];
for (var i = 0; i < this.length; ++i) {
bytes.push(this.charCodeAt(i));
}
return bytes;
};
`,
		Args: 1,
	},

	"FOOTER": Instruction{ Data: "if (typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScope) {} else { main(); }"},

	"FILE": Instruction{
		Path: "stack.js",
	},

	"JAVASCRIPT": Instruction{All:true},

	"NUMBER": is("bigInt(%s)", 1),
	"BIG": 	is("bigInt(\"%s\")", 1),
	"SIZE":   is("bigInt(%s.length)", 1),
	"STRING": is("%s.getBytes()", 1),
	"ERRORS":  is("stack.ERROR", 1),

	"LINK":  is("stack.link()"),
	"CONNECT":  is("stack.connect()"),
	"SLICE":  is("stack.slice()"),

	"SOFTWARE": Instruction{
		Data:   "function main() {\n\tstack = new Stack()\nbigInt = stack.bigInt",
		Indent: 1,
	},

	"FUNCTION": is("function %s(stack) {", 1, 1),

	"SCOPE": is(`stack.relay(stack.pipe(%s))`, 1),

	"EXE": is("%s.exe(stack)", 1),

	//Optimised
	"PUSH": is("stack.numbers.push(%s)", 1),
	"PULL": is("var %s = stack.numbers.pop()", 1),
	"SHARE": is("stack.arrays.push(%s)", 1),
	"GRAB":  is("var %s = stack.arrays.pop()", 1),
	"PUT":   is("stack.activearray.push(%s)", 1),
	"POP":   is("var %s = stack.activearray.pop()", 1),
	"PLACE": is("stack.activearray = %s", 1),
	"ARRAY":  is("stack.activearray = []; var %s = stack.activearray", 1),
	"RENAME": is("%s = stack.grab()", 1),
	"EVAL": is("eval(stack.grabstring()+'(stack)')"),

	"RELOAD": is("%s = stack.take()", 1),

	"RELAY": is("stack.relay(%s)", 1),
	"TAKE":  is("var %s = stack.take()", 1),

	"GET": is("var %s = stack.get()", 1),
	"SET": is("stack.set(%s)", 1),

	"VAR": is("var %s = bigInt()", 1),

	"OPEN":   is("stack.open()"),
	"EXECUTE": is("stack.stdout()"),
	"DELETE": is("stack.delete()"),
	"LOAD":   is("stack.load()"),
	"OUT":    is("stack.out()"),
	"STAT":   is("stack.info()"),
	"IN":     is("stack.in()"),
	"STDOUT": is("stack.stdout()"),
	"STDIN":  is("stack.stdin()"),
	"HEAP":   is("stack.heap()"),
	"HEAPIT":   is("stack.heapit()"),
	"MAKE":   is("stack.share(new Array(stack.pull().toJSNumber()).fill(bigInt.zero))"),

	"CLOSE": is("%s.close()", 1),

	"LOOP":   is("while (1) {", 0, 1),
	"BREAK":  is("break"),
	"REPEAT": is("}", 0, -1, -1),

	"IF":   is("if (%s != 0) {", 1, 1),
	"ELSE": is("} else {", 0, 0, -1),
	"END":  is("}", 0, -1, -1),

	"RUN":  is("%s(stack)", 1),
	"DATA": is("var %s = %s", 2),

	//Threading.
	"PIPE": is("%s = stack.pipe(stack.channel); stack.channel = stack.channel + 1", 1),
	"FORK": is("stack.thread('%s')", 1),

	//"INBOX": is("while (stack.inbox.length <= 0) {} stack.share(stack.inbox.shift())", 0),
	"OUTBOX": is("stack.outbox()", 0),

	"ADD": is("%s = %s.add(%s)", 3),
	"SUB": is("%s = %s.subtract(%s)", 3),
	"MUL": is("%s = %s.multiply(%s)", 3),
	"DIV": is("%s = %s.divide(%s)", 3),
	"MOD": is("%s = stack.mod(%s, %s)", 3),
	"POW": is("%s = %s.pow(%s)", 3),

	"SLT": is("%s = %s.lt(%s) ? bigInt.one : bigInt.zero;", 3),
	"SEQ": is("%s = %s.equals(%s) ? bigInt.one : bigInt.zero;", 3),
	"SGE": is("%s = %s.geq(%s) ? bigInt.one : bigInt.zero;", 3),
	"SGT": is("%s = %s.gt(%s) ? bigInt.one : bigInt.zero;", 3),
	"SNE": is("%s = %s.neq(%s) ? bigInt.one : bigInt.zero;", 3),
	"SLE": is("%s = %s.leq(%s)? bigInt.one : bigInt.zero;", 3),

	"JOIN": is("%s = %s.concat(%s)", 3),
	"ERROR": is("stack.ERROR = %s", 1),
}
<commit_msg>Use semicolons for javascript.<commit_after>package uct

import "flag"

var JavascriptReserved = []string{
	"and",       "del",       "from",      "not",       "while",
	"as",        "elif",      "global",    "or",        "with",
	"assert",    "else",      "if",        "pass",      "yield",
	"break",     "except",    "import",    "print",     "len",
	"class",     "exec",      "in",        "raise", 	"open",
	"continue",  "finally",   "is",        "return",    "bool",
	"def",       "for",       "lambda",    "try",		"copy",

}

//This is the Java compiler for uct.
var Javascript bool

func init() {
	flag.BoolVar(&Javascript, "js", false, "Target Javascript")

	JavascriptAssembly["INBOX"] = Instruction{
		Args: 0,
		Data: " ",
		Function: func(args []string) (result string) {

			result = "_unwind = setInterval(function() { if (stack.inbox <= 0) { return; } clearInterval(_unwind); stack.share(stack.inbox.shift())\n"

			JavascriptAssembly["js_Unwind"] = Instruction {
				Args: JavascriptAssembly["js_Unwind"].Args+1,
			}

			return result
		},
	}

	JavascriptAssembly["RETURN"] = Instruction{
		Indented:    1,
		Data:        "}\n",
		Indent:      -1,
		Else: &Instruction{
			Data: "return",
		},
		Function: func(args []string) (result string) {

			var times int = JavascriptAssembly["js_Unwind"].Args

			JavascriptAssembly["js_Unwind"] = Instruction{}

			for i:=0; i < times; i++ {
				result += `}, 100)`
			}

			return result+"}\n"
		},
	}

	JavascriptAssembly["EXIT"] =  Instruction{
		Indented:    1,
		Data:        "}\n",
		Indent:      -1,
		Indentation: -1,
		Else: &Instruction{
			Data: "return;",
		},
		Function: func(args []string) (result string) {

			var times int = JavascriptAssembly["js_Unwind"].Args

			JavascriptAssembly["js_Unwind"] = Instruction{}

			println(times)

			for i:=0; i < times; i++ {
				result += `}, 100)`
			}

			return result+"}\n"
		},
	}

	RegisterAssembler(JavascriptAssembly, &Javascript, "js", "//")

	for _, word := range JavascriptReserved {
		JavascriptAssembly[word] = Reserved()
	}
}

var JavascriptAssembly = Assemblable{
	//Special commands.
	"HEADER": Instruction{
		Data:   `
var bigInt = bigInt
if (typeof require != 'undefined') { require('./stack.js'); bigInt = global.bigInt }
//Helper functions
String.prototype.getBytes = function () {
var bytes = [];
for (var i = 0; i < this.length; ++i) {
bytes.push(this.charCodeAt(i));
}
return bytes;
};
`,
		Args: 1,
	},

	"FOOTER": Instruction{ Data: "if (typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScope) {} else { main(); }"},

	"FILE": Instruction{
		Path: "stack.js",
	},

	"JAVASCRIPT": Instruction{All:true},

	"NUMBER": is("bigInt(%s)", 1),
	"BIG": 	is("bigInt(\"%s\")", 1),
	"SIZE":   is("bigInt(%s.length)", 1),
	"STRING": is("%s.getBytes()", 1),
	"ERRORS":  is("stack.ERROR", 1),

	"LINK":  is("stack.link();"),
	"CONNECT":  is("stack.connect();"),
	"SLICE":  is("stack.slice();"),

	"SOFTWARE": Instruction{
		Data:   "function main() {\n\tstack = new Stack();\nbigInt = stack.bigInt;",
		Indent: 1,
	},

	"FUNCTION": is("function %s(stack) {", 1, 1),

	"SCOPE": is(`stack.relay(stack.pipe(%s));`, 1),

	"EXE": is("%s.exe(stack);", 1),

	//Optimised
	"PUSH": is("stack.numbers.push(%s);", 1),
	"PULL": is("var %s = stack.numbers.pop();", 1),
	"SHARE": is("stack.arrays.push(%s);", 1),
	"GRAB":  is("var %s = stack.arrays.pop();", 1),
	"PUT":   is("stack.activearray.push(%s);", 1),
	"POP":   is("var %s = stack.activearray.pop();", 1),
	"PLACE": is("stack.activearray = %s;", 1),
	"ARRAY":  is("stack.activearray = []; var %s = stack.activearray;", 1),
	"RENAME": is("%s = stack.grab();", 1),
	"EVAL": is("eval(stack.grabstring()+'(stack)');"),

	"RELOAD": is("%s = stack.take();", 1),

	"RELAY": is("stack.relay(%s);", 1),
	"TAKE":  is("var %s = stack.take();", 1),

	"GET": is("var %s = stack.get();", 1),
	"SET": is("stack.set(%s);", 1),

	"VAR": is("var %s = bigInt();", 1),

	"OPEN":   is("stack.open();"),
	"EXECUTE": is("stack.stdout();"),
	"DELETE": is("stack.delete();"),
	"LOAD":   is("stack.load();"),
	"OUT":    is("stack.out();"),
	"STAT":   is("stack.info();"),
	"IN":     is("stack.in();"),
	"STDOUT": is("stack.stdout();"),
	"STDIN":  is("stack.stdin();"),
	"HEAP":   is("stack.heap();"),
	"HEAPIT":   is("stack.heapit();"),
	"MAKE":   is("stack.share(new Array(stack.pull().toJSNumber()).fill(bigInt.zero));"),

	"CLOSE": is("%s.close();", 1),

	"LOOP":   is("while (1) {", 0, 1),
	"BREAK":  is("break;"),
	"REPEAT": is("}", 0, -1, -1),

	"IF":   is("if (%s != 0) {", 1, 1),
	"ELSE": is("} else {", 0, 0, -1),
	"END":  is("}", 0, -1, -1),

	"RUN":  is("%s(stack);", 1),
	"DATA": is("var %s = %s;", 2),

	//Threading.
	"PIPE": is("%s = stack.pipe(stack.channel); stack.channel = stack.channel + 1;", 1),
	"FORK": is("stack.thread('%s');", 1),

	//"INBOX": is("while (stack.inbox.length <= 0) {} stack.share(stack.inbox.shift())", 0),
	"OUTBOX": is("stack.outbox()", 0),

	"ADD": is("%s = %s.add(%s)", 3),
	"SUB": is("%s = %s.subtract(%s)", 3),
	"MUL": is("%s = %s.multiply(%s)", 3),
	"DIV": is("%s = %s.divide(%s)", 3),
	"MOD": is("%s = stack.mod(%s, %s)", 3),
	"POW": is("%s = %s.pow(%s)", 3),

	"SLT": is("%s = %s.lt(%s) ? bigInt.one : bigInt.zero;", 3),
	"SEQ": is("%s = %s.equals(%s) ? bigInt.one : bigInt.zero;", 3),
	"SGE": is("%s = %s.geq(%s) ? bigInt.one : bigInt.zero;", 3),
	"SGT": is("%s = %s.gt(%s) ? bigInt.one : bigInt.zero;", 3),
	"SNE": is("%s = %s.neq(%s) ? bigInt.one : bigInt.zero;", 3),
	"SLE": is("%s = %s.leq(%s)? bigInt.one : bigInt.zero;", 3),

	"JOIN": is("%s = %s.concat(%s)", 3),
	"ERROR": is("stack.ERROR = %s", 1),
}
<|endoftext|>


<commit_before>package fetchers

import (
	"github.com/golang/groupcache"
	"io/ioutil"
	"net/http"
)

// ProxyFetcher fetches images from an HTTP server.
type ProxyFetcher struct {
	ProxyUrlPrefix string
}

// Fetch makes an HTTP GET request to fetch the image data requested by the
// user.
func (f *ProxyFetcher) Fetch(urlPath string, dest groupcache.Sink) error {
	fullUrl := f.ProxyUrlPrefix + urlPath
	resp, err := http.Get(fullUrl)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	dest.SetBytes(data)
	return nil
}
<commit_msg>Check that we actually got a 200 response in the proxy fetcher.<commit_after>package fetchers

import (
	"errors"
	"fmt"
	"github.com/golang/groupcache"
	"io/ioutil"
	"net/http"
)

// ProxyFetcher fetches images from an HTTP server.
type ProxyFetcher struct {
	ProxyUrlPrefix string
}

// Fetch makes an HTTP GET request to fetch the image data requested by the
// user.
func (f *ProxyFetcher) Fetch(urlPath string, dest groupcache.Sink) error {
	fullUrl := f.ProxyUrlPrefix + urlPath
	resp, err := http.Get(fullUrl)
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusOK {
		errStr := fmt.Sprintf(
			"Got a bad status code back (expected 200, got %d)",
			resp.StatusCode,
		)
		return errors.New(errStr)
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	dest.SetBytes(data)
	return nil
}
<|endoftext|>


<commit_before>package paddlecloud

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"
)

// HTTPOK is ok status of http api call
const HTTPOK = "200 OK"

type RestClient struct {
	client *http.Client
}

// NewRestClient returns a new RestClient struct.
func NewRestClient() *RestClient {
	client := http.Client{Transport: &http.Transport{}}
	return &RestClient{client: &client}
}

func makeRequest(uri string, method string, body io.Reader,
	contentType string, query map[string]string,
	authHeader map[string]string) (*http.Request, error) {
	req, err := http.NewRequest(method, uri, nil)
	if err != nil {
		return nil, err
	}
	// default contentType is application/json
	if len(contentType) == 0 {
		req.Header.Set("Content-Type", "application/json")
	} else {
		req.Header.Set("Content-Type", contentType)
	}

	for k, v := range authHeader {
		req.Header.Set(k, v)
	}

	q := req.URL.Query()
	for k, v := range query {
		q.Add(k, v)
	}
	req.URL.RawQuery = q.Encode()
	return req, nil
}

// makeRequestToken use client token to make a authorized request
func makeRequestToken(uri string, method string, body io.Reader,
	contentType string, query map[string]string) (*http.Request, error) {
	// get client token
	token, err := token()
	if err != nil {
		return nil, err
	}
	authHeader := make(map[string]string)
	authHeader["Authorization"] = "Token " + token
	return makeRequest(uri, method, body, contentType, query, authHeader)
}

// NOTE: add other request makers if we need other auth methods

func (p *RestClient) getResponse(req *http.Request) ([]byte, error) {
	resp, err := p.client.Do(req)
	if err != nil {
		return []byte{}, err
	}
	defer resp.Body.Close()
	if resp.Status != HTTPOK {
		return []byte{}, errors.New("server error: " + resp.Status)
	}
	// FIXME: add more resp.Status checks
	return ioutil.ReadAll(resp.Body)
}

// GetCall make a GET call to targetURL with k-v params of query
func (p *RestClient) GetCall(targetURL string, query map[string]string) ([]byte, error) {
	req, err := makeRequestToken(targetURL, "GET", nil, "", query)
	if err != nil {
		return []byte{}, err
	}
	return p.getResponse(req)
}

// PostCall make a POST call to targetURL with a json body
func (p *RestClient) PostCall(targetURL string, jsonString []byte) ([]byte, error) {
	req, err := makeRequestToken(targetURL, "POST", bytes.NewBuffer(jsonString), "", nil)
	if err != nil {
		return []byte{}, err
	}
	return p.getResponse(req)
}

// DeleteCall make a DELETE call to targetURL with a json body
func (p *RestClient) DeleteCall(targetURL string, jsonString []byte) ([]byte, error) {
	req, err := makeRequestToken(targetURL, "DELETE", bytes.NewBuffer(jsonString), "", nil)
	if err != nil {
		return []byte{}, err
	}
	return p.getResponse(req)
}

// PostFile make a POST call to HTTP server to upload a file
func (p *RestClient) PostFile(targetURL string, filename string) ([]byte, error) {
	bodyBuf := &bytes.Buffer{}
	bodyWriter := multipart.NewWriter(bodyBuf)

	// this step is very important
	fileWriter, err := bodyWriter.CreateFormFile("uploadfile", filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error writing to buffer: %v\n", err)
		return []byte{}, err
	}

	// open file handle
	fh, err := os.Open(filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error opening file: %v\n", err)
		return []byte{}, err
	}

	//iocopy
	_, err = io.Copy(fileWriter, fh)
	if err != nil {
		return []byte{}, err
	}

	contentType := bodyWriter.FormDataContentType()
	bodyWriter.Close()

	req, err := makeRequestToken(targetURL, "POST", bodyBuf, contentType, nil)
	if err != nil {
		return []byte{}, err
	}
	return p.getResponse(req)
}

// PostChunkData makes a POST call to HTTP server to upload chunkdata
func (p *RestClient) PostChunk(targetURL string,
	chunkName string, reader io.Reader, len int64, boundary string) ([]byte, error) {
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	if err := writer.SetBoundary(boundary); err != nil {
		return nil, err
	}

	part, err := writer.CreateFormFile("chunk", chunkName)
	if err != nil {
		return nil, err
	}

	_, err = io.CopyN(part, reader, len)
	if err != nil {
		return nil, err
	}

	contentType := writer.FormDataContentType()
	writer.Close()

	req, err := makeRequestToken(targetURL, "POST", body, contentType, nil)
	if err != nil {
		return []byte{}, err
	}

	return p.getResponse(req)
}

// GetChunkData makes a GET call to HTTP server to download chunk data
func (p *RestClient) GetChunk(targetURL string,
	query map[string]string) (*http.Response, error) {
	req, err := makeRequestToken(targetURL, "GET", nil, "", query)
	if err != nil {
		return nil, err
	}

	return p.client.Do(req)
}

// GetCall makes a GET call to targetURL with k-v params of query.
func GetCall(targetURL string, query map[string]string) ([]byte, error) {
	client := NewRestClient()
	return client.GetCall(targetURL, query)
}

// PostCall makes a POST call to targetURL with a json body.
func PostCall(targetURL string, jsonString []byte) ([]byte, error) {
	client := NewRestClient()
	return client.PostCall(targetURL, jsonString)
}

// DeleteCall makes a DELETE call to targetURL with a json body.
func DeleteCall(targetURL string, jsonString []byte) ([]byte, error) {
	client := NewRestClient()
	return client.DeleteCall(targetURL, jsonString)
}
<commit_msg>change map to string<commit_after>package paddlecloud

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
)

// HTTPOK is ok status of http api call
const HTTPOK = "200 OK"

type RestClient struct {
	client *http.Client
}

// NewRestClient returns a new RestClient struct.
func NewRestClient() *RestClient {
	client := http.Client{Transport: &http.Transport{}}
	return &RestClient{client: &client}
}

func makeRequest(uri string, method string, body io.Reader,
	contentType string, query string,
	authHeader map[string]string) (*http.Request, error) {
	req, err := http.NewRequest(method, uri, nil)
	if err != nil {
		return nil, err
	}
	// default contentType is application/json
	if len(contentType) == 0 {
		req.Header.Set("Content-Type", "application/json")
	} else {
		req.Header.Set("Content-Type", contentType)
	}

	for k, v := range authHeader {
		req.Header.Set(k, v)
	}

	req.URL.RawQuery = query
	return req, nil
}

// makeRequestToken use client token to make a authorized request
func makeRequestToken(uri string, method string, body io.Reader,
	contentType string, query string) (*http.Request, error) {
	// get client token
	token, err := token()
	if err != nil {
		return nil, err
	}
	authHeader := make(map[string]string)
	authHeader["Authorization"] = "Token " + token
	return makeRequest(uri, method, body, contentType, query, authHeader)
}

// NOTE: add other request makers if we need other auth methods

func (p *RestClient) getResponse(req *http.Request) ([]byte, error) {
	resp, err := p.client.Do(req)
	if err != nil {
		return []byte{}, err
	}
	defer resp.Body.Close()
	if resp.Status != HTTPOK {
		return []byte{}, errors.New("server error: " + resp.Status)
	}
	// FIXME: add more resp.Status checks
	return ioutil.ReadAll(resp.Body)
}

// GetCall make a GET call to targetURL with query
func (p *RestClient) GetCall(targetURL string, query string) ([]byte, error) {
	req, err := makeRequestToken(targetURL, "GET", nil, "", query)
	if err != nil {
		return []byte{}, err
	}
	return p.getResponse(req)
}

// PostCall make a POST call to targetURL with a json body
func (p *RestClient) PostCall(targetURL string, jsonString []byte) ([]byte, error) {
	req, err := makeRequestToken(targetURL, "POST", bytes.NewBuffer(jsonString), "", "")
	if err != nil {
		return []byte{}, err
	}
	return p.getResponse(req)
}

// DeleteCall make a DELETE call to targetURL with a json body
func (p *RestClient) DeleteCall(targetURL string, jsonString []byte) ([]byte, error) {
	req, err := makeRequestToken(targetURL, "DELETE", bytes.NewBuffer(jsonString), "", "")
	if err != nil {
		return []byte{}, err
	}
	return p.getResponse(req)
}

// PostFile make a POST call to HTTP server to upload a file
func (p *RestClient) PostFile(targetURL string, filename string) ([]byte, error) {
	bodyBuf := &bytes.Buffer{}
	bodyWriter := multipart.NewWriter(bodyBuf)

	// this step is very important
	fileWriter, err := bodyWriter.CreateFormFile("uploadfile", filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error writing to buffer: %v\n", err)
		return []byte{}, err
	}

	// open file handle
	fh, err := os.Open(filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error opening file: %v\n", err)
		return []byte{}, err
	}

	//iocopy
	_, err = io.Copy(fileWriter, fh)
	if err != nil {
		return []byte{}, err
	}

	contentType := bodyWriter.FormDataContentType()
	bodyWriter.Close()

	req, err := makeRequestToken(targetURL, "POST", bodyBuf, contentType, "")
	if err != nil {
		return []byte{}, err
	}
	return p.getResponse(req)
}

// PostChunkData makes a POST call to HTTP server to upload chunkdata
func (p *RestClient) PostChunk(targetURL string,
	chunkName string, reader io.Reader, len int64, boundary string) ([]byte, error) {
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	if err := writer.SetBoundary(boundary); err != nil {
		return nil, err
	}

	part, err := writer.CreateFormFile("chunk", chunkName)
	if err != nil {
		return nil, err
	}

	_, err = io.CopyN(part, reader, len)
	if err != nil {
		return nil, err
	}

	contentType := writer.FormDataContentType()
	writer.Close()

	req, err := makeRequestToken(targetURL, "POST", body, contentType, "")
	if err != nil {
		return []byte{}, err
	}

	return p.getResponse(req)
}

// GetChunkData makes a GET call to HTTP server to download chunk data
func (p *RestClient) GetChunk(targetURL string,
	query string) (*http.Response, error) {
	req, err := makeRequestToken(targetURL, "GET", nil, "", query)
	if err != nil {
		return nil, err
	}

	return p.client.Do(req)
}

// GetCall makes a GET call to targetURL with k-v params of query.
func GetCall(targetURL string, query map[string]string) ([]byte, error) {
	client := NewRestClient()

	q := url.Values{}
	for k, v := range query {
		q.Add(k, v)
	}

	return client.GetCall(targetURL, q.Encode())
}

// PostCall makes a POST call to targetURL with a json body.
func PostCall(targetURL string, jsonString []byte) ([]byte, error) {
	client := NewRestClient()
	return client.PostCall(targetURL, jsonString)
}

// DeleteCall makes a DELETE call to targetURL with a json body.
func DeleteCall(targetURL string, jsonString []byte) ([]byte, error) {
	client := NewRestClient()
	return client.DeleteCall(targetURL, jsonString)
}
<|endoftext|>


<commit_before>package kafka_httpcat

import (
	"fmt"
	"log"
	"time"

	"github.com/Shopify/sarama"
	"github.com/mathpl/go-metrics"
	"github.com/mathpl/go-tsdmetrics"
)

type OffsetManager struct {
	client             sarama.Client
	broker             *sarama.Broker
	topic              string
	consumerGroup      string
	consumerID         string
	currentOffsetMap   map[int32]int64
	committedOffsetMap map[int32]int64
	commitThreshold    int64

	metricsRegistry tsdmetrics.TaggedRegistry
}

func GetDefaultSaramaConfig() *sarama.Config {
	saramaConfig := sarama.NewConfig()
	saramaConfig.Net.DialTimeout = 30 * time.Second
	saramaConfig.Net.ReadTimeout = 30 * time.Second
	saramaConfig.Net.WriteTimeout = 30 * time.Second
	saramaConfig.Metadata.Retry.Max = 3
	saramaConfig.Consumer.Fetch.Min = 1
	saramaConfig.Consumer.Fetch.Default = 32768
	saramaConfig.Consumer.Retry.Backoff = 2 * time.Second
	saramaConfig.Consumer.MaxWaitTime = 250 * time.Millisecond
	saramaConfig.Consumer.Return.Errors = true

	return saramaConfig
}

func NewOffsetManager(metricsRegistry tsdmetrics.TaggedRegistry, brokerList []string, partitionList []int32, topic string, consumerGroup string, consumerID string, initialOffset int64, commitThreshold int64) *OffsetManager {
	om := &OffsetManager{topic: topic, consumerGroup: consumerGroup, consumerID: consumerID, commitThreshold: commitThreshold, metricsRegistry: metricsRegistry}

	var err error
	if om.client, err = sarama.NewClient(brokerList, GetDefaultSaramaConfig()); err != nil {
		log.Fatalf("Unable to connect to broker with client: %s", err)
	}

	if om.broker, err = om.client.Coordinator(consumerGroup); err != nil {
		log.Fatalf("Unable to connect to fetch broker from coordinator: %s", err)
	}

	offsetRequest := sarama.OffsetFetchRequest{ConsumerGroup: consumerGroup, Version: 1}
	for _, partition := range partitionList {
		offsetRequest.AddPartition(topic, partition)
	}

	om.currentOffsetMap = make(map[int32]int64)
	om.committedOffsetMap = make(map[int32]int64)

	if resp, err := om.broker.FetchOffset(&offsetRequest); err != nil {
		log.Fatalf("Unable to fetch stored offset: %s", err)
	} else {
		for partition, offsetResponseBlock := range resp.Blocks[topic] {
			switch offsetResponseBlock.Err {
			case 0:
				om.currentOffsetMap[partition] = offsetResponseBlock.Offset
				om.committedOffsetMap[partition] = offsetResponseBlock.Offset
			case 1:
				//Not on server anymore, pick default
				om.currentOffsetMap[partition] = initialOffset
				om.committedOffsetMap[partition] = initialOffset
			default:
				log.Fatalf("Unexpected error fetching offsets: %d", offsetResponseBlock.Err)
			}
		}
	}

	return om
}

func (om *OffsetManager) Add(partition int32, offset int64) {
	om.currentOffsetMap[partition] = offset
	if om.currentOffsetMap[partition]-om.committedOffsetMap[partition] > om.commitThreshold {
		offsetReq := &sarama.OffsetCommitRequest{ConsumerGroup: om.consumerGroup, ConsumerID: om.consumerID, Version: 1}
		offsetReq.AddBlock(om.topic, partition, offset, time.Now().Unix(), "")
		if _, err := om.broker.CommitOffset(offsetReq); err != nil {
			log.Printf("Unable to commit offset: %s", err)
		} else {
			log.Printf("Commited offset for partition: %d offset %d", partition, offset)
			om.committedOffsetMap[partition] = offset

			i := om.metricsRegistry.GetOrRegister("consumer.committed", tsdmetrics.Tags{"partition": fmt.Sprintf("%d", partition)}, metrics.NewGauge())
			if m, ok := i.(metrics.Gauge); ok {
				m.Update(offset)
			} else {
				log.Print("Unexpected metric type")
			}
		}
	}

	i := om.metricsRegistry.GetOrRegister("consumer.sent", tsdmetrics.Tags{"partition": fmt.Sprintf("%d", partition)}, metrics.NewGauge())
	if m, ok := i.(metrics.Gauge); ok {
		m.Update(offset)
	} else {
		log.Printf("Unexpected metric type")
	}
}

func (om *OffsetManager) CommitAll() {
	offsetReq := &sarama.OffsetCommitRequest{ConsumerGroup: om.consumerGroup, ConsumerID: om.consumerID, Version: 1}
	for partition, offset := range om.currentOffsetMap {
		offsetReq.AddBlock(om.topic, partition, offset, time.Now().Unix(), "")
		log.Printf("Committing offset for partition: %d offset %d...", partition, offset)
	}

	if _, err := om.broker.CommitOffset(offsetReq); err != nil {
		//FIXME: disconnect on long wait
		log.Printf("Unable to commit offsets: %s", err)
	} else {
		for partition, offset := range om.currentOffsetMap {
			om.committedOffsetMap[partition] = offset
		}
		log.Print("Committed")
	}
}

func (om *OffsetManager) GetCurrentOffset(partition int32) int64 {
	if offset, ok := om.currentOffsetMap[partition]; ok {
		return offset
	} else {
		return -1
	}
}
<commit_msg>Fixing import.<commit_after>package kafka_httpcat

import (
	"fmt"
	"log"
	"time"

	"github.com/Shopify/sarama"
	"github.com/mathpl/go-tsdmetrics"
	"github.com/rcrowley/go-metrics"
)

type OffsetManager struct {
	client             sarama.Client
	broker             *sarama.Broker
	topic              string
	consumerGroup      string
	consumerID         string
	currentOffsetMap   map[int32]int64
	committedOffsetMap map[int32]int64
	commitThreshold    int64

	metricsRegistry tsdmetrics.TaggedRegistry
}

func GetDefaultSaramaConfig() *sarama.Config {
	saramaConfig := sarama.NewConfig()
	saramaConfig.Net.DialTimeout = 30 * time.Second
	saramaConfig.Net.ReadTimeout = 30 * time.Second
	saramaConfig.Net.WriteTimeout = 30 * time.Second
	saramaConfig.Metadata.Retry.Max = 3
	saramaConfig.Consumer.Fetch.Min = 1
	saramaConfig.Consumer.Fetch.Default = 32768
	saramaConfig.Consumer.Retry.Backoff = 2 * time.Second
	saramaConfig.Consumer.MaxWaitTime = 250 * time.Millisecond
	saramaConfig.Consumer.Return.Errors = true

	return saramaConfig
}

func NewOffsetManager(metricsRegistry tsdmetrics.TaggedRegistry, brokerList []string, partitionList []int32, topic string, consumerGroup string, consumerID string, initialOffset int64, commitThreshold int64) *OffsetManager {
	om := &OffsetManager{topic: topic, consumerGroup: consumerGroup, consumerID: consumerID, commitThreshold: commitThreshold, metricsRegistry: metricsRegistry}

	var err error
	if om.client, err = sarama.NewClient(brokerList, GetDefaultSaramaConfig()); err != nil {
		log.Fatalf("Unable to connect to broker with client: %s", err)
	}

	if om.broker, err = om.client.Coordinator(consumerGroup); err != nil {
		log.Fatalf("Unable to connect to fetch broker from coordinator: %s", err)
	}

	offsetRequest := sarama.OffsetFetchRequest{ConsumerGroup: consumerGroup, Version: 1}
	for _, partition := range partitionList {
		offsetRequest.AddPartition(topic, partition)
	}

	om.currentOffsetMap = make(map[int32]int64)
	om.committedOffsetMap = make(map[int32]int64)

	if resp, err := om.broker.FetchOffset(&offsetRequest); err != nil {
		log.Fatalf("Unable to fetch stored offset: %s", err)
	} else {
		for partition, offsetResponseBlock := range resp.Blocks[topic] {
			switch offsetResponseBlock.Err {
			case 0:
				om.currentOffsetMap[partition] = offsetResponseBlock.Offset
				om.committedOffsetMap[partition] = offsetResponseBlock.Offset
			case 1:
				//Not on server anymore, pick default
				om.currentOffsetMap[partition] = initialOffset
				om.committedOffsetMap[partition] = initialOffset
			default:
				log.Fatalf("Unexpected error fetching offsets: %d", offsetResponseBlock.Err)
			}
		}
	}

	return om
}

func (om *OffsetManager) Add(partition int32, offset int64) {
	om.currentOffsetMap[partition] = offset
	if om.currentOffsetMap[partition]-om.committedOffsetMap[partition] > om.commitThreshold {
		offsetReq := &sarama.OffsetCommitRequest{ConsumerGroup: om.consumerGroup, ConsumerID: om.consumerID, Version: 1}
		offsetReq.AddBlock(om.topic, partition, offset, time.Now().Unix(), "")
		if _, err := om.broker.CommitOffset(offsetReq); err != nil {
			log.Printf("Unable to commit offset: %s", err)
		} else {
			log.Printf("Commited offset for partition: %d offset %d", partition, offset)
			om.committedOffsetMap[partition] = offset

			i := om.metricsRegistry.GetOrRegister("consumer.committed", tsdmetrics.Tags{"partition": fmt.Sprintf("%d", partition)}, metrics.NewGauge())
			if m, ok := i.(metrics.Gauge); ok {
				m.Update(offset)
			} else {
				log.Print("Unexpected metric type")
			}
		}
	}

	i := om.metricsRegistry.GetOrRegister("consumer.sent", tsdmetrics.Tags{"partition": fmt.Sprintf("%d", partition)}, metrics.NewGauge())
	if m, ok := i.(metrics.Gauge); ok {
		m.Update(offset)
	} else {
		log.Printf("Unexpected metric type")
	}
}

func (om *OffsetManager) CommitAll() {
	offsetReq := &sarama.OffsetCommitRequest{ConsumerGroup: om.consumerGroup, ConsumerID: om.consumerID, Version: 1}
	for partition, offset := range om.currentOffsetMap {
		offsetReq.AddBlock(om.topic, partition, offset, time.Now().Unix(), "")
		log.Printf("Committing offset for partition: %d offset %d...", partition, offset)
	}

	if _, err := om.broker.CommitOffset(offsetReq); err != nil {
		//FIXME: disconnect on long wait
		log.Printf("Unable to commit offsets: %s", err)
	} else {
		for partition, offset := range om.currentOffsetMap {
			om.committedOffsetMap[partition] = offset
		}
		log.Print("Committed")
	}
}

func (om *OffsetManager) GetCurrentOffset(partition int32) int64 {
	if offset, ok := om.currentOffsetMap[partition]; ok {
		return offset
	} else {
		return -1
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
)

func main() {
	if os.Args[0] == "/proc/self/exe" {
		inner()
	} else {
		os.Exit(outer())
	}
}

func outer() int {
	rootFS := flag.String("rootFS", "", "rootFS")
	privileged := flag.Bool("privileged", false, "if true, user namespace is not used")
	flag.Parse()
	if *rootFS == "" {
		fmt.Println("must set -rootFS")
		return 1
	}

	cowRootFS, err := ioutil.TempDir("", "container-run")
	must(err)

	mappingSize := 100000
	chownTo := mappingSize
	if *privileged {
		chownTo = 0
	}
	containerRootFSPath, err := createUniqueRootFS(*rootFS, cowRootFS, chownTo)
	must(err)
	defer func() {
		must(syscall.Unmount(containerRootFSPath, 0))
		must(os.RemoveAll(cowRootFS))
	}()

	must(syscall.Mount("", "/", "", syscall.MS_PRIVATE|syscall.MS_REC, "remount"))

	cmd := exec.Command("/proc/self/exe", append([]string{containerRootFSPath}, flag.Args()...)...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	cmd.SysProcAttr = &syscall.SysProcAttr{
		Cloneflags: syscall.CLONE_NEWUTS | syscall.CLONE_NEWNS | syscall.CLONE_NEWPID,
	}
	if !*privileged {
		cmd.SysProcAttr.Cloneflags = cmd.SysProcAttr.Cloneflags | syscall.CLONE_NEWUSER
		mapping := []syscall.SysProcIDMap{
			{
				ContainerID: 0,
				HostID:      mappingSize,
				Size:        1,
			},
			{
				ContainerID: 1,
				HostID:      1,
				Size:        mappingSize - 1,
			},
		}
		cmd.SysProcAttr.UidMappings = mapping
		cmd.SysProcAttr.GidMappings = mapping
		cmd.SysProcAttr.Credential = &syscall.Credential{Uid: 0, Gid: 0}
	}

	if err := cmd.Run(); err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			return exitErr.Sys().(syscall.WaitStatus).ExitStatus()
		}

		must(err)
	}

	return 0
}

func inner() {
	rootFS := os.Args[1]

	oldRootFS := filepath.Join(rootFS, "oldrootfs")
	must(os.Mkdir(oldRootFS, 0700))
	must(syscall.Mount(rootFS, rootFS, "", syscall.MS_BIND, ""))
	must(syscall.PivotRoot(rootFS, oldRootFS))
	must(os.Chdir("/"))
	must(syscall.Mount("proc", "/proc", "proc", 0, ""))
	must(syscall.Unmount("/oldrootfs", syscall.MNT_DETACH))
	must(os.Remove("/oldrootfs"))

	must(syscall.Exec(os.Args[2], os.Args[2:], os.Environ()))
}

func createUniqueRootFS(rootFS, cowRootFS string, chownTo int) (string, error) {
	lowerLayer := rootFS
	if chownTo != 0 {
		var err error
		lowerLayer, err = createUnprivilegedRootFS(rootFS, chownTo)
		if err != nil {
			return "", err
		}
	}

	if err := os.Chown(cowRootFS, chownTo, chownTo); err != nil {
		return "", nil
	}

	containerRootFS := filepath.Join(cowRootFS, "union")
	workDir := filepath.Join(cowRootFS, "work")
	upperDir := filepath.Join(cowRootFS, "upper")
	for _, dir := range []string{containerRootFS, workDir, upperDir} {
		if err := os.Mkdir(dir, 0700); err != nil {
			return "", err
		}
		if err := os.Chown(dir, chownTo, chownTo); err != nil {
			return "", nil
		}
	}
	if err := mountOverlay(lowerLayer, upperDir, workDir, containerRootFS); err != nil {
		return "", err
	}
	return containerRootFS, nil
}

func createUnprivilegedRootFS(rootFS string, uid int) (string, error) {
	fssDir := filepath.Dir(rootFS)
	fsName := filepath.Base(rootFS)
	unprivilegedRootFSPath := filepath.Join(fssDir, fsName+"-unprivileged")

	lockPath := filepath.Join(fssDir, fsName+"-chownlock")
	lockFile, err := os.OpenFile(lockPath, os.O_CREATE, 0600)
	if err != nil {
		return "", err
	}
	if err := syscall.Flock(int(lockFile.Fd()), syscall.LOCK_EX); err != nil {
		return "", err
	}
	defer func() {
		syscall.Flock(int(lockFile.Fd()), syscall.LOCK_UN)
		lockFile.Close()
	}()

	if _, err := os.Stat(unprivilegedRootFSPath); err == nil {
		return unprivilegedRootFSPath, nil
	}

	if err := filepath.Walk(rootFS, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		relativePath, err := filepath.Rel(rootFS, path)
		if err != nil {
			return err
		}
		newPath := filepath.Join(unprivilegedRootFSPath, relativePath)

		if info.IsDir() {
			if err := os.MkdirAll(newPath, info.Mode()); err != nil {
				return err
			}
			return os.Chown(newPath, uid, uid)
		}

		if info.Mode()&os.ModeSymlink != 0 {
			linkTarget, err := os.Readlink(path)
			if err != nil {
				return err
			}
			if err := os.Symlink(linkTarget, newPath); err != nil {
				return err
			}
			return os.Lchown(newPath, uid, uid)
		}

		if info.Mode()&os.ModeDevice != 0 {
			// Don't bother setting up devices for the container
			return nil
		}

		originalFile, err := os.Open(path)
		if err != nil {
			return err
		}
		defer originalFile.Close()
		newFile, err := os.OpenFile(newPath, os.O_CREATE|os.O_WRONLY, info.Mode())
		if err != nil {
			return err
		}
		defer newFile.Close()
		if _, err := io.Copy(newFile, originalFile); err != nil {
			return err
		}

		return os.Chown(newPath, uid, uid)
	}); err != nil {
		return "", err
	}

	return unprivilegedRootFSPath, nil
}

func mountOverlay(lowerDir, upperDir, workDir, unionDir string) error {
	overlayMountOpts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lowerDir, upperDir, workDir)
	return syscall.Mount("overlay", unionDir, "overlay", 0, overlayMountOpts)
}

func must(err error) {
	if err != nil {
		panic(err)
	}
}
<commit_msg>Sync inner and outer process using a pipe<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
)

func main() {
	if os.Args[0] == "/proc/self/exe" {
		inner()
	} else {
		os.Exit(outer())
	}
}

func outer() int {
	rootFS := flag.String("rootFS", "", "rootFS")
	privileged := flag.Bool("privileged", false, "if true, user namespace is not used")
	flag.Parse()
	if *rootFS == "" {
		fmt.Println("must set -rootFS")
		return 1
	}

	cowRootFS, err := ioutil.TempDir("", "container-run")
	must(err)

	mappingSize := 100000
	chownTo := mappingSize
	if *privileged {
		chownTo = 0
	}
	containerRootFSPath, err := createUniqueRootFS(*rootFS, cowRootFS, chownTo)
	must(err)
	defer func() {
		must(syscall.Unmount(containerRootFSPath, 0))
		must(os.RemoveAll(cowRootFS))
	}()

	must(syscall.Mount("", "/", "", syscall.MS_PRIVATE|syscall.MS_REC, "remount"))

	syncR, syncW, err := os.Pipe()
	must(err)
	defer syncW.Close()

	cmd := exec.Command("/proc/self/exe", append([]string{containerRootFSPath}, flag.Args()...)...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Cloneflags: syscall.CLONE_NEWUTS | syscall.CLONE_NEWNS | syscall.CLONE_NEWPID,
	}
	cmd.ExtraFiles = []*os.File{syncR}

	if !*privileged {
		cmd.SysProcAttr.Cloneflags = cmd.SysProcAttr.Cloneflags | syscall.CLONE_NEWUSER
		mapping := []syscall.SysProcIDMap{
			{
				ContainerID: 0,
				HostID:      mappingSize,
				Size:        1,
			},
			{
				ContainerID: 1,
				HostID:      1,
				Size:        mappingSize - 1,
			},
		}
		cmd.SysProcAttr.UidMappings = mapping
		cmd.SysProcAttr.GidMappings = mapping
		cmd.SysProcAttr.Credential = &syscall.Credential{Uid: 0, Gid: 0}
	}

	must(cmd.Start())
	syncR.Close()
	_, err = syncW.Write([]byte{0})
	must(err)

	if err := cmd.Wait(); err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			return exitErr.Sys().(syscall.WaitStatus).ExitStatus()
		}

		must(err)
	}

	return 0
}

func inner() {
	rootFS := os.Args[1]

	oldRootFS := filepath.Join(rootFS, "oldrootfs")
	must(os.Mkdir(oldRootFS, 0700))
	must(syscall.Mount(rootFS, rootFS, "", syscall.MS_BIND, ""))
	must(syscall.PivotRoot(rootFS, oldRootFS))
	must(os.Chdir("/"))
	must(syscall.Mount("proc", "/proc", "proc", 0, ""))
	must(syscall.Unmount("/oldrootfs", syscall.MNT_DETACH))
	must(os.Remove("/oldrootfs"))

	syncR := os.NewFile(3, "sync")
	_, err := syncR.Read(make([]byte, 1))
	must(err)
	syncR.Close()

	must(syscall.Exec(os.Args[2], os.Args[2:], os.Environ()))
}

func createUniqueRootFS(rootFS, cowRootFS string, chownTo int) (string, error) {
	lowerLayer := rootFS
	if chownTo != 0 {
		var err error
		lowerLayer, err = createUnprivilegedRootFS(rootFS, chownTo)
		if err != nil {
			return "", err
		}
	}

	if err := os.Chown(cowRootFS, chownTo, chownTo); err != nil {
		return "", nil
	}

	containerRootFS := filepath.Join(cowRootFS, "union")
	workDir := filepath.Join(cowRootFS, "work")
	upperDir := filepath.Join(cowRootFS, "upper")
	for _, dir := range []string{containerRootFS, workDir, upperDir} {
		if err := os.Mkdir(dir, 0700); err != nil {
			return "", err
		}
		if err := os.Chown(dir, chownTo, chownTo); err != nil {
			return "", nil
		}
	}
	if err := mountOverlay(lowerLayer, upperDir, workDir, containerRootFS); err != nil {
		return "", err
	}
	return containerRootFS, nil
}

func createUnprivilegedRootFS(rootFS string, uid int) (string, error) {
	fssDir := filepath.Dir(rootFS)
	fsName := filepath.Base(rootFS)
	unprivilegedRootFSPath := filepath.Join(fssDir, fsName+"-unprivileged")

	lockPath := filepath.Join(fssDir, fsName+"-chownlock")
	lockFile, err := os.OpenFile(lockPath, os.O_CREATE, 0600)
	if err != nil {
		return "", err
	}
	if err := syscall.Flock(int(lockFile.Fd()), syscall.LOCK_EX); err != nil {
		return "", err
	}
	defer func() {
		syscall.Flock(int(lockFile.Fd()), syscall.LOCK_UN)
		lockFile.Close()
	}()

	if _, err := os.Stat(unprivilegedRootFSPath); err == nil {
		return unprivilegedRootFSPath, nil
	}

	if err := filepath.Walk(rootFS, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		relativePath, err := filepath.Rel(rootFS, path)
		if err != nil {
			return err
		}
		newPath := filepath.Join(unprivilegedRootFSPath, relativePath)

		if info.IsDir() {
			if err := os.MkdirAll(newPath, info.Mode()); err != nil {
				return err
			}
			return os.Chown(newPath, uid, uid)
		}

		if info.Mode()&os.ModeSymlink != 0 {
			linkTarget, err := os.Readlink(path)
			if err != nil {
				return err
			}
			if err := os.Symlink(linkTarget, newPath); err != nil {
				return err
			}
			return os.Lchown(newPath, uid, uid)
		}

		if info.Mode()&os.ModeDevice != 0 {
			// Don't bother setting up devices for the container
			return nil
		}

		originalFile, err := os.Open(path)
		if err != nil {
			return err
		}
		defer originalFile.Close()
		newFile, err := os.OpenFile(newPath, os.O_CREATE|os.O_WRONLY, info.Mode())
		if err != nil {
			return err
		}
		defer newFile.Close()
		if _, err := io.Copy(newFile, originalFile); err != nil {
			return err
		}

		return os.Chown(newPath, uid, uid)
	}); err != nil {
		return "", err
	}

	return unprivilegedRootFSPath, nil
}

func mountOverlay(lowerDir, upperDir, workDir, unionDir string) error {
	overlayMountOpts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lowerDir, upperDir, workDir)
	return syscall.Mount("overlay", unionDir, "overlay", 0, overlayMountOpts)
}

func must(err error) {
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package server

import (
    "github.com/itsankoff/gotcha/common"
    "log"
    "encoding/json"
)

type Control struct {
    input           chan *common.Message
    groups          []*Group
    outputStore     *OutputStore
}

func NewControl(input chan *common.Message,
                outputStore *OutputStore) *Control {
    c := &Control{
        input: input,
        outputStore: outputStore,
    }

    go c.listen()
    return c
}

func (c Control) listen() {
    for {
        select {
        case msg := <-c.input:
            log.Println("Control received", msg)

            valid := c.validate(msg)
            if valid {
                var payload map[string]interface{}
                err := json.Unmarshal([]byte(msg.String()), &payload)
                if err == nil {
                    cmd := msg.Cmd()
                    switch(cmd) {
                    case "register":
                    case "auth":
                    case "list_contacts":
                    case "add_contact":
                    case "remove_contact":
                    case "create_group":
                        groupId := c.CreateGroup()
                        c.AddToGroup(groupId, msg.From())
                    case "add_to_group":
                        groupId := payload["group_id"]
                        userId := payload["user_id"]
                        c.AddToGroup(groupId.(string), userId.(string))
                    case "remove_from_group":
                        groupId := payload["group_id"].(string)
                        userId := payload["user_id"].(string)
                        c.RemoveFromGroup(groupId, userId)
                    case "delete_group":
                        groupId := payload["group_id"].(string)
                        c.DeleteGroup(groupId)
                    case "list_groups":
                    case "join_group":
                    case "leave_group":
                    default:
                        log.Println("Unknown control command", cmd)
                    }
                } else {
                    log.Println("Failed to decode control message payload", msg)
                }
            } else {
                log.Println("Invalid control message", msg)
            }
        }
    }
}

func (c Control) validate(msg *common.Message) bool {
    return true
}

func (c Control) findGroup(groupId string) *Group {
    for _, g := range c.groups {
        if g.Id == groupId {
            return g
        }
    }

    return nil
}

func (c Control) RegisterUser(user string, password string) *common.User {
    return nil
}

func (c Control) AuthUser(user string, password string) bool {
    return false
}

func (c Control) ListContacts(user string) []*common.User {
    return nil
}

func (c Control) AddContact(user *common.User, contact *common.User) bool {
    return false
}

func (c Control) RemoveContact(user *common.User, contact *common.User) bool {
    return false
}

func (c *Control) CreateGroup() string {
    group := NewGroup()
    c.groups = append(c.groups, group)
    c.outputStore.AddOutput(group.Id, group.Out)
    return group.Id
}

func (c Control) AddToGroup(groupId string, userId string) bool {
    group := c.findGroup(groupId)
    if group == nil {
        log.Println("Failed to add user. No group with id", groupId, userId)
        return false
    }

    userOutput := c.outputStore.GetOutput(userId)
    if userOutput == nil {
        log.Println("Failed to add user to group. No user output",
                    groupId, userId)
        return false
    }

    return group.AddOutput(userId, userOutput)
}

func (c Control) RemoveFromGroup(groupId string, userId string) bool {
    group := c.findGroup(groupId)
    if group == nil {
        log.Printf("Failed to remove user %s from group %s. No group with id",
                   userId, groupId)
        return false
    }

    return group.RemoveOutput(userId)
}

func (c *Control) DeleteGroup(groupId string) bool {
    for i, group:= range c.groups {
        if group.Id == groupId {
            c.groups = append(c.groups[:i], c.groups[i+1:]...)
            c.outputStore.RemoveOutput(group.Id)
            close(group.Out)
            return true
        }
    }

    return false
}

func (c Control) ListGroups(user *common.User) *[]string {
    groupIds := []string{}
    for _, g := range c.groups {
        groupIds = append(groupIds, g.Id)
    }

    return &groupIds
}
<commit_msg>Add logging for control operation status<commit_after>package server

import (
    "github.com/itsankoff/gotcha/common"
    "log"
    "encoding/json"
)

type Control struct {
    input           chan *common.Message
    groups          []*Group
    outputStore     *OutputStore
}

func NewControl(input chan *common.Message,
                outputStore *OutputStore) *Control {
    c := &Control{
        input: input,
        outputStore: outputStore,
    }

    go c.listen()
    return c
}

func (c Control) listen() {
    for {
        select {
        case msg := <-c.input:
            log.Println("Control received", msg)

            valid := c.validate(msg)
            if valid {
                var payload map[string]interface{}
                err := json.Unmarshal([]byte(msg.String()), &payload)
                if err == nil {
                    cmd := msg.Cmd()
                    switch(cmd) {
                    case "register":
                    case "auth":
                    case "list_contacts":
                    case "add_contact":
                    case "remove_contact":
                    case "create_group":
                        groupId := c.CreateGroup()
                        c.AddToGroup(groupId, msg.From())
                        log.Println("Group created", groupId)
                    case "add_to_group":
                        groupId := payload["group_id"].(string)
                        userId := payload["user_id"].(string)
                        added := c.AddToGroup(groupId, userId)
                        log.Printf("User %s added to group %s %t",
                                   groupId, userId, added)
                    case "remove_from_group":
                        groupId := payload["group_id"].(string)
                        userId := payload["user_id"].(string)
                        removed := c.RemoveFromGroup(groupId, userId)
                        log.Printf("User %s removed from group %s %t",
                                   groupId, userId, removed)
                    case "delete_group":
                        groupId := payload["group_id"].(string)
                        deleted := c.DeleteGroup(groupId)
                        log.Printf("Group %s deleted %t", groupId, deleted)
                    case "list_groups":
                    case "join_group":
                    case "leave_group":
                    default:
                        log.Println("Unknown control command", cmd)
                    }
                } else {
                    log.Println("Failed to decode control message payload", msg)
                }
            } else {
                log.Println("Invalid control message", msg)
            }
        }
    }
}

func (c Control) validate(msg *common.Message) bool {
    return true
}

func (c Control) findGroup(groupId string) *Group {
    for _, g := range c.groups {
        if g.Id == groupId {
            return g
        }
    }

    return nil
}

func (c Control) RegisterUser(user string, password string) *common.User {
    return nil
}

func (c Control) AuthUser(user string, password string) bool {
    return false
}

func (c Control) ListContacts(user string) []*common.User {
    return nil
}

func (c Control) AddContact(user *common.User, contact *common.User) bool {
    return false
}

func (c Control) RemoveContact(user *common.User, contact *common.User) bool {
    return false
}

func (c *Control) CreateGroup() string {
    group := NewGroup()
    c.groups = append(c.groups, group)
    c.outputStore.AddOutput(group.Id, group.Out)
    return group.Id
}

func (c Control) AddToGroup(groupId string, userId string) bool {
    group := c.findGroup(groupId)
    if group == nil {
        log.Println("Failed to add user. No group with id", groupId, userId)
        return false
    }

    userOutput := c.outputStore.GetOutput(userId)
    if userOutput == nil {
        log.Println("Failed to add user to group. No user output",
                    groupId, userId)
        return false
    }

    return group.AddOutput(userId, userOutput)
}

func (c Control) RemoveFromGroup(groupId string, userId string) bool {
    group := c.findGroup(groupId)
    if group == nil {
        log.Printf("Failed to remove user %s from group %s. No group with id",
                   userId, groupId)
        return false
    }

    return group.RemoveOutput(userId)
}

func (c *Control) DeleteGroup(groupId string) bool {
    for i, group:= range c.groups {
        if group.Id == groupId {
            c.groups = append(c.groups[:i], c.groups[i+1:]...)
            c.outputStore.RemoveOutput(group.Id)
            close(group.Out)
            return true
        }
    }

    return false
}

func (c Control) ListGroups(user *common.User) *[]string {
    groupIds := []string{}
    for _, g := range c.groups {
        groupIds = append(groupIds, g.Id)
    }

    return &groupIds
}
<|endoftext|>


<commit_before>package heartbeat_test

import (
	"heartbeat"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/goadesign/goa"
)

func TestHeartbeat_DefaultURL(t *testing.T) {
	service := goa.New("API")
	heartbeat.Heartbeat(service, "")
	server := httptest.NewServer(service.Mux)

	res, err := http.Get(server.URL + "/health")
	if err != nil {
		t.Fatalf("Server error %s", err)
	}
	if res.StatusCode != http.StatusOK {
		t.Fatalf("Expected a 200 OK response, got %d", res.StatusCode)
	}

	server.Close()
}

func TestHeartbeat_CustomURL(t *testing.T) {
	heartbeatURL := "/custom"

	service := goa.New("API")
	heartbeat.Heartbeat(service, heartbeatURL)
	server := httptest.NewServer(service.Mux)

	res, err := http.Get(server.URL + heartbeatURL)
	if err != nil {
		t.Fatalf("Server error %s", err)
	}
	if res.StatusCode != http.StatusOK {
		t.Fatalf("Expected a 200 OK response, got %d", res.StatusCode)
	}

	server.Close()
}

func TestHeartbeat_404URL(t *testing.T) {
	service := goa.New("API")
	heartbeat.Heartbeat(service, "")
	server := httptest.NewServer(service.Mux)

	res, err := http.Get(server.URL + "/other-url-gives-a-404")
	if err != nil {
		t.Fatalf("Server error %s", err)
	}
	if res.StatusCode != http.StatusNotFound {
		t.Fatalf("Expected a 404 OK response, got %d", res.StatusCode)
	}

	server.Close()
}
<commit_msg>Fixing test import path.<commit_after>package heartbeat_test

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/goadesign/goa"
	"github.com/richardbolt/heartbeat"
)

func TestHeartbeat_DefaultURL(t *testing.T) {
	service := goa.New("API")
	heartbeat.Heartbeat(service, "")
	server := httptest.NewServer(service.Mux)

	res, err := http.Get(server.URL + "/health")
	if err != nil {
		t.Fatalf("Server error %s", err)
	}
	if res.StatusCode != http.StatusOK {
		t.Fatalf("Expected a 200 OK response, got %d", res.StatusCode)
	}

	server.Close()
}

func TestHeartbeat_CustomURL(t *testing.T) {
	heartbeatURL := "/custom"

	service := goa.New("API")
	heartbeat.Heartbeat(service, heartbeatURL)
	server := httptest.NewServer(service.Mux)

	res, err := http.Get(server.URL + heartbeatURL)
	if err != nil {
		t.Fatalf("Server error %s", err)
	}
	if res.StatusCode != http.StatusOK {
		t.Fatalf("Expected a 200 OK response, got %d", res.StatusCode)
	}

	server.Close()
}

func TestHeartbeat_404URL(t *testing.T) {
	service := goa.New("API")
	heartbeat.Heartbeat(service, "")
	server := httptest.NewServer(service.Mux)

	res, err := http.Get(server.URL + "/other-url-gives-a-404")
	if err != nil {
		t.Fatalf("Server error %s", err)
	}
	if res.StatusCode != http.StatusNotFound {
		t.Fatalf("Expected a 404 OK response, got %d", res.StatusCode)
	}

	server.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"sync"
	"time"

	"github.com/msoap/raphanus"
	tgbotapi "gopkg.in/telegram-bot-api.v2"
)

const (
	// Version - current version
	Version = "1.6"

	// DefaultBotTimeout - bot default timeout
	DefaultBotTimeout = 60

	// MessagesQueueSize - size of channel for bot messages
	MessagesQueueSize = 10

	// MaxMessageLength - max length of one bot message
	MaxMessageLength = 4096

	// SecondsForAutoSaveUsersToDB - save users to file every 1 min (if need)
	SecondsForAutoSaveUsersToDB = 60

	// DBFileName - DB json name
	DBFileName = "shell2telegram.json"

	// shell2telegram command name for get plain text without /command
	cmdPlainText = "/:plain_text"
)

// Command - one user command
type Command struct {
	shellCmd    string   // shell command
	description string   // command description for list in /help (/cmd:desc="Command name")
	vars        []string // environment vars for user text, split by `/s+` to vars (/cmd:vars=SUBCOMMAND,ARGS)
	isMarkdown  bool     // send message in markdown format
}

// Commands - list of all commands
type Commands map[string]Command

// Config - config struct
type Config struct {
	token                  string   // bot token
	botTimeout             int      // bot timeout
	predefinedAllowedUsers []string // telegram users who are allowed to chat with the bot
	predefinedRootUsers    []string // telegram users, who confirms new users in their private chat
	description            string   // description of bot
	usersDB                string   // file for store users
	shell                  string   // custom shell
	cache                  int      // caching command out (in seconds)
	shTimeout              int      // timeout for execute shell command (in seconds)
	addExit                bool     // adding /shell2telegram exit command
	allowAll               bool     // allow all user (DANGEROUS!)
	logCommands            bool     // logging all commands
	persistentUsers        bool     // load/save users from file
	isPublicBot            bool     // bot is public (dont add /auth* commands)
	oneThread              bool     // run each shell commands in one thread
}

// message types
const (
	msgIsText int8 = iota
	msgIsPhoto
)

// BotMessage - record for send via channel for send message to telegram chat
type BotMessage struct {
	message     string
	fileName    string
	photo       []byte
	chatID      int
	messageType int8
	isMarkdown  bool
}

// ----------------------------------------------------------------------------
// get config
func getConfig() (commands Commands, appConfig Config, err error) {
	flag.StringVar(&appConfig.token, "tb-token", "", "setting bot token (or set TB_TOKEN variable)")
	flag.BoolVar(&appConfig.addExit, "add-exit", false, "adding \"/shell2telegram exit\" command for terminate bot (for roots only)")
	flag.IntVar(&appConfig.botTimeout, "timeout", DefaultBotTimeout, "setting timeout for bot")
	flag.BoolVar(&appConfig.allowAll, "allow-all", false, "allow all users (DANGEROUS!)")
	flag.BoolVar(&appConfig.logCommands, "log-commands", false, "logging all commands")
	flag.StringVar(&appConfig.description, "description", "", "setting description of bot")
	flag.BoolVar(&appConfig.persistentUsers, "persistent-users", false, "load/save users from file (default ~/.config/shell2telegram.json)")
	flag.StringVar(&appConfig.usersDB, "users-db", "", "file for store users")
	flag.IntVar(&appConfig.cache, "cache", 0, "caching command out (in seconds)")
	flag.BoolVar(&appConfig.isPublicBot, "public", false, "bot is public (dont add /auth* commands)")
	flag.IntVar(&appConfig.shTimeout, "sh-timeout", 0, "set timeout for execute shell command (in seconds)")
	flag.StringVar(&appConfig.shell, "shell", "sh", "custom shell or \"\" for execute without shell")
	flag.BoolVar(&appConfig.oneThread, "one-thread", false, "run each shell command in one thread")
	logFilename := flag.String("log", "", "log filename, default - STDOUT")
	predefinedAllowedUsers := flag.String("allow-users", "", "telegram users who are allowed to chat with the bot (\"user1,user2\")")
	predefinedRootUsers := flag.String("root-users", "", "telegram users, who confirms new users in their private chat (\"user1,user2\")")
	version := flag.Bool("version", false, "get version")

	flag.Usage = func() {
		fmt.Printf("usage: %s [options] %s\n%s\n%s\n\noptions:\n",
			os.Args[0],
			`/chat_command "shell command" /chat_command2 "shell command2"`,
			"All text after /chat_command will be sent to STDIN of shell command.",
			"If chat command is /:plain_text - get user message without any /command (for private chats only)",
		)
		flag.PrintDefaults()
		os.Exit(0)
	}
	flag.Parse()

	if *version {
		fmt.Println(Version)
		os.Exit(0)
	}

	// setup log file
	if len(*logFilename) > 0 {
		fhLog, err := os.OpenFile(*logFilename, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
		if err != nil {
			log.Fatalf("error opening log file: %v", err)
		}
		log.SetOutput(fhLog)
	}

	// setup users and roots
	if *predefinedAllowedUsers != "" {
		appConfig.predefinedAllowedUsers = strings.Split(*predefinedAllowedUsers, ",")
	}
	if *predefinedRootUsers != "" {
		appConfig.predefinedRootUsers = strings.Split(*predefinedRootUsers, ",")
	}

	commands = Commands{}
	// need >= 2 arguments and count of it must be even
	args := flag.Args()
	if len(args) < 2 || len(args)%2 == 1 {
		return commands, appConfig, fmt.Errorf("error: need pairs of /chat-command and shell-command")
	}

	for i := 0; i < len(args); i += 2 {
		path, command, err := parseBotCommand(args[i], args[i+1]) // (/path, shell_command)
		if err != nil {
			return commands, appConfig, err
		}
		commands[path] = command
	}

	if appConfig.token == "" {
		if appConfig.token = os.Getenv("TB_TOKEN"); appConfig.token == "" {
			return commands, appConfig, fmt.Errorf("TB_TOKEN environment var not found. See https://core.telegram.org/bots#botfather for more information")
		}
	}

	return commands, appConfig, nil
}

// ----------------------------------------------------------------------------
func sendMessage(messageSignal chan<- BotMessage, chatID int, message []byte, isMarkdown bool) {
	go func() {
		var fileName string
		fileType := http.DetectContentType(message)
		switch fileType {
		case "image/png":
			fileName = "file.png"
		case "image/jpeg":
			fileName = "file.jpeg"
		case "image/gif":
			fileName = "file.gif"
		case "image/bmp":
			fileName = "file.bmp"
		default:
			fileName = "message"
		}

		if fileName == "message" {

			// is text message
			messageString := string(message)
			var messagesList []string

			if len(messageString) <= MaxMessageLength {
				messagesList = []string{messageString}
			} else {
				messagesList = splitStringLinesBySize(messageString, MaxMessageLength)
			}

			for _, messageChunk := range messagesList {
				messageSignal <- BotMessage{
					chatID:      chatID,
					messageType: msgIsText,
					message:     messageChunk,
					isMarkdown:  isMarkdown,
				}
			}

		} else {
			// is image
			messageSignal <- BotMessage{
				chatID:      chatID,
				messageType: msgIsPhoto,
				fileName:    fileName,
				photo:       message,
			}
		}
	}()
}

// ----------------------------------------------------------------------------
func main() {
	commands, appConfig, err := getConfig()
	if err != nil {
		log.Fatal(err)
	}

	bot, err := tgbotapi.NewBotAPI(appConfig.token)
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("Authorized on bot account: @%s", bot.Self.UserName)

	tgbotConfig := tgbotapi.NewUpdate(0)
	tgbotConfig.Timeout = appConfig.botTimeout
	botUpdatesChan, err := bot.GetUpdatesChan(tgbotConfig)
	if err != nil {
		log.Fatal(err)
	}

	users := NewUsers(appConfig)
	messageSignal := make(chan BotMessage, MessagesQueueSize)
	vacuumTicker := time.Tick(SecondsForOldUsersBeforeVacuum * time.Second)
	saveToBDTicker := make(<-chan time.Time)
	oneThreadMutex := sync.Mutex{}
	exitSignal := make(chan struct{})
	systemExitSignal := make(chan os.Signal, 1)
	signal.Notify(systemExitSignal, os.Interrupt)

	if appConfig.persistentUsers {
		saveToBDTicker = time.Tick(SecondsForAutoSaveUsersToDB * time.Second)
	}

	var cache raphanus.DB
	if appConfig.cache > 0 {
		cache = raphanus.New()
	}

	// all /shell2telegram sub-commands handlers
	internalCommands := map[string]func(Ctx) string{
		"stat":              cmdShell2telegramStat,
		"ban":               cmdShell2telegramBan,
		"search":            cmdShell2telegramSearch,
		"desc":              cmdShell2telegramDesc,
		"rm":                cmdShell2telegramRm,
		"exit":              cmdShell2telegramExit,
		"version":           cmdShell2telegramVersion,
		"broadcast_to_root": cmdShell2telegramBroadcastToRoot,
		"message_to_user":   cmdShell2telegramMessageToUser,
	}

	doExit := false
	for !doExit {
		select {
		case telegramUpdate := <-botUpdatesChan:

			var messageCmd, messageArgs string
			allUserMessage := telegramUpdate.Message.Text
			if len(allUserMessage) > 0 && allUserMessage[0] == '/' {
				messageCmd, messageArgs = splitStringHalfBySpace(allUserMessage)
			} else {
				messageCmd, messageArgs = cmdPlainText, allUserMessage
			}

			allowPlainText := false
			if _, ok := commands[cmdPlainText]; ok {
				allowPlainText = true
			}

			replayMsg := ""

			if len(messageCmd) > 0 && (messageCmd != cmdPlainText || allowPlainText) {

				users.AddNew(telegramUpdate.Message)
				userID := telegramUpdate.Message.From.ID
				allowExec := appConfig.allowAll || users.IsAuthorized(userID)

				ctx := Ctx{
					appConfig:      &appConfig,
					users:          &users,
					commands:       commands,
					userID:         userID,
					allowExec:      allowExec,
					messageCmd:     messageCmd,
					messageArgs:    messageArgs,
					messageSignal:  messageSignal,
					chatID:         telegramUpdate.Message.Chat.ID,
					exitSignal:     exitSignal,
					cache:          &cache,
					oneThreadMutex: &oneThreadMutex,
				}

				switch {
				// commands .................................
				case !appConfig.isPublicBot && (messageCmd == "/auth" || messageCmd == "/authroot"):
					replayMsg = cmdAuth(ctx)

				case messageCmd == "/help":
					replayMsg = cmdHelp(ctx)

				case messageCmd == "/shell2telegram" && users.IsRoot(userID):
					var messageSubCmd string
					messageSubCmd, messageArgs = splitStringHalfBySpace(messageArgs)
					ctx.messageArgs = messageArgs
					if cmdHandler, ok := internalCommands[messageSubCmd]; ok {
						replayMsg = cmdHandler(ctx)
					} else {
						replayMsg = "Sub-command not found"
					}

				case allowExec && (allowPlainText && messageCmd == cmdPlainText || messageCmd[0] == '/'):
					cmdUser(ctx)

				} // switch for commands

				if appConfig.logCommands {
					log.Printf("%s: %s", users.String(userID), allUserMessage)
				}

				sendMessage(messageSignal, telegramUpdate.Message.Chat.ID, []byte(replayMsg), false)
			}

		case botMessage := <-messageSignal:
			switch {
			case botMessage.messageType == msgIsText && !stringIsEmpty(botMessage.message):
				messageConfig := tgbotapi.NewMessage(botMessage.chatID, botMessage.message)
				if botMessage.isMarkdown {
					messageConfig.ParseMode = tgbotapi.ModeMarkdown
				}
				_, err = bot.Send(messageConfig)
			case botMessage.messageType == msgIsPhoto && len(botMessage.photo) > 0:
				bytesPhoto := tgbotapi.FileBytes{Name: botMessage.fileName, Bytes: botMessage.photo}
				_, err = bot.Send(tgbotapi.NewPhotoUpload(botMessage.chatID, bytesPhoto))
			}

			if err != nil {
				log.Print("Bot send message error: ", err)
			}

		case <-saveToBDTicker:
			users.SaveToDB(appConfig.usersDB)

		case <-vacuumTicker:
			users.ClearOldUsers()

		case <-systemExitSignal:
			go func() {
				exitSignal <- struct{}{}
			}()

		case <-exitSignal:
			if appConfig.persistentUsers {
				users.needSaveDB = true
				users.SaveToDB(appConfig.usersDB)
			}
			doExit = true
		}
	}
}
<commit_msg>Fixed gofmt check for the latest Go from master<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"sync"
	"time"

	"github.com/msoap/raphanus"
	tgbotapi "gopkg.in/telegram-bot-api.v2"
)

const (
	// Version - current version
	Version = "1.6"

	// DefaultBotTimeout - bot default timeout
	DefaultBotTimeout = 60

	// MessagesQueueSize - size of channel for bot messages
	MessagesQueueSize = 10

	// MaxMessageLength - max length of one bot message
	MaxMessageLength = 4096

	// SecondsForAutoSaveUsersToDB - save users to file every 1 min (if need)
	SecondsForAutoSaveUsersToDB = 60

	// DBFileName - DB json name
	DBFileName = "shell2telegram.json"

	// shell2telegram command name for get plain text without /command
	cmdPlainText = "/:plain_text"
)

// Command - one user command
type Command struct {
	shellCmd    string   // shell command
	description string   // command description for list in /help (/cmd:desc="Command name")
	vars        []string // environment vars for user text, split by `/s+` to vars (/cmd:vars=SUBCOMMAND,ARGS)
	isMarkdown  bool     // send message in markdown format
}

// Commands - list of all commands
type Commands map[string]Command

// Config - config struct
type Config struct {
	token                  string   // bot token
	botTimeout             int      // bot timeout
	predefinedAllowedUsers []string // telegram users who are allowed to chat with the bot
	predefinedRootUsers    []string // telegram users, who confirms new users in their private chat
	description            string   // description of bot
	usersDB                string   // file for store users
	shell                  string   // custom shell
	cache                  int      // caching command out (in seconds)
	shTimeout              int      // timeout for execute shell command (in seconds)
	addExit                bool     // adding /shell2telegram exit command
	allowAll               bool     // allow all user (DANGEROUS!)
	logCommands            bool     // logging all commands
	persistentUsers        bool     // load/save users from file
	isPublicBot            bool     // bot is public (dont add /auth* commands)
	oneThread              bool     // run each shell commands in one thread
}

// message types
const (
	msgIsText int8 = iota
	msgIsPhoto
)

// BotMessage - record for send via channel for send message to telegram chat
type BotMessage struct {
	message     string
	fileName    string
	photo       []byte
	chatID      int
	messageType int8
	isMarkdown  bool
}

// ----------------------------------------------------------------------------
// get config
func getConfig() (commands Commands, appConfig Config, err error) {
	flag.StringVar(&appConfig.token, "tb-token", "", "setting bot token (or set TB_TOKEN variable)")
	flag.BoolVar(&appConfig.addExit, "add-exit", false, "adding \"/shell2telegram exit\" command for terminate bot (for roots only)")
	flag.IntVar(&appConfig.botTimeout, "timeout", DefaultBotTimeout, "setting timeout for bot")
	flag.BoolVar(&appConfig.allowAll, "allow-all", false, "allow all users (DANGEROUS!)")
	flag.BoolVar(&appConfig.logCommands, "log-commands", false, "logging all commands")
	flag.StringVar(&appConfig.description, "description", "", "setting description of bot")
	flag.BoolVar(&appConfig.persistentUsers, "persistent-users", false, "load/save users from file (default ~/.config/shell2telegram.json)")
	flag.StringVar(&appConfig.usersDB, "users-db", "", "file for store users")
	flag.IntVar(&appConfig.cache, "cache", 0, "caching command out (in seconds)")
	flag.BoolVar(&appConfig.isPublicBot, "public", false, "bot is public (dont add /auth* commands)")
	flag.IntVar(&appConfig.shTimeout, "sh-timeout", 0, "set timeout for execute shell command (in seconds)")
	flag.StringVar(&appConfig.shell, "shell", "sh", "custom shell or \"\" for execute without shell")
	flag.BoolVar(&appConfig.oneThread, "one-thread", false, "run each shell command in one thread")
	logFilename := flag.String("log", "", "log filename, default - STDOUT")
	predefinedAllowedUsers := flag.String("allow-users", "", "telegram users who are allowed to chat with the bot (\"user1,user2\")")
	predefinedRootUsers := flag.String("root-users", "", "telegram users, who confirms new users in their private chat (\"user1,user2\")")
	version := flag.Bool("version", false, "get version")

	flag.Usage = func() {
		fmt.Printf("usage: %s [options] %s\n%s\n%s\n\noptions:\n",
			os.Args[0],
			`/chat_command "shell command" /chat_command2 "shell command2"`,
			"All text after /chat_command will be sent to STDIN of shell command.",
			"If chat command is /:plain_text - get user message without any /command (for private chats only)",
		)
		flag.PrintDefaults()
		os.Exit(0)
	}
	flag.Parse()

	if *version {
		fmt.Println(Version)
		os.Exit(0)
	}

	// setup log file
	if len(*logFilename) > 0 {
		fhLog, err := os.OpenFile(*logFilename, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
		if err != nil {
			log.Fatalf("error opening log file: %v", err)
		}
		log.SetOutput(fhLog)
	}

	// setup users and roots
	if *predefinedAllowedUsers != "" {
		appConfig.predefinedAllowedUsers = strings.Split(*predefinedAllowedUsers, ",")
	}
	if *predefinedRootUsers != "" {
		appConfig.predefinedRootUsers = strings.Split(*predefinedRootUsers, ",")
	}

	commands = Commands{}
	// need >= 2 arguments and count of it must be even
	args := flag.Args()
	if len(args) < 2 || len(args)%2 == 1 {
		return commands, appConfig, fmt.Errorf("error: need pairs of /chat-command and shell-command")
	}

	for i := 0; i < len(args); i += 2 {
		path, command, err := parseBotCommand(args[i], args[i+1]) // (/path, shell_command)
		if err != nil {
			return commands, appConfig, err
		}
		commands[path] = command
	}

	if appConfig.token == "" {
		if appConfig.token = os.Getenv("TB_TOKEN"); appConfig.token == "" {
			return commands, appConfig, fmt.Errorf("TB_TOKEN environment var not found. See https://core.telegram.org/bots#botfather for more information")
		}
	}

	return commands, appConfig, nil
}

// ----------------------------------------------------------------------------
func sendMessage(messageSignal chan<- BotMessage, chatID int, message []byte, isMarkdown bool) {
	go func() {
		var fileName string
		fileType := http.DetectContentType(message)
		switch fileType {
		case "image/png":
			fileName = "file.png"
		case "image/jpeg":
			fileName = "file.jpeg"
		case "image/gif":
			fileName = "file.gif"
		case "image/bmp":
			fileName = "file.bmp"
		default:
			fileName = "message"
		}

		if fileName == "message" {
			// is text message
			messageString := string(message)
			var messagesList []string

			if len(messageString) <= MaxMessageLength {
				messagesList = []string{messageString}
			} else {
				messagesList = splitStringLinesBySize(messageString, MaxMessageLength)
			}

			for _, messageChunk := range messagesList {
				messageSignal <- BotMessage{
					chatID:      chatID,
					messageType: msgIsText,
					message:     messageChunk,
					isMarkdown:  isMarkdown,
				}
			}

		} else {
			// is image
			messageSignal <- BotMessage{
				chatID:      chatID,
				messageType: msgIsPhoto,
				fileName:    fileName,
				photo:       message,
			}
		}
	}()
}

// ----------------------------------------------------------------------------
func main() {
	commands, appConfig, err := getConfig()
	if err != nil {
		log.Fatal(err)
	}

	bot, err := tgbotapi.NewBotAPI(appConfig.token)
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("Authorized on bot account: @%s", bot.Self.UserName)

	tgbotConfig := tgbotapi.NewUpdate(0)
	tgbotConfig.Timeout = appConfig.botTimeout
	botUpdatesChan, err := bot.GetUpdatesChan(tgbotConfig)
	if err != nil {
		log.Fatal(err)
	}

	users := NewUsers(appConfig)
	messageSignal := make(chan BotMessage, MessagesQueueSize)
	vacuumTicker := time.Tick(SecondsForOldUsersBeforeVacuum * time.Second)
	saveToBDTicker := make(<-chan time.Time)
	oneThreadMutex := sync.Mutex{}
	exitSignal := make(chan struct{})
	systemExitSignal := make(chan os.Signal, 1)
	signal.Notify(systemExitSignal, os.Interrupt)

	if appConfig.persistentUsers {
		saveToBDTicker = time.Tick(SecondsForAutoSaveUsersToDB * time.Second)
	}

	var cache raphanus.DB
	if appConfig.cache > 0 {
		cache = raphanus.New()
	}

	// all /shell2telegram sub-commands handlers
	internalCommands := map[string]func(Ctx) string{
		"stat":              cmdShell2telegramStat,
		"ban":               cmdShell2telegramBan,
		"search":            cmdShell2telegramSearch,
		"desc":              cmdShell2telegramDesc,
		"rm":                cmdShell2telegramRm,
		"exit":              cmdShell2telegramExit,
		"version":           cmdShell2telegramVersion,
		"broadcast_to_root": cmdShell2telegramBroadcastToRoot,
		"message_to_user":   cmdShell2telegramMessageToUser,
	}

	doExit := false
	for !doExit {
		select {
		case telegramUpdate := <-botUpdatesChan:

			var messageCmd, messageArgs string
			allUserMessage := telegramUpdate.Message.Text
			if len(allUserMessage) > 0 && allUserMessage[0] == '/' {
				messageCmd, messageArgs = splitStringHalfBySpace(allUserMessage)
			} else {
				messageCmd, messageArgs = cmdPlainText, allUserMessage
			}

			allowPlainText := false
			if _, ok := commands[cmdPlainText]; ok {
				allowPlainText = true
			}

			replayMsg := ""

			if len(messageCmd) > 0 && (messageCmd != cmdPlainText || allowPlainText) {

				users.AddNew(telegramUpdate.Message)
				userID := telegramUpdate.Message.From.ID
				allowExec := appConfig.allowAll || users.IsAuthorized(userID)

				ctx := Ctx{
					appConfig:      &appConfig,
					users:          &users,
					commands:       commands,
					userID:         userID,
					allowExec:      allowExec,
					messageCmd:     messageCmd,
					messageArgs:    messageArgs,
					messageSignal:  messageSignal,
					chatID:         telegramUpdate.Message.Chat.ID,
					exitSignal:     exitSignal,
					cache:          &cache,
					oneThreadMutex: &oneThreadMutex,
				}

				switch {
				// commands .................................
				case !appConfig.isPublicBot && (messageCmd == "/auth" || messageCmd == "/authroot"):
					replayMsg = cmdAuth(ctx)

				case messageCmd == "/help":
					replayMsg = cmdHelp(ctx)

				case messageCmd == "/shell2telegram" && users.IsRoot(userID):
					var messageSubCmd string
					messageSubCmd, messageArgs = splitStringHalfBySpace(messageArgs)
					ctx.messageArgs = messageArgs
					if cmdHandler, ok := internalCommands[messageSubCmd]; ok {
						replayMsg = cmdHandler(ctx)
					} else {
						replayMsg = "Sub-command not found"
					}

				case allowExec && (allowPlainText && messageCmd == cmdPlainText || messageCmd[0] == '/'):
					cmdUser(ctx)

				} // switch for commands

				if appConfig.logCommands {
					log.Printf("%s: %s", users.String(userID), allUserMessage)
				}

				sendMessage(messageSignal, telegramUpdate.Message.Chat.ID, []byte(replayMsg), false)
			}

		case botMessage := <-messageSignal:
			switch {
			case botMessage.messageType == msgIsText && !stringIsEmpty(botMessage.message):
				messageConfig := tgbotapi.NewMessage(botMessage.chatID, botMessage.message)
				if botMessage.isMarkdown {
					messageConfig.ParseMode = tgbotapi.ModeMarkdown
				}
				_, err = bot.Send(messageConfig)
			case botMessage.messageType == msgIsPhoto && len(botMessage.photo) > 0:
				bytesPhoto := tgbotapi.FileBytes{Name: botMessage.fileName, Bytes: botMessage.photo}
				_, err = bot.Send(tgbotapi.NewPhotoUpload(botMessage.chatID, bytesPhoto))
			}

			if err != nil {
				log.Print("Bot send message error: ", err)
			}

		case <-saveToBDTicker:
			users.SaveToDB(appConfig.usersDB)

		case <-vacuumTicker:
			users.ClearOldUsers()

		case <-systemExitSignal:
			go func() {
				exitSignal <- struct{}{}
			}()

		case <-exitSignal:
			if appConfig.persistentUsers {
				users.needSaveDB = true
				users.SaveToDB(appConfig.usersDB)
			}
			doExit = true
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"time"

	"github.com/apigee/apigee-remote-service-golib/analytics"
	"github.com/apigee/apigee-remote-service-golib/auth"
	"github.com/apigee/apigee-remote-service-golib/product"
	"github.com/apigee/apigee-remote-service-golib/quota"
)

// A Handler is the main entry
type Handler struct {
	internalAPI        *url.URL
	remoteServiceAPI   *url.URL
	orgName            string
	envName            string
	key                string
	secret             string
	apiKeyClaim        string
	apiKeyHeader       string
	targetHeader       string
	rejectUnauthorized bool

	productMan   product.Manager
	authMan      auth.Manager
	analyticsMan analytics.Manager
	quotaMan     quota.Manager
}

// InternalAPI is the internal api base (legacy)
func (h *Handler) InternalAPI() *url.URL {
	return h.internalAPI
}

// RemoteServiceAPI is the remote service base
func (h *Handler) RemoteServiceAPI() *url.URL {
	return h.remoteServiceAPI
}

// Organization is the tenant organization
func (h *Handler) Organization() string {
	return h.orgName
}

// Environment is the tenant environment
func (h *Handler) Environment() string {
	return h.envName
}

// Key is the access key for the remote service
func (h *Handler) Key() string {
	return h.key
}

// Secret is the access secret for the remote service
func (h *Handler) Secret() string {
	return h.secret
}

// NewHandler creates a handler
func NewHandler(config *Config) (*Handler, error) {

	var internalAPI, remoteServiceAPI *url.URL
	var err error
	if config.Tenant.InternalAPI != "" {
		internalAPI, err = url.Parse(config.Tenant.InternalAPI)
		if err != nil {
			return nil, err
		}
		if internalAPI.Scheme == "" {
			return nil, fmt.Errorf("invalid URL: %s", config.Tenant.InternalAPI)
		}
	}
	if config.Tenant.RemoteServiceAPI != "" {
		remoteServiceAPI, err = url.Parse(config.Tenant.RemoteServiceAPI)
		if err != nil {
			return nil, err
		}
		if remoteServiceAPI.Scheme == "" {
			return nil, fmt.Errorf("invalid URL: %s", config.Tenant.RemoteServiceAPI)
		}
	}

	tr := http.DefaultTransport
	if config.Tenant.AllowUnverifiedSSLCert {
		tr = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			DialContext: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
				DualStack: true,
			}).DialContext,
			MaxIdleConns:          100,
			IdleConnTimeout:       90 * time.Second,
			TLSHandshakeTimeout:   10 * time.Second,
			ExpectContinueTimeout: 1 * time.Second,
			TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
		}
	}
	httpClient := &http.Client{
		Timeout:   config.Tenant.ClientTimeout,
		Transport: tr,
	}

	productMan, err := product.NewManager(product.Options{
		Client:      httpClient,
		BaseURL:     remoteServiceAPI,
		RefreshRate: config.Products.RefreshRate,
		Key:         config.Tenant.Key,
		Secret:      config.Tenant.Secret,
	})
	if err != nil {
		return nil, err
	}

	authMan, err := auth.NewManager(auth.Options{
		PollInterval:        config.Auth.JWKSPollInterval,
		Client:              httpClient,
		APIKeyCacheDuration: config.Auth.APIKeyCacheDuration,
	})
	if err != nil {
		return nil, err
	}

	quotaMan, err := quota.NewManager(quota.Options{
		BaseURL: remoteServiceAPI,
		Client:  httpClient,
		Key:     config.Tenant.Key,
		Secret:  config.Tenant.Secret,
	})
	if err != nil {
		return nil, err
	}

	tempDirMode := os.FileMode(0700)
	tempDir := config.Global.TempDir
	analyticsDir := filepath.Join(tempDir, "analytics")
	if err := os.MkdirAll(analyticsDir, tempDirMode); err != nil {
		return nil, err
	}

	analyticsMan, err := analytics.NewManager(analytics.Options{
		LegacyEndpoint:     false,
		BufferPath:         analyticsDir,
		StagingFileLimit:   2024,
		BaseURL:            internalAPI,
		Key:                config.Tenant.Key,
		Secret:             config.Tenant.Secret,
		Client:             httpClient,
		SendChannelSize:    10,
		CollectionInterval: time.Minute,
		FluentdEndpoint:    config.Analytics.FluentdEndpoint,
		TLSCAFile:          config.Analytics.TLS.CAFile,
		TLSCertFile:        config.Analytics.TLS.CertFile,
		TLSKeyFile:         config.Analytics.TLS.KeyFile,
		TLSSkipVerify:      config.Analytics.TLS.AllowUnverifiedSSLCert,
	})
	if err != nil {
		return nil, err
	}

	h := &Handler{
		remoteServiceAPI:   remoteServiceAPI,
		internalAPI:        internalAPI,
		orgName:            config.Tenant.OrgName,
		envName:            config.Tenant.EnvName,
		key:                config.Tenant.Key,
		secret:             config.Tenant.Secret,
		productMan:         productMan,
		authMan:            authMan,
		analyticsMan:       analyticsMan,
		quotaMan:           quotaMan,
		apiKeyClaim:        config.Auth.APIKeyClaim,
		apiKeyHeader:       config.Auth.APIKeyHeader,
		targetHeader:       config.Auth.TargetHeader,
		rejectUnauthorized: config.Auth.RejectUnauthorized,
	}

	return h, nil
}
<commit_msg>handle config options<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"time"

	"github.com/apigee/apigee-remote-service-golib/analytics"
	"github.com/apigee/apigee-remote-service-golib/auth"
	"github.com/apigee/apigee-remote-service-golib/product"
	"github.com/apigee/apigee-remote-service-golib/quota"
)

// A Handler is the main entry
type Handler struct {
	internalAPI        *url.URL
	remoteServiceAPI   *url.URL
	orgName            string
	envName            string
	key                string
	secret             string
	apiKeyClaim        string
	apiKeyHeader       string
	targetHeader       string
	rejectUnauthorized bool

	productMan   product.Manager
	authMan      auth.Manager
	analyticsMan analytics.Manager
	quotaMan     quota.Manager
}

// InternalAPI is the internal api base (legacy)
func (h *Handler) InternalAPI() *url.URL {
	return h.internalAPI
}

// RemoteServiceAPI is the remote service base
func (h *Handler) RemoteServiceAPI() *url.URL {
	return h.remoteServiceAPI
}

// Organization is the tenant organization
func (h *Handler) Organization() string {
	return h.orgName
}

// Environment is the tenant environment
func (h *Handler) Environment() string {
	return h.envName
}

// Key is the access key for the remote service
func (h *Handler) Key() string {
	return h.key
}

// Secret is the access secret for the remote service
func (h *Handler) Secret() string {
	return h.secret
}

// NewHandler creates a handler
func NewHandler(config *Config) (*Handler, error) {

	var internalAPI, remoteServiceAPI *url.URL
	var err error
	if config.Tenant.InternalAPI != "" {
		internalAPI, err = url.Parse(config.Tenant.InternalAPI)
		if err != nil {
			return nil, err
		}
		if internalAPI.Scheme == "" {
			return nil, fmt.Errorf("invalid URL: %s", config.Tenant.InternalAPI)
		}
	}
	if config.Tenant.RemoteServiceAPI != "" {
		remoteServiceAPI, err = url.Parse(config.Tenant.RemoteServiceAPI)
		if err != nil {
			return nil, err
		}
		if remoteServiceAPI.Scheme == "" {
			return nil, fmt.Errorf("invalid URL: %s", config.Tenant.RemoteServiceAPI)
		}
	}

	tr := http.DefaultTransport
	if config.Tenant.AllowUnverifiedSSLCert {
		tr = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			DialContext: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
				DualStack: true,
			}).DialContext,
			MaxIdleConns:          100,
			IdleConnTimeout:       90 * time.Second,
			TLSHandshakeTimeout:   10 * time.Second,
			ExpectContinueTimeout: 1 * time.Second,
			TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
		}
	}
	httpClient := &http.Client{
		Timeout:   config.Tenant.ClientTimeout,
		Transport: tr,
	}

	productMan, err := product.NewManager(product.Options{
		Client:      httpClient,
		BaseURL:     remoteServiceAPI,
		RefreshRate: config.Products.RefreshRate,
		Key:         config.Tenant.Key,
		Secret:      config.Tenant.Secret,
	})
	if err != nil {
		return nil, err
	}

	authMan, err := auth.NewManager(auth.Options{
		PollInterval:        config.Auth.JWKSPollInterval,
		Client:              httpClient,
		APIKeyCacheDuration: config.Auth.APIKeyCacheDuration,
	})
	if err != nil {
		return nil, err
	}

	quotaMan, err := quota.NewManager(quota.Options{
		BaseURL: remoteServiceAPI,
		Client:  httpClient,
		Key:     config.Tenant.Key,
		Secret:  config.Tenant.Secret,
	})
	if err != nil {
		return nil, err
	}

	tempDirMode := os.FileMode(0700)
	tempDir := config.Global.TempDir
	analyticsDir := filepath.Join(tempDir, "analytics")
	if err := os.MkdirAll(analyticsDir, tempDirMode); err != nil {
		return nil, err
	}

	analyticsMan, err := analytics.NewManager(analytics.Options{
		LegacyEndpoint:     config.Analytics.LegacyEndpoint,
		BufferPath:         analyticsDir,
		StagingFileLimit:   config.Analytics.FileLimit,
		BaseURL:            internalAPI,
		Key:                config.Tenant.Key,
		Secret:             config.Tenant.Secret,
		Client:             httpClient,
		SendChannelSize:    config.Analytics.SendChannelSize,
		CollectionInterval: time.Minute,
		FluentdEndpoint:    config.Analytics.FluentdEndpoint,
		TLSCAFile:          config.Analytics.TLS.CAFile,
		TLSCertFile:        config.Analytics.TLS.CertFile,
		TLSKeyFile:         config.Analytics.TLS.KeyFile,
		TLSSkipVerify:      config.Analytics.TLS.AllowUnverifiedSSLCert,
	})
	if err != nil {
		return nil, err
	}

	h := &Handler{
		remoteServiceAPI:   remoteServiceAPI,
		internalAPI:        internalAPI,
		orgName:            config.Tenant.OrgName,
		envName:            config.Tenant.EnvName,
		key:                config.Tenant.Key,
		secret:             config.Tenant.Secret,
		productMan:         productMan,
		authMan:            authMan,
		analyticsMan:       analyticsMan,
		quotaMan:           quotaMan,
		apiKeyClaim:        config.Auth.APIKeyClaim,
		apiKeyHeader:       config.Auth.APIKeyHeader,
		targetHeader:       config.Auth.TargetHeader,
		rejectUnauthorized: config.Auth.RejectUnauthorized,
	}

	return h, nil
}
<|endoftext|>