<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package Scanner

export EOF;
const (
	ILLEGAL = iota;
	EOF = iota;
	IDENT = iota;
	STRING = iota;
	NUMBER = iota;

	COMMA = iota;
	COLON = iota;
	SEMICOLON = iota;
	PERIOD = iota;

	LPAREN = iota;
	RPAREN = iota;
	LBRACK = iota;
	RBRACK = iota;
	LBRACE = iota;
	RBRACE = iota;

	ASSIGN = iota;
	DEFINE = iota;

	INC = iota;
	DEC = iota;
	NOT = iota;

	OR = iota;
	BOR = iota;
	AND = iota;
	BAND = iota;

	ADD = iota;
	SUB = iota;
	MUL = iota;
	QUO = iota;
	REM = iota;

	EQL = iota;
	NEQ = iota;
	LSS = iota;
	LEQ = iota;
	GTR = iota;
	GEQ = iota;

	// keywords
	BREAK = iota;
	CASE = iota;
	CONST = iota;
	CONTINUE = iota;
	DEFAULT = iota;
	ELSE = iota;
	EXPORT = iota;
	FALLTHROUGH = iota;
	FALSE = iota;
	FOR = iota;
	FUNC = iota;
	GO = iota;
	GOTO = iota;
	IF = iota;
	IMPORT = iota;
	INTERFACE = iota;
	MAP = iota;
	NEW = iota;
	NIL = iota;
	PACKAGE = iota;
	RANGE = iota;
	RETURN = iota;
	SELECT = iota;
	STRUCT = iota;
	SWITCH = iota;
	TRUE = iota;
	TYPE = iota;
	VAR = iota;
)

var (
	Keywords *map [string] int;
)

export TokenName
func TokenName(tok int) string {
	switch (tok) {
	case ILLEGAL: return "ILLEGAL";
	case EOF: return "EOF";
	case IDENT: return "IDENT";
	case STRING: return "STRING";
	case NUMBER: return "NUMBER";

	case COMMA: return "COMMA";
	case COLON: return "COLON";
	case SEMICOLON: return "SEMICOLON";
	case PERIOD: return "PERIOD";

	case LPAREN: return "LPAREN";
	case RPAREN: return "RPAREN";
	case LBRACK: return "LBRACK";
	case RBRACK: return "RBRACK";
	case LBRACE: return "LBRACE";
	case RBRACE: return "RBRACE";

	case ASSIGN: return "ASSIGN";
	case DEFINE: return "DEFINE";

	case INC: return "INC";
	case DEC: return "DEC";
	case NOT: return "NOT";

	case OR: return "OR";
	case BOR: return "BOR";
	case AND: return "AND";
	case BAND: return "BAND";

	case ADD: return "ADD";
	case SUB: return "SUB";
	case MUL: return "MUL";
	case REM: return "REM";
	case QUO: return "QUO";
	case REM: return "REM";

	case EQL: return "EQL";
	case NEQ: return "NEQ";
	case LSS: return "LSS";
	case LEQ: return "LEQ";
	case GTR: return "GTR";
	case GEQ: return "GEQ";

	case BREAK: return "BREAK";
	case CASE: return "CASE";
	case CONST: return "CONST";
	case CONTINUE: return "CONTINUE";
	case DEFAULT: return "DEFAULT";
	case ELSE: return "ELSE";
	case EXPORT: return "EXPORT";
	case FALLTHROUGH: return "FALLTHROUGH";
	case FALSE: return "FALSE";
	case FOR: return "FOR";
	case FUNC: return "FUNC";
	case GO: return "GO";
	case GOTO: return "GOTO";
	case IF: return "IF";
	case IMPORT: return "IMPORT";
	case INTERFACE: return "INTERFACE";
	case MAP: return "MAP";
	case NEW: return "NEW";
	case NIL: return "NIL";
	case PACKAGE: return "PACKAGE";
	case RANGE: return "RANGE";
	case RETURN: return "RETURN";
	case SELECT: return "SELECT";
	case STRUCT: return "STRUCT";
	case SWITCH: return "SWITCH";
	case TRUE: return "TRUE";
	case TYPE: return "TYPE";
	case VAR: return "VAR";
	}

	return "???";
}

func is_whitespace (ch int) bool {
	return ch == ' ' || ch == '\r' || ch == '\n' || ch == '\t';
}

func is_letter (ch int) bool {
	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= 128 ;
}

func is_oct_digit (ch int) bool {
	return '0' <= ch && ch <= '7';
}

func is_dec_digit (ch int) bool {
	return '0' <= ch && ch <= '9';
}

func is_hex_digit (ch int) bool {
	return '0' <= ch && ch <= '9' || 'a' <= ch && ch <= 'f' || 'A' <= ch && ch <= 'F';
}

export Scanner
type Scanner struct {
	src string;
	pos int;
	ch int;  // one char look-ahead
}

func (S *Scanner) Next () {
	src := S.src;  // TODO only needed because of 6g bug
	if S.pos < len(src) {
		S.ch = int(S.src[S.pos]);
		S.pos++;
		if (S.ch >= 128) {
			panic "UTF-8 not handled"
		}
	} else {
		S.ch = -1;
	}
}

func Init () {
	Keywords = new(map [string] int);

	Keywords["break"] = BREAK;
	Keywords["case"] = CASE;
	Keywords["const"] = CONST;
	Keywords["continue"] = CONTINUE;
	Keywords["default"] = DEFAULT;
	Keywords["else"] = ELSE;
	Keywords["export"] = EXPORT;
	Keywords["fallthrough"] = FALLTHROUGH;
	Keywords["false"] = FALSE;
	Keywords["for"] = FOR;
	Keywords["func"] = FUNC;
	Keywords["go"] = GO;
	Keywords["goto"] = GOTO;
	Keywords["if"] = IF;
	Keywords["import"] = IMPORT;
	Keywords["interface"] = INTERFACE;
	Keywords["map"] = MAP;
	Keywords["new"] = NEW;
	Keywords["nil"] = NIL;
	Keywords["package"] = PACKAGE;
	Keywords["range"] = RANGE;
	Keywords["return"] = RETURN;
	Keywords["select"] = SELECT;
	Keywords["struct"] = STRUCT;
	Keywords["switch"] = SWITCH;
	Keywords["true"] = TRUE;
	Keywords["type"] = TYPE;
	Keywords["var"] = VAR;
}

func (S *Scanner) Open (src string) {
	if Keywords == nil {
		Init();
	}

	S.src = src;
	S.pos = 0;
	S.Next();
}

func (S *Scanner) SkipWhitespace () {
	for is_whitespace(S.ch) {
		S.Next();
	}
}

func (S *Scanner) SkipComment () {
	if S.ch == '/' {
		// comment
		for S.Next(); S.ch != '\n' && S.ch >= 0; S.Next() {}

	} else {
		/* comment */
		for S.Next(); S.ch >= 0; {
			c := S.ch;
			S.Next();
			if c == '*' && S.ch == '/' {
				S.Next();
				return;
			}
		}
		panic "comment not terminated";
	}
}

func (S *Scanner) ScanIdentifier () int {
	beg := S.pos - 1;
	for is_letter(S.ch) || is_dec_digit(S.ch) {
		S.Next();
	}
	end := S.pos - 1;

	var tok int;
	var present bool;
	tok, present = Keywords[S.src[beg : end]];
	if !present {
		tok = IDENT;
	}

	return tok;
}

func (S *Scanner) ScanNumber () {
	// TODO complete this routine

	for is_dec_digit(S.ch) {
		S.Next();
	}
}

func (S *Scanner) ScanOctDigits(n int) {
	for ; n > 0; n-- {
		if !is_oct_digit(S.ch) {
			panic "illegal char escape";
		}
		S.Next();
	}
}

func (S *Scanner) ScanHexDigits(n int) {
	for ; n > 0; n-- {
		if !is_hex_digit(S.ch) {
			panic "illegal char escape";
		}
		S.Next();
	}
}

func (S *Scanner) ScanEscape () {
	// TODO: fix this routine

	switch (S.ch) {
	case 'a': fallthrough;
	case 'b': fallthrough;
	case 'f': fallthrough;
	case 'n': fallthrough;
	case 'r': fallthrough;
	case 't': fallthrough;
	case 'v': fallthrough;
	case '\\': fallthrough;
	case '\'': fallthrough;
	case '"':
		S.Next();

	case '0', '1', '2', '3', '4', '5', '6', '7':
		S.ScanOctDigits(3);

	case 'x':
		S.Next();
		S.ScanHexDigits(2);

	case 'u':
		S.Next();
		S.ScanHexDigits(4);

	case 'U':
		S.Next();
		S.ScanHexDigits(8);

	default:
		panic "illegal char escape";
	}
}

func (S *Scanner) ScanChar () {
	S.Next();  // consume '\'

	if (S.ch == '\\') {
		S.Next();
		S.ScanEscape();
	} else {
		S.Next();
	}

	if S.ch == '\'' {
		S.Next();
	} else {
		panic "char not terminated";
	}
}

func (S *Scanner) ScanString () {
	for S.Next(); S.ch != '"'; S.Next() {
		if S.ch == '\n' || S.ch < 0 {
			panic "string not terminated";
		}
	}
	S.Next();
}

func (S *Scanner) ScanRawString () {
	for S.Next(); S.ch != '`'; S.Next() {
		if S.ch == '\n' || S.ch < 0 {
			panic "string not terminated";
		}
	}
	S.Next();
}

func (S *Scanner) Scan () (tok, beg, end int) {
	S.SkipWhitespace();

	var tok int = ILLEGAL;
	var beg int = S.pos - 1;
	var end int = beg;

	if is_letter(S.ch) {
		tok = S.ScanIdentifier();

	} else if is_dec_digit(S.ch) {
		S.ScanNumber();
		tok = NUMBER;

	} else {
		switch S.ch {
			case -1:
				tok = EOF;

			case '/':
				S.Next();
				if S.ch == '/' || S.ch == '*' {
					S.SkipComment();
					tok, beg, end = S.Scan();
					return tok, beg, end;
				} else {
					tok = QUO;
				}

			case '"':
				S.ScanString();
				tok = STRING;

			case '\'':
				S.ScanChar();
				tok = NUMBER;

			case '`':
				S.ScanRawString();
				tok = STRING;

			case ':':
				S.Next();
				if (S.ch == '=') {
					S.Next();
					tok = DEFINE;
				} else {
					tok = COLON;
				}

			case '.':
				S.Next();
				tok = PERIOD;

			case ',':
				S.Next();
				tok = COMMA;

			case '+':
				S.Next();
				if (S.ch == '+') {
					S.Next();
					tok = INC;
				} else {
					tok = ADD;
				}

			case '-':
				S.Next();
				if (S.ch == '-') {
					S.Next();
					tok = DEC;
				} else {
					tok = SUB;
				}

			case '*':
				S.Next();
				tok = MUL;

			case '/':
				S.Next();
				tok = QUO;

			case '%':
				S.Next();
				tok = REM;

			case '<':
				S.Next();
				if (S.ch == '=') {
					S.Next();
					tok = LEQ;
				} else {
					tok = LSS;
				}

			case '>':
				S.Next();
				if (S.ch == '=') {
					S.Next();
					tok = GEQ;
				} else {
					tok = GTR;
				}

			case '=':
				S.Next();
				if (S.ch == '=') {
					S.Next();
					tok = EQL;
				} else {
					tok = ASSIGN;
				}

			case '!':
				S.Next();
				if (S.ch == '=') {
					S.Next();
					tok = NEQ;
				} else {
					tok = NOT;
				}

			case ';':
				S.Next();
				tok = SEMICOLON;

			case '(':
				S.Next();
				tok = LPAREN;

			case ')':
				S.Next();
				tok = LPAREN;

			case '[':
				S.Next();
				tok = LBRACK;

			case ']':
				S.Next();
				tok = RBRACK;

			case '{':
				S.Next();
				tok = LBRACE;

			case '}':
				S.Next();
				tok = RBRACE;

			case '&':
				S.Next();
				if S.ch == '&' {
					S.Next();
					tok = AND;
				} else {
					tok = BAND;
				}

			case '|':
				S.Next();
				if S.ch == '|' {
					S.Next();
					tok = OR;
				} else {
					tok = BOR;
				}

			default:
				S.Next();  // make progress
		}
	}

	end = S.pos - 1;
	return tok, beg, end;
}
<commit_msg>- more Go scanner work<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package Scanner

export EOF;
const (
	ILLEGAL = iota;
	EOF = iota;
	IDENT = iota;
	STRING = iota;
	NUMBER = iota;

	COMMA = iota;
	COLON = iota;
	SEMICOLON = iota;
	PERIOD = iota;

	LPAREN = iota;
	RPAREN = iota;
	LBRACK = iota;
	RBRACK = iota;
	LBRACE = iota;
	RBRACE = iota;

	ASSIGN = iota;
	DEFINE = iota;

	INC = iota;
	DEC = iota;
	NOT = iota;

	AND = iota;
	OR = iota;
	XOR = iota;

	ADD = iota;
	SUB = iota;
	MUL = iota;
	QUO = iota;
	REM = iota;

	EQL = iota;
	NEQ = iota;
	LSS = iota;
	LEQ = iota;
	GTR = iota;
	GEQ = iota;

	SHL = iota;
	SHR = iota;

	ADD_ASSIGN = iota;
	SUB_ASSIGN = iota;
	MUL_ASSIGN = iota;
	QUO_ASSIGN = iota;
	REM_ASSIGN = iota;

	AND_ASSIGN = iota;
	OR_ASSIGN = iota;
	XOR_ASSIGN = iota;

	SHL_ASSIGN = iota;
	SHR_ASSIGN = iota;

	CAND = iota;
	COR = iota;

	// keywords
	BREAK = iota;
	CASE = iota;
	CONST = iota;
	CONTINUE = iota;
	DEFAULT = iota;
	ELSE = iota;
	EXPORT = iota;
	FALLTHROUGH = iota;
	FALSE = iota;
	FOR = iota;
	FUNC = iota;
	GO = iota;
	GOTO = iota;
	IF = iota;
	IMPORT = iota;
	INTERFACE = iota;
	MAP = iota;
	NEW = iota;
	NIL = iota;
	PACKAGE = iota;
	RANGE = iota;
	RETURN = iota;
	SELECT = iota;
	STRUCT = iota;
	SWITCH = iota;
	TRUE = iota;
	TYPE = iota;
	VAR = iota;
)

var (
	Keywords *map [string] int;
)

export TokenName
func TokenName(tok int) string {
	switch (tok) {
	case ILLEGAL: return "ILLEGAL";
	case EOF: return "EOF";
	case IDENT: return "IDENT";
	case STRING: return "STRING";
	case NUMBER: return "NUMBER";

	case COMMA: return "COMMA";
	case COLON: return "COLON";
	case SEMICOLON: return "SEMICOLON";
	case PERIOD: return "PERIOD";

	case LPAREN: return "LPAREN";
	case RPAREN: return "RPAREN";
	case LBRACK: return "LBRACK";
	case RBRACK: return "RBRACK";
	case LBRACE: return "LBRACE";
	case RBRACE: return "RBRACE";

	case ASSIGN: return "ASSIGN";
	case DEFINE: return "DEFINE";

	case INC: return "INC";
	case DEC: return "DEC";
	case NOT: return "NOT";

	case AND: return "AND";
	case OR: return "OR";
	case XOR: return "XOR";

	case ADD: return "ADD";
	case SUB: return "SUB";
	case MUL: return "MUL";
	case REM: return "REM";
	case QUO: return "QUO";
	case REM: return "REM";

	case EQL: return "EQL";
	case NEQ: return "NEQ";
	case LSS: return "LSS";
	case LEQ: return "LEQ";
	case GTR: return "GTR";
	case GEQ: return "GEQ";

	case SHL: return SHL;
	case SHR: return SHR;

	case ADD_ASSIGN: return "ADD_ASSIGN";
	case SUB_ASSIGN: return "SUB_ASSIGN";
	case MUL_ASSIGN: return "MUL_ASSIGN";
	case QUO_ASSIGN: return "QUO_ASSIGN";
	case REM_ASSIGN: return "REM_ASSIGN";

	case AND_ASSIGN: return "AND_ASSIGN";
	case OR_ASSIGN: return "OR_ASSIGN";
	case XOR_ASSIGN: return "XOR_ASSIGN";

	case SHL_ASSIGN: return "SHL_ASSIGN";
	case SHR_ASSIGN: return "SHR_ASSIGN";

	case CAND: return "CAND";
	case COR: return "COR";

	case BREAK: return "BREAK";
	case CASE: return "CASE";
	case CONST: return "CONST";
	case CONTINUE: return "CONTINUE";
	case DEFAULT: return "DEFAULT";
	case ELSE: return "ELSE";
	case EXPORT: return "EXPORT";
	case FALLTHROUGH: return "FALLTHROUGH";
	case FALSE: return "FALSE";
	case FOR: return "FOR";
	case FUNC: return "FUNC";
	case GO: return "GO";
	case GOTO: return "GOTO";
	case IF: return "IF";
	case IMPORT: return "IMPORT";
	case INTERFACE: return "INTERFACE";
	case MAP: return "MAP";
	case NEW: return "NEW";
	case NIL: return "NIL";
	case PACKAGE: return "PACKAGE";
	case RANGE: return "RANGE";
	case RETURN: return "RETURN";
	case SELECT: return "SELECT";
	case STRUCT: return "STRUCT";
	case SWITCH: return "SWITCH";
	case TRUE: return "TRUE";
	case TYPE: return "TYPE";
	case VAR: return "VAR";
	}

	return "???";
}

func is_whitespace (ch int) bool {
	return ch == ' ' || ch == '\r' || ch == '\n' || ch == '\t';
}

func is_letter (ch int) bool {
	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= 128 ;
}

func is_oct_digit (ch int) bool {
	return '0' <= ch && ch <= '7';
}

func is_dec_digit (ch int) bool {
	return '0' <= ch && ch <= '9';
}

func is_hex_digit (ch int) bool {
	return '0' <= ch && ch <= '9' || 'a' <= ch && ch <= 'f' || 'A' <= ch && ch <= 'F';
}

export Scanner
type Scanner struct {
	src string;
	pos int;
	ch int;  // one char look-ahead
}

func (S *Scanner) Next () {
	src := S.src;  // TODO only needed because of 6g bug
	if S.pos < len(src) {
		S.ch = int(S.src[S.pos]);
		S.pos++;
		if (S.ch >= 128) {
			panic "UTF-8 not handled"
		}
	} else {
		S.ch = -1;
	}
}

func Init () {
	Keywords = new(map [string] int);

	Keywords["break"] = BREAK;
	Keywords["case"] = CASE;
	Keywords["const"] = CONST;
	Keywords["continue"] = CONTINUE;
	Keywords["default"] = DEFAULT;
	Keywords["else"] = ELSE;
	Keywords["export"] = EXPORT;
	Keywords["fallthrough"] = FALLTHROUGH;
	Keywords["false"] = FALSE;
	Keywords["for"] = FOR;
	Keywords["func"] = FUNC;
	Keywords["go"] = GO;
	Keywords["goto"] = GOTO;
	Keywords["if"] = IF;
	Keywords["import"] = IMPORT;
	Keywords["interface"] = INTERFACE;
	Keywords["map"] = MAP;
	Keywords["new"] = NEW;
	Keywords["nil"] = NIL;
	Keywords["package"] = PACKAGE;
	Keywords["range"] = RANGE;
	Keywords["return"] = RETURN;
	Keywords["select"] = SELECT;
	Keywords["struct"] = STRUCT;
	Keywords["switch"] = SWITCH;
	Keywords["true"] = TRUE;
	Keywords["type"] = TYPE;
	Keywords["var"] = VAR;
}

func (S *Scanner) Open (src string) {
	if Keywords == nil {
		Init();
	}

	S.src = src;
	S.pos = 0;
	S.Next();
}

func (S *Scanner) SkipWhitespace () {
	for is_whitespace(S.ch) {
		S.Next();
	}
}

func (S *Scanner) SkipComment () {
	if S.ch == '/' {
		// comment
		for S.Next(); S.ch != '\n' && S.ch >= 0; S.Next() {}

	} else {
		/* comment */
		for S.Next(); S.ch >= 0; {
			c := S.ch;
			S.Next();
			if c == '*' && S.ch == '/' {
				S.Next();
				return;
			}
		}
		panic "comment not terminated";
	}
}

func (S *Scanner) ScanIdentifier () int {
	beg := S.pos - 1;
	for is_letter(S.ch) || is_dec_digit(S.ch) {
		S.Next();
	}
	end := S.pos - 1;

	var tok int;
	var present bool;
	tok, present = Keywords[S.src[beg : end]];
	if !present {
		tok = IDENT;
	}

	return tok;
}

func (S *Scanner) ScanMantissa () {
     	for is_dec_digit(S.ch) {
	      	S.Next();
	}
}

func (S *Scanner) ScanNumber () int {
	// TODO complete this routine
	if S.ch == '.' {
	   	S.Next();
	}
	S.ScanMantissa();
	if S.ch == 'e' || S.ch == 'E' {
	   	S.Next();
		if S.ch == '-' || S.ch == '+' {
		   	S.Next();
		}
		S.ScanMantissa();
	}
	return NUMBER;
}

func (S *Scanner) ScanOctDigits(n int) {
	for ; n > 0; n-- {
		if !is_oct_digit(S.ch) {
			panic "illegal char escape";
		}
		S.Next();
	}
}

func (S *Scanner) ScanHexDigits(n int) {
	for ; n > 0; n-- {
		if !is_hex_digit(S.ch) {
			panic "illegal char escape";
		}
		S.Next();
	}
}

func (S *Scanner) ScanEscape () {
	// TODO: fix this routine

	switch (S.ch) {
	case 'a': fallthrough;
	case 'b': fallthrough;
	case 'f': fallthrough;
	case 'n': fallthrough;
	case 'r': fallthrough;
	case 't': fallthrough;
	case 'v': fallthrough;
	case '\\': fallthrough;
	case '\'': fallthrough;
	case '"':
		S.Next();

	case '0', '1', '2', '3', '4', '5', '6', '7':
		S.ScanOctDigits(3);

	case 'x':
		S.Next();
		S.ScanHexDigits(2);

	case 'u':
		S.Next();
		S.ScanHexDigits(4);

	case 'U':
		S.Next();
		S.ScanHexDigits(8);

	default:
		panic "illegal char escape";
	}
}

func (S *Scanner) ScanChar () int {
	if (S.ch == '\\') {
		S.Next();
		S.ScanEscape();
	} else {
		S.Next();
	}

	if S.ch == '\'' {
		S.Next();
	} else {
		panic "char not terminated";
	}
	return NUMBER;
}

func (S *Scanner) ScanString () int {
	for ; S.ch != '"'; S.Next() {
		if S.ch == '\n' || S.ch < 0 {
			panic "string not terminated";
		}
	}
	S.Next();
	return STRING;
}

func (S *Scanner) ScanRawString () int {
	for ; S.ch != '`'; S.Next() {
		if S.ch == '\n' || S.ch < 0 {
			panic "string not terminated";
		}
	}
	S.Next();
	return STRING;
}

func (S *Scanner) Select2 (tok0, tok1 int) int {
     	if S.ch == '=' {
	   	S.Next();
	   	return tok1;
	}
	return tok0;
}

func (S *Scanner) Select3 (tok0, tok1, ch2, tok2 int) int {
	if S.ch == '=' {
	   	S.Next();
		return tok1;
	}
	if S.ch == ch2 {
	   	S.Next();
		return tok2;
	}
	return tok0;
}

func (S *Scanner) Select4 (tok0, tok1, ch2, tok2, tok3 int) int {
	if S.ch == '=' {
	   	S.Next();
		return tok1;
	}
	if S.ch == ch2 {
	   	S.Next();
		if S.ch == '=' {
		   	S.Next();
			return tok3;
		}
		return tok2;
	}
	return tok0;
}

func (S *Scanner) Scan () (tok, beg, end int) {
	S.SkipWhitespace();

	var tok int = ILLEGAL;
	var beg int = S.pos - 1;
	var end int = beg;

	switch ch := S.ch; {
	case is_letter(ch): tok = S.ScanIdentifier();
	case is_dec_digit(ch): tok = S.ScanNumber();
	default:
		S.Next();
		switch ch {
			case -1: tok = EOF;
			case '"': tok = S.ScanString();
			case '\'': tok = S.ScanChar();
			case '`': tok = S.ScanRawString();
			case ':': tok = S.Select2(COLON, DEFINE);
			case '.':
				if is_dec_digit(S.ch) {
				     	tok = S.ScanNumber();
				} else {
				        tok = PERIOD;
				}
			case ',': tok = COMMA;
			case ';': tok = SEMICOLON;
			case '(': tok = LPAREN;
			case ')': tok = RPAREN;
			case '[': tok = LBRACK;
			case ']': tok = RBRACK;
			case '{': tok = LBRACE;
			case '}': tok = RBRACE;
			case '+': tok = S.Select3(ADD, ADD_ASSIGN, '+', INC);
			case '-': tok = S.Select3(SUB, SUB_ASSIGN, '-', DEC);
			case '*': tok = S.Select2(MUL, MUL_ASSIGN);
			case '/':
				if S.ch == '/' || S.ch == '*' {
					S.SkipComment();
					// cannot simply return because of 6g bug
					tok, beg, end = S.Scan();
					return tok, beg, end;
				}
				tok = S.Select2(QUO, QUO_ASSIGN);
			case '%': tok = S.Select2(REM, REM_ASSIGN);
			case '^': tok = S.Select2(XOR, XOR_ASSIGN);
			case '<': tok = S.Select4(LSS, LEQ, '<', SHL, SHL_ASSIGN);
			case '>': tok = S.Select4(GTR, GEQ, '>', SHR, SHR_ASSIGN);
			case '=': tok = S.Select2(ASSIGN, EQL);
			case '!': tok = S.Select2(NOT, NEQ);
			case '&': tok = S.Select3(AND, AND_ASSIGN, '&', CAND);
			case '|': tok = S.Select3(OR, OR_ASSIGN, '|', COR);
			default: tok = ILLEGAL;

		}
	}

	end = S.pos - 1;
	return tok, beg, end;
}
<|endoftext|>


<commit_before><commit_msg>fix: Continue writing control sequences on interrupted string<commit_after><|endoftext|>


<commit_before><commit_msg>Run goimports<commit_after><|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"sync"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"

	"github.com/jacobsa/gcloud/gcs"
)

var g_bucketOnce sync.Once
var g_bucket gcs.Bucket

func makeHTTPClient() (c *http.Client, err error) {
	cfg := getConfig()

	// Attempt to read the JSON file.
	contents, err := ioutil.ReadFile(cfg.KeyFile)
	if err != nil {
		err = fmt.Errorf("ReadFile(%q): %v", cfg.KeyFile, err)
		return
	}

	// Create a config struct based on its contents.
	jwtConfig, err := google.JWTConfigFromJSON(contents, gcs.Scope_FullControl)
	if err != nil {
		err = fmt.Errorf("JWTConfigFromJSON: %v", err)
		return
	}

	// Create the HTTP client.
	c = jwtConfig.Client(context.Background())

	return
}

func makeBucket() (bucket gcs.Bucket, err error) {
	cfg := getConfig()

	// Create an authenticated HTTP client.
	httpClient, err := makeHTTPClient()
	if err != nil {
		err = fmt.Errorf("makeHTTPClient: %v", err)
		return
	}

	// Turn that into a connection.
	connCfg := &gcs.ConnConfig{
		HTTPClient:      httpClient,
		MaxBackoffSleep: time.Minute,
	}

	conn, err := gcs.NewConn(connCfg)
	if err != nil {
		err = fmt.Errorf("NewConn: %v", err)
		return
	}

	// Grab the bucket.
	bucket = conn.GetBucket(cfg.BucketName)

	return
}

func initBucket() {
	var err error

	g_bucket, err = makeBucket()
	if err != nil {
		panic(err)
	}
}

func getBucket() gcs.Bucket {
	g_bucketOnce.Do(initBucket)
	return g_bucket
}
<commit_msg>Fixed the build.<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"io/ioutil"
	"sync"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"

	"github.com/jacobsa/gcloud/gcs"
)

var g_bucketOnce sync.Once
var g_bucket gcs.Bucket

func makeTokenSource() (ts oauth2.TokenSource, err error) {
	cfg := getConfig()

	// Attempt to read the JSON file.
	contents, err := ioutil.ReadFile(cfg.KeyFile)
	if err != nil {
		err = fmt.Errorf("ReadFile(%q): %v", cfg.KeyFile, err)
		return
	}

	// Create a config struct based on its contents.
	jwtConfig, err := google.JWTConfigFromJSON(contents, gcs.Scope_FullControl)
	if err != nil {
		err = fmt.Errorf("JWTConfigFromJSON: %v", err)
		return
	}

	// Create the token source.
	ts = jwtConfig.TokenSource(context.Background())

	return
}

func makeBucket() (bucket gcs.Bucket, err error) {
	cfg := getConfig()

	// Create an oauth2 token source.
	tokenSrc, err := makeTokenSource()
	if err != nil {
		err = fmt.Errorf("makeTokenSource: %v", err)
		return
	}

	// Turn that into a connection.
	connCfg := &gcs.ConnConfig{
		TokenSource:     tokenSrc,
		MaxBackoffSleep: time.Minute,
	}

	conn, err := gcs.NewConn(connCfg)
	if err != nil {
		err = fmt.Errorf("NewConn: %v", err)
		return
	}

	// Grab the bucket.
	bucket, err = conn.OpenBucket(
		context.Background(),
		cfg.BucketName)

	if err != nil {
		err = fmt.Errorf("OpenBucket: %v", err)
		return
	}

	return
}

func initBucket() {
	var err error

	g_bucket, err = makeBucket()
	if err != nil {
		panic(err)
	}
}

func getBucket() gcs.Bucket {
	g_bucketOnce.Do(initBucket)
	return g_bucket
}
<|endoftext|>


<commit_before>/* Copyright (c) 2013 Matthias S. Benkmann
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this file (originally named buffer.go) and associated documentation files
 * (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is furnished
 * to do so, subject to the following conditions:
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

// Alternative to the Go standard lib's bytes package that avoids the GC problems.
package bytes

/*
#include <stdlib.h>
*/
import "C"
import "unsafe"
import "runtime"
import "errors"

const mAX_GROW_SIZE = 1024*1024
const maxInt = int(^uint(0) >> 1)

// ErrTooLarge is passed to panic if memory cannot be allocated to store data in a buffer.
var ErrTooLarge = errors.New("bytes.Buffer: too large")

type Buffer struct {
  ptr unsafe.Pointer
  sz int
  capa int
}

// Returns nil if no space is currently allocated for the Buffer,
// otherwise a raw memory pointer to the buffer space.
func (b *Buffer) Pointer() unsafe.Pointer {
  return b.ptr
}

// Returns the total buffer space (used + unused).
func (b *Buffer) Capacity() int { return b.capa }

// Returns the number of meaningful bytes in the buffer (as opposed to Capacity()).
func (b *Buffer) Len() int { return b.sz }

// Grow() grows the buffer to guarantee space for n more bytes.
// It returns Len().
// If the buffer can't grow it will panic with ErrTooLarge.
// Grow() does not change Len(), only Capacity().
func (b *Buffer) Grow(n int) int {
  if n > 0 {
    rest := b.capa - b.sz
    if rest < n {

      rest = n - rest  // minimum number of bytes we need to grow by

      // if we're allocating memory for the first time, set a finalizer
      if b.capa == 0 { runtime.SetFinalizer(b, (*Buffer).Reset) }
      growth := b.capa
      if growth > mAX_GROW_SIZE { growth = mAX_GROW_SIZE }
      if rest > growth { growth = rest }
      b.capa += growth
      p := C.realloc(b.ptr, C.size_t(b.capa))
      if p == nil { panic(ErrTooLarge) }
      b.ptr = p
    }
  } else if n < 0 { panic(ErrTooLarge) } // not really too large, but this case is just a precaution.

  return b.sz
}

// Appends n 0-bytes to the buffer.
func (b* Buffer) Write0(n int) {
  if n <= 0 { return }
  b.Grow(n);
  data := ((*[maxInt]byte)(b.ptr))[b.sz:b.capa]
  for i := 0; i < n; i++ { data[i] = 0 }
  b.sz += n
}

// WriteByte appends the byte c to the buffer.
// The returned error is always nil, but is included to match bufio.Writer's
// WriteByte. If the buffer becomes too large, WriteByte will panic with ErrTooLarge.
func (b *Buffer) WriteByte(c byte) error {
  b.Grow(1)
  ((*[maxInt]byte)(b.ptr))[b.sz] = c
  b.sz++
  return nil
}

// Appends the bytes from p to the buffer. Always returns len(p), nil.
// If out of memory occurs trying to grow the buffer, the function will
// panic with ErrTooLarge.
func (b *Buffer) Write(p []byte) (n int, err error) {
  if len(p) == 0 { return 0, nil }
  b.Grow(len(p))
  b.sz += copy(((*[maxInt]byte)(b.ptr))[b.sz:b.capa], p)
  return len(p),nil
}

// Appends the bytes from s to the buffer. Always returns len(s), nil.
// If out of memory occurs trying to grow the buffer, the function will
// panic with ErrTooLarge.
func (b *Buffer) WriteString(s string) (n int, err error) {
  if len(s) == 0 { return 0, nil }
  b.Grow(len(s))
  b.sz += copy(((*[maxInt]byte)(b.ptr))[b.sz:b.capa], s)
  return len(s),nil
}

// Returns a copy of the Buffer contents as string.
func (b *Buffer) String() string {
  return C.GoStringN((*C.char)(b.ptr), C.int(b.sz))
}

// Returns a byte slice that directly accesses the buffer's data.
// WARNING! Appending anything to the buffer invalidates any
// slices obtained via this function. They may end up pointing at
// invalid memory locations.
//
// NOTE: The return value is always a valid slice, even if the
// buffer is empty. The function never returns nil.
func (b *Buffer) Bytes() []byte {
  if b.ptr == nil { return []byte{} }
  return ((*[maxInt]byte)(b.ptr))[0:b.sz]
}

// Return true if the buffer contains the string s. Returns true if s == "".
func (b *Buffer) Contains(s string) bool {
  if s == "" { return true }
  if b.sz == 0 { return false }
  data := ((*[maxInt]byte)(b.ptr))[0:b.sz]
  for i := 0; i <= b.sz - len(s); i++ {
    if data[i] == s[0] {
      k := 0
      for ; k < len(s); k++ {
        if s[k] != data[i+k] { break }
      }
      if k == len(s) { return true }
    }
  }
  return false
}

// removes all characters <= ' ' from both ends of the buffer.
func (b *Buffer) TrimSpace() {
  if b.ptr == nil { return }
  data := ((*[maxInt]byte)(b.ptr))[0:b.sz]
  i := 0
  for ; i < len(data); i++ { if data[i] > ' ' { break } }
  switch i {
    case 0:
      // nothing to do
    case len(data):
      b.Reset()
      return
    default:
      b.sz = copy(data,data[i:])
  }

  // no need to test for b.sz == 0 because we know there's at least
  // 1 non-whitespace character in the buffer or we would have run into
  // the case len(data) in the switch above.
  for data[b.sz-1] <= ' ' { b.sz-- }
}

// Replaces the contents of the buffer b with b.Bytes()[start:end].
// Unlike a sublicing operation this function permits start and end to
// be out of bounds. If start >= end, the buffer will be Reset().
//
// NOTE: The operation does NOT free any memory. The buffer's Capacity() will
// remain unchanged.
func (b *Buffer) Trim(start, end int) {
  if start < 0 { start = 0 }
  if end > b.sz { end = b.sz }
  if start >= end {
    b.Reset()
    return
  }

  b.sz = end

  if start > 0 {
    data := ((*[maxInt]byte)(b.ptr))[0:b.sz]
    b.sz = copy(data, data[start:])
  }
}

// Split slices the buffer into all substrings separated by sep and returns
// a slice of the substrings between those separators. The buffer itself is
// unchanged and the strings are copies (of course).
// If sep is empty, this function panics.
func (b *Buffer) Split(sep string) []string {
  if sep == "" { panic("UTF-8 splitting like strings.Split() has is not implemented") }
  if b.sz == 0 { return []string{} }

  result := make([]string,0,2)
  buf := ((*[maxInt]byte)(b.ptr))[0:b.sz]
  last_idx := 0
  idx := 0
  ch := sep[0]
  last_possible_idx := len(buf)-len(sep)
  for idx <= last_possible_idx {
    if buf[idx] == ch {
      for k := 1; k < len(sep); k++ {
        if buf[idx+k] != sep[k] { goto notfound }
      }
      result = append(result, string(buf[last_idx:idx]))
      last_idx = idx + len(sep)
      idx = last_idx - 1
    }
  notfound:
    idx++
  }
  result = append(result, string(buf[last_idx:]))
  return result
}

// Frees the memory held by the Buffer. The Buffer remains valid and is ready
// to take new data.
func (b *Buffer) Reset() {
  C.free(b.ptr)
  b.ptr = nil
  b.sz = 0
  b.capa = 0
  runtime.SetFinalizer(b, nil)
}
<commit_msg>fix "structure larger than address space" error on 64bit<commit_after>/* Copyright (c) 2013 Matthias S. Benkmann
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this file (originally named buffer.go) and associated documentation files
 * (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is furnished
 * to do so, subject to the following conditions:
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

// Alternative to the Go standard lib's bytes package that avoids the GC problems.
package bytes

/*
#include <stdlib.h>
*/
import "C"
import "unsafe"
import "runtime"
import "errors"

const mAX_GROW_SIZE = 1024*1024
const size2GB = int(^uint32(0) >> 1)

// ErrTooLarge is passed to panic if memory cannot be allocated to store data in a buffer.
var ErrTooLarge = errors.New("bytes.Buffer: too large")

type Buffer struct {
  ptr unsafe.Pointer
  sz int
  capa int
}

// Returns nil if no space is currently allocated for the Buffer,
// otherwise a raw memory pointer to the buffer space.
func (b *Buffer) Pointer() unsafe.Pointer {
  return b.ptr
}

// Returns the total buffer space (used + unused).
func (b *Buffer) Capacity() int { return b.capa }

// Returns the number of meaningful bytes in the buffer (as opposed to Capacity()).
func (b *Buffer) Len() int { return b.sz }

// Grow() grows the buffer to guarantee space for n more bytes.
// It returns Len().
// If the buffer can't grow it will panic with ErrTooLarge.
// Grow() does not change Len(), only Capacity().
func (b *Buffer) Grow(n int) int {
  if n > 0 {
    rest := b.capa - b.sz
    if rest < n {

      rest = n - rest  // minimum number of bytes we need to grow by

      // if we're allocating memory for the first time, set a finalizer
      if b.capa == 0 { runtime.SetFinalizer(b, (*Buffer).Reset) }
      growth := b.capa
      if growth > mAX_GROW_SIZE { growth = mAX_GROW_SIZE }
      if rest > growth { growth = rest }
      b.capa += growth
      p := C.realloc(b.ptr, C.size_t(b.capa))
      if p == nil { panic(ErrTooLarge) }
      b.ptr = p
    }
  } else if n < 0 { panic(ErrTooLarge) } // not really too large, but this case is just a precaution.

  return b.sz
}

// Appends n 0-bytes to the buffer.
func (b* Buffer) Write0(n int) {
  if n <= 0 { return }
  b.Grow(n);
  data := ((*[size2GB]byte)(b.ptr))[b.sz:b.capa]
  for i := 0; i < n; i++ { data[i] = 0 }
  b.sz += n
}

// WriteByte appends the byte c to the buffer.
// The returned error is always nil, but is included to match bufio.Writer's
// WriteByte. If the buffer becomes too large, WriteByte will panic with ErrTooLarge.
func (b *Buffer) WriteByte(c byte) error {
  b.Grow(1)
  ((*[size2GB]byte)(b.ptr))[b.sz] = c
  b.sz++
  return nil
}

// Appends the bytes from p to the buffer. Always returns len(p), nil.
// If out of memory occurs trying to grow the buffer, the function will
// panic with ErrTooLarge.
func (b *Buffer) Write(p []byte) (n int, err error) {
  if len(p) == 0 { return 0, nil }
  b.Grow(len(p))
  b.sz += copy(((*[size2GB]byte)(b.ptr))[b.sz:b.capa], p)
  return len(p),nil
}

// Appends the bytes from s to the buffer. Always returns len(s), nil.
// If out of memory occurs trying to grow the buffer, the function will
// panic with ErrTooLarge.
func (b *Buffer) WriteString(s string) (n int, err error) {
  if len(s) == 0 { return 0, nil }
  b.Grow(len(s))
  b.sz += copy(((*[size2GB]byte)(b.ptr))[b.sz:b.capa], s)
  return len(s),nil
}

// Returns a copy of the Buffer contents as string.
func (b *Buffer) String() string {
  return C.GoStringN((*C.char)(b.ptr), C.int(b.sz))
}

// Returns a byte slice that directly accesses the buffer's data.
// WARNING! Appending anything to the buffer invalidates any
// slices obtained via this function. They may end up pointing at
// invalid memory locations.
//
// NOTE: The return value is always a valid slice, even if the
// buffer is empty. The function never returns nil.
func (b *Buffer) Bytes() []byte {
  if b.ptr == nil { return []byte{} }
  return ((*[size2GB]byte)(b.ptr))[0:b.sz]
}

// Return true if the buffer contains the string s. Returns true if s == "".
func (b *Buffer) Contains(s string) bool {
  if s == "" { return true }
  if b.sz == 0 { return false }
  data := ((*[size2GB]byte)(b.ptr))[0:b.sz]
  for i := 0; i <= b.sz - len(s); i++ {
    if data[i] == s[0] {
      k := 0
      for ; k < len(s); k++ {
        if s[k] != data[i+k] { break }
      }
      if k == len(s) { return true }
    }
  }
  return false
}

// removes all characters <= ' ' from both ends of the buffer.
func (b *Buffer) TrimSpace() {
  if b.ptr == nil { return }
  data := ((*[size2GB]byte)(b.ptr))[0:b.sz]
  i := 0
  for ; i < len(data); i++ { if data[i] > ' ' { break } }
  switch i {
    case 0:
      // nothing to do
    case len(data):
      b.Reset()
      return
    default:
      b.sz = copy(data,data[i:])
  }

  // no need to test for b.sz == 0 because we know there's at least
  // 1 non-whitespace character in the buffer or we would have run into
  // the case len(data) in the switch above.
  for data[b.sz-1] <= ' ' { b.sz-- }
}

// Replaces the contents of the buffer b with b.Bytes()[start:end].
// Unlike a sublicing operation this function permits start and end to
// be out of bounds. If start >= end, the buffer will be Reset().
//
// NOTE: The operation does NOT free any memory. The buffer's Capacity() will
// remain unchanged.
func (b *Buffer) Trim(start, end int) {
  if start < 0 { start = 0 }
  if end > b.sz { end = b.sz }
  if start >= end {
    b.Reset()
    return
  }

  b.sz = end

  if start > 0 {
    data := ((*[size2GB]byte)(b.ptr))[0:b.sz]
    b.sz = copy(data, data[start:])
  }
}

// Split slices the buffer into all substrings separated by sep and returns
// a slice of the substrings between those separators. The buffer itself is
// unchanged and the strings are copies (of course).
// If sep is empty, this function panics.
func (b *Buffer) Split(sep string) []string {
  if sep == "" { panic("UTF-8 splitting like strings.Split() has is not implemented") }
  if b.sz == 0 { return []string{} }

  result := make([]string,0,2)
  buf := ((*[size2GB]byte)(b.ptr))[0:b.sz]
  last_idx := 0
  idx := 0
  ch := sep[0]
  last_possible_idx := len(buf)-len(sep)
  for idx <= last_possible_idx {
    if buf[idx] == ch {
      for k := 1; k < len(sep); k++ {
        if buf[idx+k] != sep[k] { goto notfound }
      }
      result = append(result, string(buf[last_idx:idx]))
      last_idx = idx + len(sep)
      idx = last_idx - 1
    }
  notfound:
    idx++
  }
  result = append(result, string(buf[last_idx:]))
  return result
}

// Frees the memory held by the Buffer. The Buffer remains valid and is ready
// to take new data.
func (b *Buffer) Reset() {
  C.free(b.ptr)
  b.ptr = nil
  b.sz = 0
  b.capa = 0
  runtime.SetFinalizer(b, nil)
}
<|endoftext|>


<commit_before>package memio

import "io"

// Buffer grants a byte slice very straightforward IO methods.
type Buffer []byte

// Read satisfies the io.Reader interface
func (s *Buffer) Read(p []byte) (int, error) {
	if len(p) == 0 {
		return 0, nil
	}
	if len(*s) == 0 {
		return 0, io.EOF
	}
	n := copy(p, *s)
	*s = (*s)[n:]
	return n, nil
}

// WriteTo satisfies the io.WriterTo interface
func (s *Buffer) WriteTo(w io.Writer) (int64, error) {
	n, err := w.Write(*s)
	*s = (*s)[n:]
	return int64(n), err
}

// Write satisfies the io.Writer interface
func (s *Buffer) Write(p []byte) (int, error) {
	*s = append(*s, p...)
	return len(p), nil
}

// WriteString writes a string to the buffer without casting to a byte slice
func (s *Buffer) WriteString(str string) (int, error) {
	*s = append(*s, str...)
	return len(str), nil
}

// ReadByte satisfies the io.ByteReader interface
func (s *Buffer) ReadByte() (byte, error) {
	if len(*s) == 0 {
		return 0, io.EOF
	}
	b := (*s)[0]
	*s = (*s)[1:]
	return b, nil
}

// WriteByte satisfies the io.ByteWriter interface
func (s *Buffer) WriteByte(b byte) error {
	*s = append(*s, b)
	return nil
}

// Peek reads the next n bytes without advancing the position
func (s *Buffer) Peek(n int) ([]byte, error) {
	if *s == nil {
		return nil, ErrClosed
	} else if n > len(*s) {
		return *s, io.EOF
	}
	return (*s)[:n], nil
}

// Close satisfies the io.Closer interface
func (s *Buffer) Close() error {
	*s = nil
	return nil
}
<commit_msg>added ReadRune to buffer<commit_after>package memio

import (
	"io"
	"unicode/utf8"
)

// Buffer grants a byte slice very straightforward IO methods.
type Buffer []byte

// Read satisfies the io.Reader interface
func (s *Buffer) Read(p []byte) (int, error) {
	if len(p) == 0 {
		return 0, nil
	}
	if len(*s) == 0 {
		return 0, io.EOF
	}
	n := copy(p, *s)
	*s = (*s)[n:]
	return n, nil
}

// WriteTo satisfies the io.WriterTo interface
func (s *Buffer) WriteTo(w io.Writer) (int64, error) {
	n, err := w.Write(*s)
	*s = (*s)[n:]
	return int64(n), err
}

// Write satisfies the io.Writer interface
func (s *Buffer) Write(p []byte) (int, error) {
	*s = append(*s, p...)
	return len(p), nil
}

// WriteString writes a string to the buffer without casting to a byte slice
func (s *Buffer) WriteString(str string) (int, error) {
	*s = append(*s, str...)
	return len(str), nil
}

// ReadByte satisfies the io.ByteReader interface
func (s *Buffer) ReadByte() (byte, error) {
	if len(*s) == 0 {
		return 0, io.EOF
	}
	b := (*s)[0]
	*s = (*s)[1:]
	return b, nil
}

// ReadRune satisfies the io.RuneReader interface
func (s *Buffer) ReadRune() (rune, int, error) {
	if len(*s) == 0 {
		return 0, 0, io.EOF
	}
	r, n := utf8.DecodeRune(*s)
	*s = (*s)[n:]
	return r, n, nil
}

// WriteByte satisfies the io.ByteWriter interface
func (s *Buffer) WriteByte(b byte) error {
	*s = append(*s, b)
	return nil
}

// Peek reads the next n bytes without advancing the position
func (s *Buffer) Peek(n int) ([]byte, error) {
	if *s == nil {
		return nil, ErrClosed
	} else if n > len(*s) {
		return *s, io.EOF
	}
	return (*s)[:n], nil
}

// Close satisfies the io.Closer interface
func (s *Buffer) Close() error {
	*s = nil
	return nil
}
<|endoftext|>


<commit_before>package job

import (
	"github.com/PeopleAdmin/pdfcombiner/stat"
)

// A Document is a reference to one part of a combined PDF.
// it is is identified by its Key field, which is required.  It can also
// have a Title, used for TOC bookmarks in the final combined document.
// The actual PDF data comes from one of two places:
//  - If the Data field is empty, the Key is treated as a S3 key that is
//    fetched from the enclosing job's bucket.
//  - If the Data field is nonempty, it must contain a zlib-compressed
//    and Base64-encoded string containing the PDF.
type Document struct {
	Key   string `json:"key"`
	Title string `json:"title"`
	Data  string `json:"data,omitempty"`
}

// The Key is the only required field
func (doc *Document) isValid() bool {
	return doc.Key != ""
}

// Given a slice of document names, return a slice of Documents.
func docsFromStrings(names []string) (docs []Document) {
	docs = make([]Document, len(names))
	for i, name := range names {
		docs[i] = Document{Key: name}
	}
	return
}

// MarkComplete adds a document to the list of downloaded docs.
// TODO should be a Document.
func (j *Job) MarkComplete(newdoc string, info stat.Stat) {
	j.Downloaded = append(j.Downloaded, newdoc)
	j.PerfStats[newdoc] = info
}

// HasDownloadedDocs determines whether any documents been successfully
// downloaded.
// TODO is it appropriate to use this to determine success in ToJSON()?
func (j *Job) HasDownloadedDocs() bool {
	return len(j.Downloaded) > 0
}
<commit_msg>Add representation of pdf bookmarks<commit_after>package job

import (
	"github.com/PeopleAdmin/pdfcombiner/stat"
)

// A Document is a reference to one part of a combined PDF.
// it is is identified by its Key field, which is required.  It can also
// have a Title, used for TOC bookmarks in the final combined document.
// The actual PDF data comes from one of two places:
//  - If the Data field is empty, the Key is treated as a S3 key that is
//    fetched from the enclosing job's bucket.
//  - If the Data field is nonempty, it must contain a zlib-compressed
//    and Base64-encoded string containing the PDF.
// The bookmarks field is filled in by processing and is not serialized.
type Document struct {
	Key       string `json:"key"`
	Title     string `json:"title"`
	Data      string `json:"data,omitempty"`
	bookmarks []Bookmark
}

type Bookmark struct {
	Name string
	Page int
}

func (doc *Document) isValid() bool {
	return doc.Key != ""
}

// Given a slice of document names, return a slice of Documents.
func docsFromStrings(names []string) (docs []Document) {
	docs = make([]Document, len(names))
	for i, name := range names {
		docs[i] = Document{Key: name}
	}
	return
}

// MarkComplete adds a document to the list of downloaded docs.
// TODO should be a Document.
func (j *Job) MarkComplete(newdoc string, info stat.Stat) {
	j.Downloaded = append(j.Downloaded, newdoc)
	j.PerfStats[newdoc] = info
}

// HasDownloadedDocs determines whether any documents been successfully
// downloaded.
// TODO is it appropriate to use this to determine success in ToJSON()?
func (j *Job) HasDownloadedDocs() bool {
	return len(j.Downloaded) > 0
}
<|endoftext|>


<commit_before>package meow-ml-kmeans

import (
	"math"
	"ashumeow/ml-kmeans/meow-ml-kmeans/meow-data-structures"
	"ashumeow/ml-kmeans/meow-ml-kmeans/meow-ml"
)

type meowKmeans struct {
	dots *meow-data-structures.meowArrayList
	num_of_clusters int
	meowta float64
}<commit_msg>add new Kmeans<commit_after>package meow-ml-kmeans

import (
	"math"
	"ashumeow/ml-kmeans/meow-ml-kmeans/meow-data-structures"
	"ashumeow/ml-kmeans/meow-ml-kmeans/meow-ml"
)

type meowKmeans struct {
	dots *meow-data-structures.meowArrayList
	num_of_clusters int
	meowta float64
}

func meowNewKmeans(num_of_clusters int) *meowKmeans {
	my := &meowKmeans{}
	my.dots = meow-data-structures.meowNewArrayList()
	my.num_of_clusters = num_of_clusters
	my.meowta = 0.001
	return my
}<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package resourcekey contains standard resource attribute keys as defined
// by the OpenTelemetry specification
// (https://github.com/open-telemetry/opentelemetry-specification/tree/v0.4.0/specification/resource/semantic_conventions).
package resourcekey // import "go.opentelemetry.io/otel/sdk/resource/resourcekey"

// Constants for Service resources.
const (
	// A uniquely identifying name for a Service.
	ServiceName       = "service.name"
	ServiceNamespace  = "service.namespace"
	ServiceInstanceID = "service.instance.id"
	ServiceVersion    = "service.version"
)

// Constants for Library resources.
const (
	// A uniquely identifying name for a Library.
	LibraryName     = "library.name"
	LibraryLanguage = "library.language"
	LibraryVersion  = "library.version"
)

// Constants for Kubernetes resources.
const (
	// A uniquely identifying name for the Kubernetes cluster. Kubernetes
	// does not have cluster names as an internal concept so this may be
	// set to any meaningful value within the environment. For example,
	// GKE clusters have a name which can be used for this label.
	K8SClusterName    = "k8s.cluster.name"
	K8SNamespaceName  = "k8s.namespace.name"
	K8SPodName        = "k8s.pod.name"
	K8SDeploymentName = "k8s.deployment.name"
)

// Constants for Container resources.
const (
	// A uniquely identifying name for the Container.
	ContainerName      = "container.name"
	ContainerImageName = "container.image.name"
	ContainerImageTag  = "container.image.tag"
)

// Constants for Cloud resources.
const (
	CloudProvider  = "cloud.provider"
	CloudAccountID = "cloud.account.id"
	CloudRegion    = "cloud.region"
	CloudZone      = "cloud.zone"

	// Cloud Providers
	CloudProviderAWS   = "aws"
	CloudProviderGCP   = "gcp"
	CloudProviderAZURE = "azure"
)

// Constants for Host resources.
const (
	// A uniquely identifying name for the host.
	HostName = "host.name"

	// A hostname as returned by the 'hostname' command on host machine.
	HostHostName     = "host.hostname"
	HostID           = "host.id"
	HostType         = "host.type"
	HostImageName    = "host.image.name"
	HostImageID      = "host.image.id"
	HostImageVersion = "host.image.version"
)
<commit_msg>Switch to kv.Key types<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package resourcekey contains standard resource attribute keys as defined
// by the OpenTelemetry specification
// (https://github.com/open-telemetry/opentelemetry-specification/tree/v0.4.0/specification/resource/semantic_conventions).
package resourcekey // import "go.opentelemetry.io/otel/sdk/resource/resourcekey"

import "go.opentelemetry.io/otel/api/kv"

// Constants for Service resources.
const (
	// A uniquely identifying name for a Service.
	ServiceName       = kv.Key("service.name")
	ServiceNamespace  = kv.Key("service.namespace")
	ServiceInstanceID = kv.Key("service.instance.id")
	ServiceVersion    = kv.Key("service.version")
)

// Constants for Library resources.
const (
	// A uniquely identifying name for a Library.
	LibraryName     = kv.Key("library.name")
	LibraryLanguage = kv.Key("library.language")
	LibraryVersion  = kv.Key("library.version")
)

// Constants for Kubernetes resources.
const (
	// A uniquely identifying name for the Kubernetes cluster. Kubernetes
	// does not have cluster names as an internal concept so this may be
	// set to any meaningful value within the environment. For example,
	// GKE clusters have a name which can be used for this label.
	K8SClusterName    = kv.Key("k8s.cluster.name")
	K8SNamespaceName  = kv.Key("k8s.namespace.name")
	K8SPodName        = kv.Key("k8s.pod.name")
	K8SDeploymentName = kv.Key("k8s.deployment.name")
)

// Constants for Container resources.
const (
	// A uniquely identifying name for the Container.
	ContainerName      = kv.Key("container.name")
	ContainerImageName = kv.Key("container.image.name")
	ContainerImageTag  = kv.Key("container.image.tag")
)

// Constants for Cloud resources.
const (
	CloudProvider  = kv.Key("cloud.provider")
	CloudAccountID = kv.Key("cloud.account.id")
	CloudRegion    = kv.Key("cloud.region")
	CloudZone      = kv.Key("cloud.zone")
)

// Constants for Host resources.
const (
	// A uniquely identifying name for the host.
	HostName = kv.Key("host.name")

	// A hostname as returned by the 'hostname' command on host machine.
	HostHostName     = kv.Key("host.hostname")
	HostID           = kv.Key("host.id")
	HostType         = kv.Key("host.type")
	HostImageName    = kv.Key("host.image.name")
	HostImageID      = kv.Key("host.image.id")
	HostImageVersion = kv.Key("host.image.version")
)
<|endoftext|>


<commit_before>package command

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/gofmt"
)

type UnderReplicated struct {
	Ui  cli.Ui
	Cmd string

	zone    string
	cluster string
	debug   bool
}

func (this *UnderReplicated) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("underreplicated", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", "", "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	cmdFlags.BoolVar(&this.debug, "debug", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if this.debug {
		sarama.Logger = log.New(os.Stderr, color.Magenta("[sarama]"), log.LstdFlags)
	}

	if this.zone == "" {
		forSortedZones(func(zkzone *zk.ZkZone) {
			this.Ui.Output(zkzone.Name())
			zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
				lines := this.displayUnderReplicatedPartitionsOfCluster(zkcluster)
				if len(lines) > 0 {
					this.Ui.Output(fmt.Sprintf("%s %s", strings.Repeat(" ", 4), zkcluster.Name()))
					for _, l := range lines {
						this.Ui.Warn(l)
					}
				}
			})

			printSwallowedErrors(this.Ui, zkzone)
		})

		return
	}

	// a single zone
	ensureZoneValid(this.zone)
	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	this.Ui.Output(zkzone.Name())
	if this.cluster != "" {
		zkcluster := zkzone.NewCluster(this.cluster)
		lines := this.displayUnderReplicatedPartitionsOfCluster(zkcluster)
		if len(lines) > 0 {
			this.Ui.Output(fmt.Sprintf("%s %s", strings.Repeat(" ", 4), zkcluster.Name()))
			for _, l := range lines {
				this.Ui.Warn(l)
			}
		}
	} else {
		// all clusters
		zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
			lines := this.displayUnderReplicatedPartitionsOfCluster(zkcluster)
			if len(lines) > 0 {
				this.Ui.Output(fmt.Sprintf("%s %s", strings.Repeat(" ", 4), zkcluster.Name()))
				for _, l := range lines {
					this.Ui.Warn(l)
				}
			}
		})
	}

	printSwallowedErrors(this.Ui, zkzone)

	return
}

func (this *UnderReplicated) displayUnderReplicatedPartitionsOfCluster(zkcluster *zk.ZkCluster) []string {
	brokerList := zkcluster.BrokerList()
	if len(brokerList) == 0 {
		this.Ui.Warn(fmt.Sprintf("%s empty brokers", zkcluster.Name()))
		return nil
	}

	cf := sarama.NewConfig()
	cf.Net.ReadTimeout = time.Second * 4
	cf.Net.WriteTimeout = time.Second * 4
	kfk, err := sarama.NewClient(brokerList, cf)
	if err != nil {
		this.Ui.Error(fmt.Sprintf("%s %+v %s", zkcluster.Name(), brokerList, err.Error()))

		return nil
	}
	defer kfk.Close()

	topics, err := kfk.Topics()
	swallow(err)
	if len(topics) == 0 {
		return nil
	}

	lines := make([]string, 0, 10)
	for _, topic := range topics {
		// get partitions and check if some dead
		alivePartitions, err := kfk.WritablePartitions(topic)
		if err != nil {
			this.Ui.Error(fmt.Sprintf("%s topic[%s] cannot fetch writable partitions: %v", zkcluster.Name(), topic, err))
			continue
		}
		partions, err := kfk.Partitions(topic)
		if err != nil {
			this.Ui.Error(fmt.Sprintf("%s topic[%s] cannot fetch partitions: %v", zkcluster.Name(), topic, err))
			continue
		}
		if len(alivePartitions) != len(partions) {
			this.Ui.Error(fmt.Sprintf("%s topic[%s] has %s partitions: %+v/%+v", zkcluster.Name(),
				topic, color.Red("dead"), alivePartitions, partions))
		}

		for _, partitionID := range alivePartitions {
			replicas, err := kfk.Replicas(topic, partitionID)
			if err != nil {
				this.Ui.Error(fmt.Sprintf("%s topic[%s] P:%d: %v", zkcluster.Name(), topic, partitionID, err))
				continue
			}

			isr, isrMtime, partitionCtime := zkcluster.Isr(topic, partitionID)

			underReplicated := false
			if len(isr) != len(replicas) {
				underReplicated = true
			}

			if underReplicated {
				leader, err := kfk.Leader(topic, partitionID)
				swallow(err)

				latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest)
				swallow(err)

				oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest)
				swallow(err)

				lines = append(lines, fmt.Sprintf("\t%s Partition:%d/%s Leader:%d Replicas:%+v Isr:%+v/%s Offset:%d-%d Num:%d",
					topic, partitionID,
					gofmt.PrettySince(partitionCtime),
					leader.ID(), replicas, isr,
					gofmt.PrettySince(isrMtime),
					oldestOffset, latestOffset, latestOffset-oldestOffset))
			}
		}
	}

	return lines
}

func (*UnderReplicated) Synopsis() string {
	return "Display under-replicated partitions"
}

func (this *UnderReplicated) Help() string {
	help := fmt.Sprintf(`
Usage: %s underreplicated [options]

    Display under-replicated partitions

Options:

    -z zone

    -c cluster

    -debug
`, this.Cmd)
	return strings.TrimSpace(help)
}
<commit_msg>add default zone<commit_after>package command

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/gofmt"
)

type UnderReplicated struct {
	Ui  cli.Ui
	Cmd string

	zone    string
	cluster string
	debug   bool
}

func (this *UnderReplicated) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("underreplicated", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	cmdFlags.BoolVar(&this.debug, "debug", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if this.debug {
		sarama.Logger = log.New(os.Stderr, color.Magenta("[sarama]"), log.LstdFlags)
	}

	if this.zone == "" {
		forSortedZones(func(zkzone *zk.ZkZone) {
			this.Ui.Output(zkzone.Name())
			zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
				lines := this.displayUnderReplicatedPartitionsOfCluster(zkcluster)
				if len(lines) > 0 {
					this.Ui.Output(fmt.Sprintf("%s %s", strings.Repeat(" ", 4), zkcluster.Name()))
					for _, l := range lines {
						this.Ui.Warn(l)
					}
				}
			})

			printSwallowedErrors(this.Ui, zkzone)
		})

		return
	}

	// a single zone
	ensureZoneValid(this.zone)
	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	this.Ui.Output(zkzone.Name())
	if this.cluster != "" {
		zkcluster := zkzone.NewCluster(this.cluster)
		lines := this.displayUnderReplicatedPartitionsOfCluster(zkcluster)
		if len(lines) > 0 {
			this.Ui.Output(fmt.Sprintf("%s %s", strings.Repeat(" ", 4), zkcluster.Name()))
			for _, l := range lines {
				this.Ui.Warn(l)
			}
		}
	} else {
		// all clusters
		zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
			lines := this.displayUnderReplicatedPartitionsOfCluster(zkcluster)
			if len(lines) > 0 {
				this.Ui.Output(fmt.Sprintf("%s %s", strings.Repeat(" ", 4), zkcluster.Name()))
				for _, l := range lines {
					this.Ui.Warn(l)
				}
			}
		})
	}

	printSwallowedErrors(this.Ui, zkzone)

	return
}

func (this *UnderReplicated) displayUnderReplicatedPartitionsOfCluster(zkcluster *zk.ZkCluster) []string {
	brokerList := zkcluster.BrokerList()
	if len(brokerList) == 0 {
		this.Ui.Warn(fmt.Sprintf("%s empty brokers", zkcluster.Name()))
		return nil
	}

	cf := sarama.NewConfig()
	cf.Net.ReadTimeout = time.Second * 4
	cf.Net.WriteTimeout = time.Second * 4
	kfk, err := sarama.NewClient(brokerList, cf)
	if err != nil {
		this.Ui.Error(fmt.Sprintf("%s %+v %s", zkcluster.Name(), brokerList, err.Error()))

		return nil
	}
	defer kfk.Close()

	topics, err := kfk.Topics()
	swallow(err)
	if len(topics) == 0 {
		return nil
	}

	lines := make([]string, 0, 10)
	for _, topic := range topics {
		// get partitions and check if some dead
		alivePartitions, err := kfk.WritablePartitions(topic)
		if err != nil {
			this.Ui.Error(fmt.Sprintf("%s topic[%s] cannot fetch writable partitions: %v", zkcluster.Name(), topic, err))
			continue
		}
		partions, err := kfk.Partitions(topic)
		if err != nil {
			this.Ui.Error(fmt.Sprintf("%s topic[%s] cannot fetch partitions: %v", zkcluster.Name(), topic, err))
			continue
		}
		if len(alivePartitions) != len(partions) {
			this.Ui.Error(fmt.Sprintf("%s topic[%s] has %s partitions: %+v/%+v", zkcluster.Name(),
				topic, color.Red("dead"), alivePartitions, partions))
		}

		for _, partitionID := range alivePartitions {
			replicas, err := kfk.Replicas(topic, partitionID)
			if err != nil {
				this.Ui.Error(fmt.Sprintf("%s topic[%s] P:%d: %v", zkcluster.Name(), topic, partitionID, err))
				continue
			}

			isr, isrMtime, partitionCtime := zkcluster.Isr(topic, partitionID)

			underReplicated := false
			if len(isr) != len(replicas) {
				underReplicated = true
			}

			if underReplicated {
				leader, err := kfk.Leader(topic, partitionID)
				swallow(err)

				latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest)
				swallow(err)

				oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest)
				swallow(err)

				lines = append(lines, fmt.Sprintf("\t%s Partition:%d/%s Leader:%d Replicas:%+v Isr:%+v/%s Offset:%d-%d Num:%d",
					topic, partitionID,
					gofmt.PrettySince(partitionCtime),
					leader.ID(), replicas, isr,
					gofmt.PrettySince(isrMtime),
					oldestOffset, latestOffset, latestOffset-oldestOffset))
			}
		}
	}

	return lines
}

func (*UnderReplicated) Synopsis() string {
	return "Display under-replicated partitions"
}

func (this *UnderReplicated) Help() string {
	help := fmt.Sprintf(`
Usage: %s underreplicated [options]

    Display under-replicated partitions

Options:

    -z zone

    -c cluster

    -debug
`, this.Cmd)
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os/exec"
	"regexp"
	"strconv"
	"strings"

	"github.com/NetSys/quilt/api"
	"github.com/NetSys/quilt/api/client/getter"
	log "github.com/Sirupsen/logrus"
)

var machineRegex = regexp.MustCompile(`Machine-(\d+){(.+?), .*, PublicIP=(.*?),`)

func main() {
	printQuiltPs()

	c, err := getter.New().Client(api.DefaultSocket)
	if err != nil {
		log.WithError(err).Fatal("FAILED, couldn't get quiltctl client")
	}
	defer c.Close()

	machines, err := c.QueryMachines()
	if err != nil {
		log.WithError(err).Fatal("FAILED, couldn't query the machines")
	}

	failed := false
	for _, machine := range machines {
		fmt.Println(machine)
		logsOutput, err := exec.Command("quilt", "ssh", strconv.Itoa(machine.ID),
			"docker", "logs", "minion").CombinedOutput()
		if err != nil {
			log.WithError(err).Error("Unable to get minion logs")
			failed = true
			continue
		}
		outputStr := string(logsOutput)
		fmt.Println(outputStr)

		if strings.Contains(outputStr, "ERROR") ||
			strings.Contains(outputStr, "WARN") {
			failed = true
		}
	}

	if failed {
		fmt.Println("FAILED")
	} else {
		fmt.Println("PASSED")
	}
}

func printQuiltPs() {
	psout, err := exec.Command("quilt", "ps").CombinedOutput()
	if err != nil {
		log.WithError(err).Fatal("Failed to run `quilt ps`")
	}
	fmt.Println(string(psout))
}
<commit_msg>ci: Use journalctl to get minion logs<commit_after>package main

import (
	"fmt"
	"os/exec"
	"regexp"
	"strconv"
	"strings"

	"github.com/NetSys/quilt/api"
	"github.com/NetSys/quilt/api/client/getter"
	log "github.com/Sirupsen/logrus"
)

var machineRegex = regexp.MustCompile(`Machine-(\d+){(.+?), .*, PublicIP=(.*?),`)

func main() {
	printQuiltPs()

	c, err := getter.New().Client(api.DefaultSocket)
	if err != nil {
		log.WithError(err).Fatal("FAILED, couldn't get quiltctl client")
	}
	defer c.Close()

	machines, err := c.QueryMachines()
	if err != nil {
		log.WithError(err).Fatal("FAILED, couldn't query the machines")
	}

	failed := false
	for _, machine := range machines {
		fmt.Println(machine)
		logsOutput, err := exec.Command("quilt", "ssh", strconv.Itoa(machine.ID),
			"sudo", "journalctl", "-o", "cat", "-u", "minion").
			CombinedOutput()
		if err != nil {
			log.WithError(err).Error("Unable to get minion logs")
			failed = true
			continue
		}
		outputStr := string(logsOutput)
		fmt.Println(outputStr)

		if strings.Contains(outputStr, "ERROR") ||
			strings.Contains(outputStr, "WARN") {
			failed = true
		}
	}

	if failed {
		fmt.Println("FAILED")
	} else {
		fmt.Println("PASSED")
	}
}

func printQuiltPs() {
	psout, err := exec.Command("quilt", "ps").CombinedOutput()
	if err != nil {
		log.WithError(err).Fatal("Failed to run `quilt ps`")
	}
	fmt.Println(string(psout))
}
<|endoftext|>


<commit_before>package checks

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/armon/circbuf"
	"github.com/docker/go-connections/sockets"
)

// DockerClient is a simplified client for the Docker Engine API
// to execute the health checks and avoid significant dependencies.
// It also consumes all data returned from the Docker API through
// a ring buffer with a fixed limit to avoid excessive resource
// consumption.
type DockerClient struct {
	host     string
	scheme   string
	proto    string
	addr     string
	basepath string
	maxbuf   int64
	client   *http.Client
}

func NewDockerClient(host string, maxbuf int64) (*DockerClient, error) {
	if host == "" {
		host = DefaultDockerHost
	}

	proto, addr, basepath, err := ParseHost(host)
	if err != nil {
		return nil, err
	}

	transport := new(http.Transport)
	sockets.ConfigureTransport(transport, proto, addr)
	client := &http.Client{Transport: transport}

	return &DockerClient{
		host:     host,
		scheme:   "http",
		proto:    proto,
		addr:     addr,
		basepath: basepath,
		maxbuf:   maxbuf,
		client:   client,
	}, nil
}

func (c *DockerClient) Host() string {
	return c.host
}

// ParseHost verifies that the given host strings is valid.
// copied from github.com/docker/docker/client.go
func ParseHost(host string) (string, string, string, error) {
	protoAddrParts := strings.SplitN(host, "://", 2)
	if len(protoAddrParts) == 1 {
		return "", "", "", fmt.Errorf("unable to parse docker host `%s`", host)
	}

	var basePath string
	proto, addr := protoAddrParts[0], protoAddrParts[1]
	if proto == "tcp" {
		parsed, err := url.Parse("tcp://" + addr)
		if err != nil {
			return "", "", "", err
		}
		addr = parsed.Host
		basePath = parsed.Path
	}
	return proto, addr, basePath, nil
}

func (c *DockerClient) call(method, uri string, v interface{}) (*circbuf.Buffer, int, error) {
	req, err := http.NewRequest(method, uri, nil)
	if err != nil {
		return nil, 0, err
	}

	if c.proto == "unix" || c.proto == "npipe" {
		// For local communications, it doesn't matter what the host is. We just
		// need a valid and meaningful host name. (See #189)
		req.Host = "docker"
	}

	req.URL.Host = c.addr
	req.URL.Scheme = c.scheme

	if v != nil {
		var b bytes.Buffer
		if err := json.NewEncoder(&b).Encode(v); err != nil {
			return nil, 0, err
		}
		req.Body = ioutil.NopCloser(&b)
		req.Header.Set("Content-Type", "application/json")
	}

	resp, err := c.client.Do(req)
	if err != nil {
		return nil, 0, err
	}
	defer resp.Body.Close()

	b, err := circbuf.NewBuffer(c.maxbuf)
	if err != nil {
		return nil, 0, err
	}
	_, err = io.Copy(b, resp.Body)
	return b, resp.StatusCode, err
}

func (c *DockerClient) CreateExec(containerID string, cmd []string) (string, error) {
	data := struct {
		AttachStdin  bool
		AttachStdout bool
		AttachStderr bool
		Tty          bool
		Cmd          []string
	}{
		AttachStderr: true,
		AttachStdout: true,
		Cmd:          cmd,
	}

	uri := fmt.Sprintf("/containers/%s/exec", url.QueryEscape(containerID))
	b, code, err := c.call("POST", uri, data)
	switch {
	case err != nil:
		return "", fmt.Errorf("create exec failed for container %s: %s", containerID, err)
	case code == 201:
		var resp struct{ Id string }
		if err = json.NewDecoder(bytes.NewReader(b.Bytes())).Decode(&resp); err != nil {
			return "", fmt.Errorf("create exec response for container %s cannot be parsed: %s", containerID, err)
		}
		return resp.Id, nil
	case code == 404:
		return "", fmt.Errorf("create exec failed for unknown container %s", containerID)
	case code == 409:
		return "", fmt.Errorf("create exec failed since container %s is paused or stopped", containerID)
	default:
		return "", fmt.Errorf("create exec failed for container %s with status %d: %s", containerID, code, b)
	}
}

func (c *DockerClient) StartExec(containerID, execID string) (*circbuf.Buffer, error) {
	data := struct{ Detach, Tty bool }{Detach: false, Tty: true}
	uri := fmt.Sprintf("/exec/%s/start", execID)
	b, code, err := c.call("POST", uri, data)
	switch {
	case err != nil:
		return nil, fmt.Errorf("start exec failed for container %s: %s", containerID, err)
	case code == 200:
		return b, nil
	case code == 404:
		return nil, fmt.Errorf("start exec failed for container %s: invalid exec id %s", containerID, execID)
	case code == 409:
		return nil, fmt.Errorf("start exec failed since container %s is paused or stopped", containerID)
	default:
		return nil, fmt.Errorf("start exec failed for container %s with status %d: %s", containerID, code, b)
	}
}

func (c *DockerClient) InspectExec(containerID, execID string) (int, error) {
	uri := fmt.Sprintf("/exec/%s/json", execID)
	b, code, err := c.call("GET", uri, nil)
	switch {
	case err != nil:
		return 0, fmt.Errorf("inspect exec failed for container %s: %s", containerID, err)
	case code == 200:
		var resp struct{ ExitCode int }
		if err := json.NewDecoder(bytes.NewReader(b.Bytes())).Decode(&resp); err != nil {
			return 0, fmt.Errorf("inspect exec response for container %s cannot be parsed: %s", containerID, err)
		}
		return resp.ExitCode, nil
	case code == 404:
		return 0, fmt.Errorf("inspect exec failed for container %s: invalid exec id %s", containerID, execID)
	default:
		return 0, fmt.Errorf("inspect exec failed for container %s with status %d: %s", containerID, code, b)
	}
}
<commit_msg>docker: ignore "connection reset by peer"<commit_after>package checks

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/armon/circbuf"
	"github.com/docker/go-connections/sockets"
)

// DockerClient is a simplified client for the Docker Engine API
// to execute the health checks and avoid significant dependencies.
// It also consumes all data returned from the Docker API through
// a ring buffer with a fixed limit to avoid excessive resource
// consumption.
type DockerClient struct {
	host     string
	scheme   string
	proto    string
	addr     string
	basepath string
	maxbuf   int64
	client   *http.Client
}

func NewDockerClient(host string, maxbuf int64) (*DockerClient, error) {
	if host == "" {
		host = DefaultDockerHost
	}

	proto, addr, basepath, err := ParseHost(host)
	if err != nil {
		return nil, err
	}

	transport := new(http.Transport)
	sockets.ConfigureTransport(transport, proto, addr)
	client := &http.Client{Transport: transport}

	return &DockerClient{
		host:     host,
		scheme:   "http",
		proto:    proto,
		addr:     addr,
		basepath: basepath,
		maxbuf:   maxbuf,
		client:   client,
	}, nil
}

func (c *DockerClient) Host() string {
	return c.host
}

// ParseHost verifies that the given host strings is valid.
// copied from github.com/docker/docker/client.go
func ParseHost(host string) (string, string, string, error) {
	protoAddrParts := strings.SplitN(host, "://", 2)
	if len(protoAddrParts) == 1 {
		return "", "", "", fmt.Errorf("unable to parse docker host `%s`", host)
	}

	var basePath string
	proto, addr := protoAddrParts[0], protoAddrParts[1]
	if proto == "tcp" {
		parsed, err := url.Parse("tcp://" + addr)
		if err != nil {
			return "", "", "", err
		}
		addr = parsed.Host
		basePath = parsed.Path
	}
	return proto, addr, basePath, nil
}

func (c *DockerClient) call(method, uri string, v interface{}) (*circbuf.Buffer, int, error) {
	req, err := http.NewRequest(method, uri, nil)
	if err != nil {
		return nil, 0, err
	}

	if c.proto == "unix" || c.proto == "npipe" {
		// For local communications, it doesn't matter what the host is. We just
		// need a valid and meaningful host name. (See #189)
		req.Host = "docker"
	}

	req.URL.Host = c.addr
	req.URL.Scheme = c.scheme

	if v != nil {
		var b bytes.Buffer
		if err := json.NewEncoder(&b).Encode(v); err != nil {
			return nil, 0, err
		}
		req.Body = ioutil.NopCloser(&b)
		req.Header.Set("Content-Type", "application/json")
	}

	resp, err := c.client.Do(req)
	if err != nil {
		return nil, 0, err
	}
	defer resp.Body.Close()

	b, err := circbuf.NewBuffer(c.maxbuf)
	if err != nil {
		return nil, 0, err
	}
	_, err = io.Copy(b, resp.Body)
	return b, resp.StatusCode, err
}

func (c *DockerClient) CreateExec(containerID string, cmd []string) (string, error) {
	data := struct {
		AttachStdin  bool
		AttachStdout bool
		AttachStderr bool
		Tty          bool
		Cmd          []string
	}{
		AttachStderr: true,
		AttachStdout: true,
		Cmd:          cmd,
	}

	uri := fmt.Sprintf("/containers/%s/exec", url.QueryEscape(containerID))
	b, code, err := c.call("POST", uri, data)
	switch {
	case err != nil:
		return "", fmt.Errorf("create exec failed for container %s: %s", containerID, err)
	case code == 201:
		var resp struct{ Id string }
		if err = json.NewDecoder(bytes.NewReader(b.Bytes())).Decode(&resp); err != nil {
			return "", fmt.Errorf("create exec response for container %s cannot be parsed: %s", containerID, err)
		}
		return resp.Id, nil
	case code == 404:
		return "", fmt.Errorf("create exec failed for unknown container %s", containerID)
	case code == 409:
		return "", fmt.Errorf("create exec failed since container %s is paused or stopped", containerID)
	default:
		return "", fmt.Errorf("create exec failed for container %s with status %d: %s", containerID, code, b)
	}
}

func (c *DockerClient) StartExec(containerID, execID string) (*circbuf.Buffer, error) {
	data := struct{ Detach, Tty bool }{Detach: false, Tty: true}
	uri := fmt.Sprintf("/exec/%s/start", execID)
	b, code, err := c.call("POST", uri, data)
	switch {
	case err != nil && !strings.Contains(err.Error(), "connection reset by peer"):
		return nil, fmt.Errorf("start exec failed for container %s: %s", containerID, err)
	case code == 200:
		return b, nil
	case code == 404:
		return nil, fmt.Errorf("start exec failed for container %s: invalid exec id %s", containerID, execID)
	case code == 409:
		return nil, fmt.Errorf("start exec failed since container %s is paused or stopped", containerID)
	default:
		return nil, fmt.Errorf("start exec failed for container %s with status %d: %s", containerID, code, b)
	}
}

func (c *DockerClient) InspectExec(containerID, execID string) (int, error) {
	uri := fmt.Sprintf("/exec/%s/json", execID)
	b, code, err := c.call("GET", uri, nil)
	switch {
	case err != nil:
		return 0, fmt.Errorf("inspect exec failed for container %s: %s", containerID, err)
	case code == 200:
		var resp struct{ ExitCode int }
		if err := json.NewDecoder(bytes.NewReader(b.Bytes())).Decode(&resp); err != nil {
			return 0, fmt.Errorf("inspect exec response for container %s cannot be parsed: %s", containerID, err)
		}
		return resp.ExitCode, nil
	case code == 404:
		return 0, fmt.Errorf("inspect exec failed for container %s: invalid exec id %s", containerID, execID)
	default:
		return 0, fmt.Errorf("inspect exec failed for container %s with status %d: %s", containerID, code, b)
	}
}
<|endoftext|>


<commit_before>package filehandler

import (
	"bufio"
	"errors"
	"github.com/jzipfler/htw-ava/server"
	"github.com/jzipfler/htw-ava/utils"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
)

const (
	ERROR_HEADER = "-------------ERROR-------------"
	ERROR_FOOTER = "^^^^^^^^^^^^^ERROR^^^^^^^^^^^^^"
)

// This function tries to read the content of the file with the given name
// as parameter. If it is not readable or nothing can be parsed, the function
// returns nil for the map and sets the value for the error type.
func CollectAllFromNodeListFile(nodeListFile string) (allNodes map[int]server.NetworkServer, err error) {
	if err := utils.CheckIfFileIsReadable(nodeListFile); err != nil {
		return nil, err
	}
	nodeListFileObject, _ := os.Open(nodeListFile)
	defer nodeListFileObject.Close()
	allNodes = make(map[int]server.NetworkServer, 10)
	scanner := bufio.NewScanner(nodeListFileObject)
	for scanner.Scan() {
		var scanId int
		var scanServerObject server.NetworkServer
		line := scanner.Text()
		if line == "" {
			log.Printf("Leere Zeile gelesen.\n")
			continue
		}
		if strings.HasPrefix(line, "#") {
			log.Printf("Kommentar gelesen: \"%s\"\n", line)
			continue
		}
		idAndIpPortArray := strings.Split(line, " ")
		if len(idAndIpPortArray) == 0 {
			log.Printf("Could not split the line with a space: \"%s\".\n%s\n", line, ERROR_FOOTER)
			continue
		}
		scanId, err := strconv.Atoi(idAndIpPortArray[0])
		if err != nil {
			log.Printf("Could not parse the first part of the line to a number : \"%s\".\n%s\n", idAndIpPortArray[0], ERROR_FOOTER)
			continue
		} else {
			scanServerObject.SetClientName(idAndIpPortArray[0])
		}
		ipAndPortArray := strings.Split(idAndIpPortArray[1], ":")
		if len(ipAndPortArray) == 0 {
			log.Printf("Could not split the ip address and port with a colon: \"%s\".\n%s\n", idAndIpPortArray[1], ERROR_FOOTER)
			continue
		}
		//Check if the given part is a ip address or a host.
		if splitIpArray, err := net.LookupIP(ipAndPortArray[0]); err != nil {
			log.Printf("Could not lookup this ip/host: \"%s\".\n%s\n", ipAndPortArray[0], ERROR_FOOTER)
			continue
		} else {
			if len(splitIpArray) == 0 {
				log.Printf("No ip found: \"%s\".\n%s\n", ipAndPortArray[0], ERROR_FOOTER)
				continue
			}
			//If we have some ip addresses, lets check if they are from version 4.
			ipv4Found := false
			for _, value := range splitIpArray {
				if value.To4() != nil {
					ipv4Found = true
					scanServerObject.SetIpAddress(value)
					break
				}
			}
			if !ipv4Found {
				log.Printf("No ipv4 found: \"%s\".\n%s\n", ipAndPortArray[0], ERROR_FOOTER)
				continue
			}
		}
		if splitPort, err := strconv.Atoi(ipAndPortArray[1]); err != nil {
			log.Printf("Could not parse the port: \"%s\".\n%s\n", ipAndPortArray[1], ERROR_FOOTER)
			continue
		} else {
			scanServerObject.SetPort(splitPort)
		}
		//TODO: Maybe the usedProtocol should be set in a other way.
		scanServerObject.SetUsedProtocol("tcp")
		allNodes[scanId] = scanServerObject
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	if len(allNodes) == 0 {
		return nil, errors.New("No nodes present... ABORT")
	}
	return allNodes, nil
}
<commit_msg>Commented out the "skipped" logging and removed the local constants (using the consts in utils)<commit_after>package filehandler

import (
	"bufio"
	"errors"
	"github.com/jzipfler/htw-ava/server"
	"github.com/jzipfler/htw-ava/utils"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
)

// This function tries to read the content of the file with the given name
// as parameter. If it is not readable or nothing can be parsed, the function
// returns nil for the map and sets the value for the error type.
func CollectAllFromNodeListFile(nodeListFile string) (allNodes map[int]server.NetworkServer, err error) {
	if err := utils.CheckIfFileIsReadable(nodeListFile); err != nil {
		return nil, err
	}
	nodeListFileObject, _ := os.Open(nodeListFile)
	defer nodeListFileObject.Close()
	allNodes = make(map[int]server.NetworkServer, 10)
	scanner := bufio.NewScanner(nodeListFileObject)
	for scanner.Scan() {
		var scanId int
		var scanServerObject server.NetworkServer
		line := scanner.Text()
		if line == "" {
			//log.Printf("Leere Zeile gelesen.\n")
			continue
		}
		if strings.HasPrefix(line, "#") {
			//log.Printf("Kommentar gelesen: \"%s\"\n", line)
			continue
		}
		idAndIpPortArray := strings.Split(line, " ")
		if len(idAndIpPortArray) == 0 {
			log.Printf("Could not split the line with a space: \"%s\".\n%s\n", line, utils.ERROR_FOOTER)
			continue
		}
		scanId, err := strconv.Atoi(idAndIpPortArray[0])
		if err != nil {
			log.Printf("Could not parse the first part of the line to a number : \"%s\".\n%s\n", idAndIpPortArray[0], utils.ERROR_FOOTER)
			continue
		} else {
			scanServerObject.SetClientName(idAndIpPortArray[0])
		}
		ipAndPortArray := strings.Split(idAndIpPortArray[1], ":")
		if len(ipAndPortArray) == 0 {
			log.Printf("Could not split the ip address and port with a colon: \"%s\".\n%s\n", idAndIpPortArray[1], utils.ERROR_FOOTER)
			continue
		}
		//Check if the given part is a ip address or a host.
		if splitIpArray, err := net.LookupIP(ipAndPortArray[0]); err != nil {
			log.Printf("Could not lookup this ip/host: \"%s\".\n%s\n", ipAndPortArray[0], utils.ERROR_FOOTER)
			continue
		} else {
			if len(splitIpArray) == 0 {
				log.Printf("No ip found: \"%s\".\n%s\n", ipAndPortArray[0], utils.ERROR_FOOTER)
				continue
			}
			//If we have some ip addresses, lets check if they are from version 4.
			ipv4Found := false
			for _, value := range splitIpArray {
				if value.To4() != nil {
					ipv4Found = true
					scanServerObject.SetIpAddress(value)
					break
				}
			}
			if !ipv4Found {
				log.Printf("No ipv4 found: \"%s\".\n%s\n", ipAndPortArray[0], utils.ERROR_FOOTER)
				continue
			}
		}
		if splitPort, err := strconv.Atoi(ipAndPortArray[1]); err != nil {
			log.Printf("Could not parse the port: \"%s\".\n%s\n", ipAndPortArray[1], utils.ERROR_FOOTER)
			continue
		} else {
			scanServerObject.SetPort(splitPort)
		}
		//TODO: Maybe the usedProtocol should be set in a other way.
		scanServerObject.SetUsedProtocol("tcp")
		allNodes[scanId] = scanServerObject
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	if len(allNodes) == 0 {
		return nil, errors.New("No nodes present... ABORT")
	}
	return allNodes, nil
}
<|endoftext|>


<commit_before>package firehosenozzle

import (
	"crypto/tls"
	"time"

	"github.com/cloudfoundry/noaa/consumer"
	noaerrors "github.com/cloudfoundry/noaa/errors"
	"github.com/cloudfoundry/sonde-go/events"
	"github.com/gorilla/websocket"
	"github.com/prometheus/common/log"

	"github.com/cloudfoundry-community/firehose_exporter/metrics"
)

type FirehoseNozzle struct {
	url                string
	skipSSLValidation  bool
	subscriptionID     string
	idleTimeout        time.Duration
	minRetryDelay      time.Duration
	maxRetryDelay      time.Duration
	maxRetryCount      int
	authTokenRefresher consumer.TokenRefresher
	metricsStore       *metrics.Store
	errs               <-chan error
	messages           <-chan *events.Envelope
	consumer           *consumer.Consumer
}

func New(
	url string,
	skipSSLValidation bool,
	subscriptionID string,
	idleTimeout time.Duration,
	minRetryDelay time.Duration,
	maxRetryDelay time.Duration,
	maxRetryCount int,
	authTokenRefresher consumer.TokenRefresher,
	metricsStore *metrics.Store,
) *FirehoseNozzle {
	return &FirehoseNozzle{
		url:                url,
		skipSSLValidation:  skipSSLValidation,
		subscriptionID:     subscriptionID,
		idleTimeout:        idleTimeout,
		minRetryDelay:      minRetryDelay,
		maxRetryDelay:      maxRetryDelay,
		maxRetryCount:      maxRetryCount,
		authTokenRefresher: authTokenRefresher,
		metricsStore:       metricsStore,
		errs:               make(<-chan error),
		messages:           make(<-chan *events.Envelope),
	}
}

func (n *FirehoseNozzle) Start() error {
	log.Info("Starting Firehose Nozzle...")
	n.consumeFirehose()
	err := n.parseEnvelopes()
	log.Info("Firehose Nozzle shutting down...")
	return err
}

func (n *FirehoseNozzle) consumeFirehose() {
	n.consumer = consumer.New(
		n.url,
		&tls.Config{InsecureSkipVerify: n.skipSSLValidation},
		nil,
	)
	n.consumer.RefreshTokenFrom(n.authTokenRefresher)
	n.consumer.SetDebugPrinter(DebugPrinter{})
	if n.idleTimeout > 0 {
		n.consumer.SetIdleTimeout(n.idleTimeout)
	}
	if n.minRetryDelay > 0 {
		n.consumer.SetMinRetryDelay(n.minRetryDelay)
	}
	if n.maxRetryDelay > 0 {
		n.consumer.SetMaxRetryDelay(n.maxRetryDelay)
	}
	if n.maxRetryCount > 0 {
		n.consumer.SetMaxRetryCount(n.maxRetryCount)
	}
	n.messages, n.errs = n.consumer.Firehose(n.subscriptionID, "")
}

func (n *FirehoseNozzle) parseEnvelopes() error {
	for {
		select {
		case envelope := <-n.messages:
			n.handleMessage(envelope)
			n.metricsStore.AddMetric(envelope)
		case err := <-n.errs:
			retryError := n.handleError(err)
			if !retryError {
				return err
			}
		}
	}
}

func (n *FirehoseNozzle) handleMessage(envelope *events.Envelope) {
	if envelope.GetEventType() == events.Envelope_CounterEvent && envelope.CounterEvent.GetName() == "TruncatingBuffer.DroppedMessages" && envelope.GetOrigin() == "doppler" {
		log.Infof("We've intercepted an upstream message which indicates that the Nozzle or the TrafficController is not keeping up. Please try scaling up the Nozzle.")
		n.metricsStore.AlertSlowConsumerError()
	}
}

func (n *FirehoseNozzle) handleError(err error) bool {
	log.Errorf("Error while reading from the Firehose: %v", err)

	switch err.(type) {
	case noaerrors.RetryError:
		switch noaRetryError := err.(noaerrors.RetryError).Err.(type) {
		case *websocket.CloseError:
			switch noaRetryError.Code {
			case websocket.CloseNormalClosure:
			// no op
			case websocket.ClosePolicyViolation:
				log.Errorf("Nozzle couldn't keep up. Please try scaling up the Nozzle.")
				n.metricsStore.AlertSlowConsumerError()
			}
		}
		return true
	}

	log.Info("Closing connection with Firehose...")
	n.consumer.Close()

	return false
}

type DebugPrinter struct{}

func (dp DebugPrinter) Print(title, dump string) {
	log.Debugf("%s: %s", title, dump)
}
<commit_msg>Remove useless switch clause<commit_after>package firehosenozzle

import (
	"crypto/tls"
	"time"

	"github.com/cloudfoundry/noaa/consumer"
	noaerrors "github.com/cloudfoundry/noaa/errors"
	"github.com/cloudfoundry/sonde-go/events"
	"github.com/gorilla/websocket"
	"github.com/prometheus/common/log"

	"github.com/cloudfoundry-community/firehose_exporter/metrics"
)

type FirehoseNozzle struct {
	url                string
	skipSSLValidation  bool
	subscriptionID     string
	idleTimeout        time.Duration
	minRetryDelay      time.Duration
	maxRetryDelay      time.Duration
	maxRetryCount      int
	authTokenRefresher consumer.TokenRefresher
	metricsStore       *metrics.Store
	errs               <-chan error
	messages           <-chan *events.Envelope
	consumer           *consumer.Consumer
}

func New(
	url string,
	skipSSLValidation bool,
	subscriptionID string,
	idleTimeout time.Duration,
	minRetryDelay time.Duration,
	maxRetryDelay time.Duration,
	maxRetryCount int,
	authTokenRefresher consumer.TokenRefresher,
	metricsStore *metrics.Store,
) *FirehoseNozzle {
	return &FirehoseNozzle{
		url:                url,
		skipSSLValidation:  skipSSLValidation,
		subscriptionID:     subscriptionID,
		idleTimeout:        idleTimeout,
		minRetryDelay:      minRetryDelay,
		maxRetryDelay:      maxRetryDelay,
		maxRetryCount:      maxRetryCount,
		authTokenRefresher: authTokenRefresher,
		metricsStore:       metricsStore,
		errs:               make(<-chan error),
		messages:           make(<-chan *events.Envelope),
	}
}

func (n *FirehoseNozzle) Start() error {
	log.Info("Starting Firehose Nozzle...")
	n.consumeFirehose()
	err := n.parseEnvelopes()
	log.Info("Firehose Nozzle shutting down...")
	return err
}

func (n *FirehoseNozzle) consumeFirehose() {
	n.consumer = consumer.New(
		n.url,
		&tls.Config{InsecureSkipVerify: n.skipSSLValidation},
		nil,
	)
	n.consumer.RefreshTokenFrom(n.authTokenRefresher)
	n.consumer.SetDebugPrinter(DebugPrinter{})
	if n.idleTimeout > 0 {
		n.consumer.SetIdleTimeout(n.idleTimeout)
	}
	if n.minRetryDelay > 0 {
		n.consumer.SetMinRetryDelay(n.minRetryDelay)
	}
	if n.maxRetryDelay > 0 {
		n.consumer.SetMaxRetryDelay(n.maxRetryDelay)
	}
	if n.maxRetryCount > 0 {
		n.consumer.SetMaxRetryCount(n.maxRetryCount)
	}
	n.messages, n.errs = n.consumer.Firehose(n.subscriptionID, "")
}

func (n *FirehoseNozzle) parseEnvelopes() error {
	for {
		select {
		case envelope := <-n.messages:
			n.handleMessage(envelope)
			n.metricsStore.AddMetric(envelope)
		case err := <-n.errs:
			retryError := n.handleError(err)
			if !retryError {
				return err
			}
		}
	}
}

func (n *FirehoseNozzle) handleMessage(envelope *events.Envelope) {
	if envelope.GetEventType() == events.Envelope_CounterEvent && envelope.CounterEvent.GetName() == "TruncatingBuffer.DroppedMessages" && envelope.GetOrigin() == "doppler" {
		log.Infof("We've intercepted an upstream message which indicates that the Nozzle or the TrafficController is not keeping up. Please try scaling up the Nozzle.")
		n.metricsStore.AlertSlowConsumerError()
	}
}

func (n *FirehoseNozzle) handleError(err error) bool {
	log.Errorf("Error while reading from the Firehose: %v", err)

	switch err.(type) {
	case noaerrors.RetryError:
		switch noaRetryError := err.(noaerrors.RetryError).Err.(type) {
		case *websocket.CloseError:
			switch noaRetryError.Code {
			case websocket.ClosePolicyViolation:
				log.Errorf("Nozzle couldn't keep up. Please try scaling up the Nozzle.")
				n.metricsStore.AlertSlowConsumerError()
			}
		}
		return true
	}

	log.Info("Closing connection with Firehose...")
	n.consumer.Close()

	return false
}

type DebugPrinter struct{}

func (dp DebugPrinter) Print(title, dump string) {
	log.Debugf("%s: %s", title, dump)
}
<|endoftext|>


<commit_before>package opt

import "encoding/json"

type FiltersOption struct {
	filters string
}

func Filters(filters string) FiltersOption {
	return FiltersOption{filters}
}

func ExtractFilters(opts ...interface{}) string {
	for _, opt := range opts {
		v, ok := opt.(FiltersOption)
		if ok {
			return v.filters
		}
	}
	return ""
}

func (o FiltersOption) Get() string {
	return o.filters
}

func (o FiltersOption) MarshalJSON() ([]byte, error) {
	return json.Marshal(o.filters)
}

func (o *FiltersOption) UnmarshalJSON(data []byte) error {
	return json.Unmarshal(data, &o.filters)
}
<commit_msg>changed: remove duplicated code<commit_after>package opt

import "encoding/json"

type FiltersOption struct {
	filters string
}

func Filters(filters string) FiltersOption {
	return FiltersOption{filters}
}

func (o FiltersOption) Get() string {
	return o.filters
}

func (o FiltersOption) MarshalJSON() ([]byte, error) {
	return json.Marshal(o.filters)
}

func (o *FiltersOption) UnmarshalJSON(data []byte) error {
	return json.Unmarshal(data, &o.filters)
}
<|endoftext|>


<commit_before>package service

import (
	"mime/multipart"

	"github.com/guregu/null"

	"github.com/torinos-io/api/type/model"
)

// UploadRequest holds uploaded files
type UploadRequest struct {
	CartfileContent    *multipart.FileHeader `json:"cartfile_content"`
	PodfileLockContent *multipart.FileHeader `json:"podfile_content"`
	RepositoryName     string                `json:"repository_name"`
}

// Upload uploads files to Analyze service
func (s *service) Upload(userID null.Int, req *UploadRequest) (*model.Project, error) {

	files := &model.UploadedData{
		CartfileContent:    req.CartfileContent,
		PodfileLockContent: req.PodfileLockContent,
		RepositoryName:     req.RepositoryName,
	}

	return s.ProjectStore.Upload(userID, files)
}
<commit_msg>Remove empty line<commit_after>package service

import (
	"mime/multipart"

	"github.com/guregu/null"

	"github.com/torinos-io/api/type/model"
)

// UploadRequest holds uploaded files
type UploadRequest struct {
	CartfileContent    *multipart.FileHeader `json:"cartfile_content"`
	PodfileLockContent *multipart.FileHeader `json:"podfile_content"`
	RepositoryName     string                `json:"repository_name"`
}

// Upload uploads files to Analyze service
func (s *service) Upload(userID null.Int, req *UploadRequest) (*model.Project, error) {
	files := &model.UploadedData{
		CartfileContent:    req.CartfileContent,
		PodfileLockContent: req.PodfileLockContent,
		RepositoryName:     req.RepositoryName,
	}

	return s.ProjectStore.Upload(userID, files)
}
<|endoftext|>