<commit_before>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"io"
	"regexp"
	"strings"
	"flag"
	"time"
	"compress/gzip"
)

// Time layouts must use the reference time `Mon Jan 2 15:04:05 MST 2006` to
// convey the pattern with which to format/parse a given time/string
const TIME_LAYOUT string = "[2006-01-02 15:04:05 MST]"
const VERSION = "v2.1 - Bizarro"

var job_regexp       *regexp.Regexp = regexp.MustCompile("^P[0-9]+(DJ|PW)[0-9]*")
var timestamp_regexp *regexp.Regexp = regexp.MustCompile("^(\\[[0-9-]+ [0-9:]+ UTC\\])")
var request_regexp   *regexp.Regexp = regexp.MustCompile("\\] (P[0-9]+[A-Za-z]+[0-9]+) ")
var sql_regexp       *regexp.Regexp = regexp.MustCompile("(SQL \\()|(EXEC sp_executesql N)|( CACHE \\()")
var ntlm_regexp      *regexp.Regexp = regexp.MustCompile(" \\(NTLM\\) ")
var debug_regexp     *regexp.Regexp = regexp.MustCompile(" DEBUG ")

func main() {
	hide_jobs_flag  := flag.Bool("hide_jobs", false, "Hide background jobs")
	hide_sql_flag   := flag.Bool("hide_sql", false, "Hide SQL statements")
	hide_ntlm_flag  := flag.Bool("hide_ntlm", false, "Hide NTLM lines")
	hide_debug_flag := flag.Bool("hide_debug", false, "Hide DEBUG lines")
	full_flag       := flag.Bool("full", false, "Show the full request/job for each found line")
	neat_flag       := flag.Bool("neat", false, "Hide clutter - equivalent to -hide_jobs -hide_sql -hide_ntlm")
	after_str       := flag.String("after", "", "Show logs after this time (YYYY-MM-DD HH:II::SS")
	find_str        := flag.String("find", "", "Find lines matching this regexp")
	hide_str        := flag.String("hide", "", "Hide lines matching this regexp")

	flag.Parse()
	args := flag.Args()

	time_after, e    := time.Parse(TIME_LAYOUT, fmt.Sprintf("[%s UTC]", *after_str))
	parse_time       := false

	if e != nil {
		if len(*after_str) > 0 {
			msg(fmt.Sprintf("Invalid time format \"%s\" - Must be YYYY-MM-DD HH::II::SS", *after_str))
			usage()
			os.Exit(2)
		}
	} else {
		parse_time = true
	}

	if len(args) < 1 {
		usage()
		os.Exit(2)
	}

	if *neat_flag {
		*hide_jobs_flag = true
		*hide_sql_flag = true
		*hide_ntlm_flag = true
	}

	msg(fmt.Sprintf("Show full requests/jobs: %t", *full_flag))
	msg(fmt.Sprintf("Show background job lines: %t", !*hide_jobs_flag))
	msg(fmt.Sprintf("Show SQL lines: %t", !*hide_sql_flag))
	msg(fmt.Sprintf("Show NTLM lines: %t", !*hide_ntlm_flag))
	msg(fmt.Sprintf("Show DEBUG lines: %t", !*hide_debug_flag))
	msg(fmt.Sprintf("Show lines after: %s", *after_str))

	filename := args[0]
	msg(fmt.Sprintf("Opening file: %s", filename))

	file := openFile(filename)
	defer file.Close()

	is_gzip      := isGzip(filename)
	file_size    := float64(fileSize(file))
	show_percent := !is_gzip
	var read_size int64 = 0

	var reader io.Reader = file
	var unique_map map[string]bool;

	find_regexp, err := regexp.Compile(*find_str)
	has_find := len(*find_str) > 0 && err == nil

	hide_regexp, err := regexp.Compile(*hide_str)
	has_hide := len(*hide_str) > 0 && err == nil

	if *full_flag && has_find {
		if is_gzip {
			// for some reason if you create a reader but don't use it,
			// an error is given when the output reader is created below
			parse_gz_reader := getGzipReader(file)
			defer parse_gz_reader.Close()

			reader = parse_gz_reader
		}

		line_count  := 0
		line_after  := !parse_time // if not parsing time, then all lines are valid
		request_ids := make([]string, 0)

		scanner := bufio.NewScanner(reader);

		for scanner.Scan() {
			line := scanner.Text();
			if find_regexp.MatchString(line) {

				if !line_after {
					if timestamp := extractTimestamp(line); len(timestamp) > 1 {
						if isAfterTime(timestamp, &time_after) {
							line_after = true
						}
					}
				}

				if line_after {
					if request_id := extractRequestId(line); len(request_id) > 1 {
						if !*hide_jobs_flag || !isJob(request_id) {
							request_ids = append(request_ids, request_id)
						}
					}
				}
			}

			read_size += int64(len(line))

			if line_count++; line_count % 20000 == 0 {
				if show_percent {
					showPercent(line_count, float64(read_size) / file_size, line_after, len(request_ids))
				} else {
					showBytes(line_count, float64(read_size), line_after, len(request_ids))
				}
			}
		}

		file_size = float64(read_size) // set the filesize to the total known size
		msg("") // empty line

		if err := scanner.Err(); err != nil {
			log.Fatal(err)
		}

		msg(fmt.Sprintf("Found %d lines matching \"%s\"", len(request_ids), *find_str))
		unique_map = generateRequestIdMap(&request_ids)

		if len(unique_map) < 1 {
			msg(fmt.Sprintf("Found 0 request identifiers for \"%s\"", *find_str))
			os.Exit(2)
		}

		rewindFile(file)
	} else {
		msg("Not printing -full requests, skipping request collection phase")
	}

	if is_gzip {
		output_gz_reader := getGzipReader(file)
		defer output_gz_reader.Close()

		reader = output_gz_reader
	}

	show_percent = read_size > int64(0)
	read_size = 0

	line_count := 0
	line_after := !parse_time // if not parsing time, then all lines are valid
	has_requests := len(unique_map) > 0

	output_scanner := bufio.NewScanner(reader);

	for output_scanner.Scan() {
		line := output_scanner.Text();

		output := false

		if !line_after {
			read_size += int64(len(line))

			if line_count++; line_count % 5000 == 0 {
				if show_percent {
					fmt.Fprintf(os.Stderr, "Reading: %.2f%%\r", (float64(read_size) / file_size) * 100)
				} else {
					fmt.Fprintf(os.Stderr, "Reading: %d lines, %0.3f GB\r", line_count, float64(read_size)/1024/1024/1024)
				}
			}

			if timestamp := extractTimestamp(line); len(timestamp) > 1 {
				if isAfterTime(timestamp, &time_after) {
					msg("\n") // empty line
					line_after = true
				}
			}
		}

		if line_after {
			request_id := extractRequestId(line)

			if has_requests {
				if len(request_id) > 0 && unique_map[request_id] {
					if *hide_jobs_flag && isJob(request_id) {
						output = false
					} else {
						output = true
					}
				}
			} else if has_find {
				output = find_regexp.MatchString(line)
			} else {
				output = true
			}
		}

		if output {
			if *hide_sql_flag && sql_regexp.MatchString(line) {
				output = false
			} else if *hide_ntlm_flag && ntlm_regexp.MatchString(line) {
				output = false
			} else if *hide_debug_flag && debug_regexp.MatchString(line) {
				output = false
			} else if has_hide && hide_regexp.MatchString(line) {
				output = false
			}
		}

		if output {
			fmt.Println(line)
		}
	}

	if err := output_scanner.Err(); err != nil {
		log.Fatal(err)
	}
}

func usage() {
	msg("AppVolumes Manager Log Tool - " + VERSION)
	msg("This tool can be used to extract the logs for specific requests from an AppVolumes Manager log")
	msg("")
	msg("Example:avmlog -after=\"2015-10-19 09:00:00\" -find \"apvuser2599\" -full -neat ~/Documents/scale.log.gz")
	msg("")
	flag.PrintDefaults()
	msg("")
}

func isAfterTime(timestamp string, time_after *time.Time) bool {
	if line_time, e := time.Parse(TIME_LAYOUT, timestamp); e != nil {
		msg(fmt.Sprintf("Got error %s", e))
		return false
	} else if line_time.Before(*time_after) {
		return false
	}

	return true
}

func isJob(request_id string) bool {
	return job_regexp.MatchString(request_id)
}

func extractTimestamp(line string) string {
	if timestamp_match := timestamp_regexp.FindStringSubmatch(line); len(timestamp_match) > 1 {
		return timestamp_match[1]
	} else {
		return ""
	}
}

func extractRequestId(line string) string {
	if request_match := request_regexp.FindStringSubmatch(line); len(request_match) > 1 {
		return request_match[1]
	} else {
		return ""
	}
}

func generateRequestIdMap(request_ids *[]string) map[string]bool {
	unique_map := make(map[string]bool, len(*request_ids))

	for _, x := range *request_ids {
		unique_map[x] = true
	}

	for k, _ := range unique_map {
		msg(fmt.Sprintf("Request ID: %s", k))
	}

	return unique_map
}

func openFile(filename string) *os.File {
	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	return file
}

func fileSize(file *os.File) int64 {
	if fi, err := file.Stat(); err != nil {
		msg("Unable to determine file size")

		return 1;
	} else {
		msg(fmt.Sprintf("The file is %d bytes long", fi.Size()))

		return fi.Size();
	}
}

func isGzip(filename string) bool {
	return strings.HasSuffix(filename, ".gz")
}

func getGzipReader(file *os.File) *gzip.Reader {
	gz_reader, err := gzip.NewReader(file)
	if err != nil {
	log.Fatal(err)
	}

	return gz_reader
}

func rewindFile(file *os.File) {
	file.Seek(0, 0)  // go back to the top (rewind)
}

func msg(output string) {
	fmt.Fprintln(os.Stderr, output)
}

func showPercent(line_count int, position float64, after bool, matches int) {
	fmt.Fprintf(
		os.Stderr,
		"Reading: %d lines, %.2f%% (after: %v, matches: %d)\r",
		line_count,
		position * 100,
		after,
		matches)
}

func showBytes(line_count int, position float64, after bool, matches int) {
	fmt.Fprintf(
		os.Stderr,
		"Reading: %d lines, %0.3f GB (after: %v, matches: %d)\r",
		line_count,
		position / 1024 / 1024 / 1024,
		after,
		matches)
}<commit_msg>When -after is used with -full, remember the line count at which line_after became true during the first pass so it can be used on the second pass.<commit_after>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"io"
	"regexp"
	"strings"
	"flag"
	"time"
	"compress/gzip"
)

// Time layouts must use the reference time `Mon Jan 2 15:04:05 MST 2006` to
// convey the pattern with which to format/parse a given time/string
const TIME_LAYOUT string = "[2006-01-02 15:04:05 MST]"
const VERSION = "v2.1 - Bizarro"

var job_regexp       *regexp.Regexp = regexp.MustCompile("^P[0-9]+(DJ|PW)[0-9]*")
var timestamp_regexp *regexp.Regexp = regexp.MustCompile("^(\\[[0-9-]+ [0-9:]+ UTC\\])")
var request_regexp   *regexp.Regexp = regexp.MustCompile("\\] (P[0-9]+[A-Za-z]+[0-9]+) ")
var sql_regexp       *regexp.Regexp = regexp.MustCompile("(SQL \\()|(EXEC sp_executesql N)|( CACHE \\()")
var ntlm_regexp      *regexp.Regexp = regexp.MustCompile(" \\(NTLM\\) ")
var debug_regexp     *regexp.Regexp = regexp.MustCompile(" DEBUG ")

func main() {
	hide_jobs_flag  := flag.Bool("hide_jobs", false, "Hide background jobs")
	hide_sql_flag   := flag.Bool("hide_sql", false, "Hide SQL statements")
	hide_ntlm_flag  := flag.Bool("hide_ntlm", false, "Hide NTLM lines")
	hide_debug_flag := flag.Bool("hide_debug", false, "Hide DEBUG lines")
	full_flag       := flag.Bool("full", false, "Show the full request/job for each found line")
	neat_flag       := flag.Bool("neat", false, "Hide clutter - equivalent to -hide_jobs -hide_sql -hide_ntlm")
	after_str       := flag.String("after", "", "Show logs after this time (YYYY-MM-DD HH:II::SS")
	find_str        := flag.String("find", "", "Find lines matching this regexp")
	hide_str        := flag.String("hide", "", "Hide lines matching this regexp")

	flag.Parse()
	args := flag.Args()

	time_after, err := time.Parse(TIME_LAYOUT, fmt.Sprintf("[%s UTC]", *after_str))
	parse_time      := false
	after_count     := 0

	if err != nil {
		if len(*after_str) > 0 {
			msg(fmt.Sprintf("Invalid time format \"%s\" - Must be YYYY-MM-DD HH::II::SS", *after_str))
			usage()
			os.Exit(2)
		}
	} else {
		parse_time = true
	}

	if len(args) < 1 {
		usage()
		os.Exit(2)
	}

	if *neat_flag {
		*hide_jobs_flag = true
		*hide_sql_flag = true
		*hide_ntlm_flag = true
	}

	msg(fmt.Sprintf("Show full requests/jobs: %t", *full_flag))
	msg(fmt.Sprintf("Show background job lines: %t", !*hide_jobs_flag))
	msg(fmt.Sprintf("Show SQL lines: %t", !*hide_sql_flag))
	msg(fmt.Sprintf("Show NTLM lines: %t", !*hide_ntlm_flag))
	msg(fmt.Sprintf("Show DEBUG lines: %t", !*hide_debug_flag))
	msg(fmt.Sprintf("Show lines after: %s", *after_str))

	filename := args[0]
	msg(fmt.Sprintf("Opening file: %s", filename))

	file := openFile(filename)
	defer file.Close()

	is_gzip      := isGzip(filename)
	file_size    := float64(fileSize(file))
	show_percent := !is_gzip
	var read_size int64 = 0

	var reader io.Reader = file
	var unique_map map[string]bool;

	find_regexp, err := regexp.Compile(*find_str)
	has_find := len(*find_str) > 0 && err == nil

	hide_regexp, err := regexp.Compile(*hide_str)
	has_hide := len(*hide_str) > 0 && err == nil

	if *full_flag && has_find {
		if is_gzip {
			// for some reason if you create a reader but don't use it,
			// an error is given when the output reader is created below
			parse_gz_reader := getGzipReader(file)
			defer parse_gz_reader.Close()

			reader = parse_gz_reader
		}

		line_count  := 0
		line_after  := !parse_time // if not parsing time, then all lines are valid
		request_ids := make([]string, 0)

		scanner := bufio.NewScanner(reader);

		for scanner.Scan() {
			line := scanner.Text();
			if find_regexp.MatchString(line) {

				if !line_after {
					if timestamp := extractTimestamp(line); len(timestamp) > 1 {
						if isAfterTime(timestamp, &time_after) {
							line_after = true
							after_count = line_count
						}
					}
				}

				if line_after {
					if request_id := extractRequestId(line); len(request_id) > 1 {
						if !*hide_jobs_flag || !isJob(request_id) {
							request_ids = append(request_ids, request_id)
						}
					}
				}
			}

			read_size += int64(len(line))

			if line_count++; line_count % 20000 == 0 {
				if show_percent {
					showPercent(line_count, float64(read_size) / file_size, line_after, len(request_ids))
				} else {
					showBytes(line_count, float64(read_size), line_after, len(request_ids))
				}
			}
		}

		file_size = float64(read_size) // set the filesize to the total known size
		msg("") // empty line

		if err := scanner.Err(); err != nil {
			log.Fatal(err)
		}

		msg(fmt.Sprintf("Found %d lines matching \"%s\"", len(request_ids), *find_str))
		unique_map = generateRequestIdMap(&request_ids)

		if len(unique_map) < 1 {
			msg(fmt.Sprintf("Found 0 request identifiers for \"%s\"", *find_str))
			os.Exit(2)
		}

		rewindFile(file)
	} else {
		msg("Not printing -full requests, skipping request collection phase")
	}

	if is_gzip {
		output_gz_reader := getGzipReader(file)
		defer output_gz_reader.Close()

		reader = output_gz_reader
	}

	show_percent = read_size > int64(0)
	read_size = 0

	line_count := 0
	line_after := !parse_time // if not parsing time, then all lines are valid
	has_requests := len(unique_map) > 0

	output_scanner := bufio.NewScanner(reader);

	for output_scanner.Scan() {
		line := output_scanner.Text();

		output := false

		if !line_after {
			read_size += int64(len(line))

			if line_count++; line_count % 5000 == 0 {
				if show_percent {
					fmt.Fprintf(os.Stderr, "Reading: %.2f%%\r", (float64(read_size) / file_size) * 100)
				} else {
					fmt.Fprintf(os.Stderr, "Reading: %d lines, %0.3f GB\r", line_count, float64(read_size)/1024/1024/1024)
				}
			}

			if after_count < line_count {
				if timestamp := extractTimestamp(line); len(timestamp) > 1 {
					if isAfterTime(timestamp, &time_after) {
						msg("\n") // empty line
						line_after = true
					}
				}
			}
		}

		if line_after {
			request_id := extractRequestId(line)

			if has_requests {
				if len(request_id) > 0 && unique_map[request_id] {
					if *hide_jobs_flag && isJob(request_id) {
						output = false
					} else {
						output = true
					}
				}
			} else if has_find {
				output = find_regexp.MatchString(line)
			} else {
				output = true
			}
		}

		if output {
			if *hide_sql_flag && sql_regexp.MatchString(line) {
				output = false
			} else if *hide_ntlm_flag && ntlm_regexp.MatchString(line) {
				output = false
			} else if *hide_debug_flag && debug_regexp.MatchString(line) {
				output = false
			} else if has_hide && hide_regexp.MatchString(line) {
				output = false
			}
		}

		if output {
			fmt.Println(line)
		}
	}

	if err := output_scanner.Err(); err != nil {
		log.Fatal(err)
	}
}

func usage() {
	msg("AppVolumes Manager Log Tool - " + VERSION)
	msg("This tool can be used to extract the logs for specific requests from an AppVolumes Manager log")
	msg("")
	msg("Example:avmlog -after=\"2015-10-19 09:00:00\" -find \"apvuser2599\" -full -neat ~/Documents/scale.log.gz")
	msg("")
	flag.PrintDefaults()
	msg("")
}

func isAfterTime(timestamp string, time_after *time.Time) bool {
	if line_time, e := time.Parse(TIME_LAYOUT, timestamp); e != nil {
		msg(fmt.Sprintf("Got error %s", e))
		return false
	} else if line_time.Before(*time_after) {
		return false
	}

	return true
}

func isJob(request_id string) bool {
	return job_regexp.MatchString(request_id)
}

func extractTimestamp(line string) string {
	if timestamp_match := timestamp_regexp.FindStringSubmatch(line); len(timestamp_match) > 1 {
		return timestamp_match[1]
	} else {
		return ""
	}
}

func extractRequestId(line string) string {
	if request_match := request_regexp.FindStringSubmatch(line); len(request_match) > 1 {
		return request_match[1]
	} else {
		return ""
	}
}

func generateRequestIdMap(request_ids *[]string) map[string]bool {
	unique_map := make(map[string]bool, len(*request_ids))

	for _, x := range *request_ids {
		unique_map[x] = true
	}

	for k, _ := range unique_map {
		msg(fmt.Sprintf("Request ID: %s", k))
	}

	return unique_map
}

func openFile(filename string) *os.File {
	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	return file
}

func fileSize(file *os.File) int64 {
	if fi, err := file.Stat(); err != nil {
		msg("Unable to determine file size")

		return 1;
	} else {
		msg(fmt.Sprintf("The file is %d bytes long", fi.Size()))

		return fi.Size();
	}
}

func isGzip(filename string) bool {
	return strings.HasSuffix(filename, ".gz")
}

func getGzipReader(file *os.File) *gzip.Reader {
	gz_reader, err := gzip.NewReader(file)
	if err != nil {
	log.Fatal(err)
	}

	return gz_reader
}

func rewindFile(file *os.File) {
	file.Seek(0, 0)  // go back to the top (rewind)
}

func msg(output string) {
	fmt.Fprintln(os.Stderr, output)
}

func showPercent(line_count int, position float64, after bool, matches int) {
	fmt.Fprintf(
		os.Stderr,
		"Reading: %d lines, %.2f%% (after: %v, matches: %d)\r",
		line_count,
		position * 100,
		after,
		matches)
}

func showBytes(line_count int, position float64, after bool, matches int) {
	fmt.Fprintf(
		os.Stderr,
		"Reading: %d lines, %0.3f GB (after: %v, matches: %d)\r",
		line_count,
		position / 1024 / 1024 / 1024,
		after,
		matches)
}<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"regexp"
	"strings"
	"flag"
	"time"
)

func main() {
	job_flag := flag.Int("jobs", 0, "Show background jobs")
	sql_flag := flag.Int("sql", 0, "Show SQL statements")
	after_str := flag.String("after", "", "Show logs after this time (YYYY-MM-DD HH:II::SS")
	match_str := flag.String("match", "", "Regexp for requests to gather")

	flag.Parse()
	args := flag.Args()

	// Time layouts must use the
	// reference time `Mon Jan 2 15:04:05 MST 2006` to show the
	// pattern with which to format/parse a given time/string
	time_layout      := "[2006-01-02 15:04:05 MST]"
	time_after, e    := time.Parse(time_layout, fmt.Sprintf("[%s UTC]", *after_str))
	parse_time       := false

	if e != nil {
		if len(*after_str) > 0 {
			fmt.Println(fmt.Sprintf("Invalid time format \"%s\" - Must be YYYY-MM-DD HH::II::SS", *after_str))
			os.Exit(4)
		}
	} else {
		parse_time = true
	}

	if len(args) < 1 {
		usage()
		os.Exit(1)
	}

	fmt.Println(fmt.Sprintf("Show background jobs: %d", *job_flag))
	fmt.Println(fmt.Sprintf("Show SQL: %d", *sql_flag))
	fmt.Println(fmt.Sprintf("After: %s", *after_str))

	filename := args[0]
	fmt.Println(fmt.Sprintf("Opening file: %s", filename))

	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	line_count       := 0
	request_ids      := make([]string, 0)
	line_strexp      := *match_str
	unique_strexp    := ""

	timestamp_regexp := regexp.MustCompile("^(\\[.*?\\])")
	sql_regexp       := regexp.MustCompile("(SQL \\()|(EXEC sp_executesql N)|( CACHE \\()")
	nltm_regexp      := regexp.MustCompile(" \\(NTLM\\) ")
	target_regexp    := regexp.MustCompile("\\] (P[0-9]+[A-Za-z]+[0-9]+) ")

	scanner := bufio.NewScanner(file)

	if line_regexp, err := regexp.Compile(line_strexp); len(line_strexp) > 0 && err == nil {
		for scanner.Scan() {
			line := scanner.Text();
			if line_regexp.MatchString(line) {
				request := target_regexp.FindStringSubmatch(line)
				timestamp := timestamp_regexp.FindStringSubmatch(line)

				input := true

				if len(request) < 2 {
					input = false
				} else if parse_time && len(timestamp) > 1 {
					line_time, e := time.Parse(time_layout, timestamp[1])

					if e != nil {
						fmt.Println("Got error %s", e)
						input = false
					} else if line_time.Before(time_after) {
						input = false
					}
				}

				if input {
					is_job := strings.Contains(request[1], "DJ")

					if is_job {
						if *job_flag > 0 {
							request_ids = append(request_ids, request[1])
						} else {
							fmt.Println(line)
						}
					} else {
						request_ids = append(request_ids, request[1])
					}
				}
			}

			line_count++

			if line_count % 10000 == 0 {
				fmt.Print(".")
			}
		}

		if err := scanner.Err(); err != nil {
			log.Fatal(err)
		}

		fmt.Println("Generating unique request identifiers", len(request_ids))
		unique_set := make(map[string]bool, len(request_ids))

		for _, x := range request_ids {
			unique_set[x] = true
		}

		unique_ids := make([]string, 0, len(unique_set))

		for x := range unique_set {
			if len(x) > 0 {
				unique_ids = append(unique_ids, x)
			}
		}

		for i := 0; i < len(unique_ids); i++ {
			fmt.Println(fmt.Sprintf("Request ID: %s", unique_ids[i]))
		}

		unique_strexp = strings.Join(unique_ids, "|")
		fmt.Println(unique_strexp)

		if len(unique_strexp) < 1 {
			fmt.Println(fmt.Sprintf("Found 0 AVM Request IDs for %s", line_strexp))
			os.Exit(2)
		}
	}

	file.Seek(0, 0)

	output_match   := len(unique_strexp) > 0
	output_regexp  := regexp.MustCompile(unique_strexp)
	output_scanner := bufio.NewScanner(file)

	for output_scanner.Scan() {
		line := output_scanner.Text();

		output := true

		if output_match && !output_regexp.MatchString(line) {
			output = false
		} else if *sql_flag < 1 && sql_regexp.MatchString(line) {
			output = false
		} else if nltm_regexp.MatchString(line) {
			output = false
		}

		if output {
			fmt.Println(line)
		}
	}

	if err := output_scanner.Err(); err != nil {
		log.Fatal(err)
	}
}

func usage() {
	fmt.Println(fmt.Sprintf("Usage: avmlog -match=\"regexp\" -jobs=0|1 -sql=0|1 -after=\"YYYY-MM-DD HH:II::SS\" avmanager_filename.log"))
	fmt.Println("Example: avm -match=\"username|computername\" \"/path/to/manager/log/production.log\"")
}
<commit_msg>Don't parse timestamps during the request collection phase if -after is not specified<commit_after>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"regexp"
	"strings"
	"flag"
	"time"
)

func main() {
	job_flag := flag.Int("jobs", 0, "Show background jobs")
	sql_flag := flag.Int("sql", 0, "Show SQL statements")
	after_str := flag.String("after", "", "Show logs after this time (YYYY-MM-DD HH:II::SS")
	match_str := flag.String("match", "", "Regexp for requests to gather")

	flag.Parse()
	args := flag.Args()

	// Time layouts must use the
	// reference time `Mon Jan 2 15:04:05 MST 2006` to show the
	// pattern with which to format/parse a given time/string
	time_layout      := "[2006-01-02 15:04:05 MST]"
	time_after, e    := time.Parse(time_layout, fmt.Sprintf("[%s UTC]", *after_str))
	parse_time       := false

	if e != nil {
		if len(*after_str) > 0 {
			fmt.Println(fmt.Sprintf("Invalid time format \"%s\" - Must be YYYY-MM-DD HH::II::SS", *after_str))
			os.Exit(4)
		}
	} else {
		parse_time = true
	}

	if len(args) < 1 {
		usage()
		os.Exit(1)
	}

	fmt.Println(fmt.Sprintf("Show background jobs: %d", *job_flag))
	fmt.Println(fmt.Sprintf("Show SQL: %d", *sql_flag))
	fmt.Println(fmt.Sprintf("After: %s", *after_str))

	filename := args[0]
	fmt.Println(fmt.Sprintf("Opening file: %s", filename))

	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	line_count       := 0
	request_ids      := make([]string, 0)
	line_strexp      := *match_str
	unique_strexp    := ""

	timestamp_regexp := regexp.MustCompile("^(\\[.*?\\])")
	sql_regexp       := regexp.MustCompile("(SQL \\()|(EXEC sp_executesql N)|( CACHE \\()")
	nltm_regexp      := regexp.MustCompile(" \\(NTLM\\) ")
	target_regexp    := regexp.MustCompile("\\] (P[0-9]+[A-Za-z]+[0-9]+) ")

	scanner := bufio.NewScanner(file)

	if line_regexp, err := regexp.Compile(line_strexp); len(line_strexp) > 0 && err == nil {
		for scanner.Scan() {
			line := scanner.Text();
			if line_regexp.MatchString(line) {
				request := target_regexp.FindStringSubmatch(line)

				input := true

				if len(request) < 2 {
					input = false
				} else if parse_time {
					if timestamp := timestamp_regexp.FindStringSubmatch(line); len(timestamp) > 1 {
						line_time, e := time.Parse(time_layout, timestamp[1])

						if e != nil {
							fmt.Println("Got error %s", e)
							input = false
						} else if line_time.Before(time_after) {
							input = false
						}
					}
				}

				if input {
					is_job := strings.Contains(request[1], "DJ")

					if is_job {
						if *job_flag > 0 {
							request_ids = append(request_ids, request[1])
						} else {
							fmt.Println(line)
						}
					} else {
						request_ids = append(request_ids, request[1])
					}
				}
			}

			line_count++

			if line_count % 10000 == 0 {
				fmt.Print(".")
			}
		}

		if err := scanner.Err(); err != nil {
			log.Fatal(err)
		}

		fmt.Println("Generating unique request identifiers", len(request_ids))
		unique_set := make(map[string]bool, len(request_ids))

		for _, x := range request_ids {
			unique_set[x] = true
		}

		unique_ids := make([]string, 0, len(unique_set))

		for x := range unique_set {
			if len(x) > 0 {
				unique_ids = append(unique_ids, x)
			}
		}

		for i := 0; i < len(unique_ids); i++ {
			fmt.Println(fmt.Sprintf("Request ID: %s", unique_ids[i]))
		}

		unique_strexp = strings.Join(unique_ids, "|")
		fmt.Println(unique_strexp)

		if len(unique_strexp) < 1 {
			fmt.Println(fmt.Sprintf("Found 0 AVM Request IDs for %s", line_strexp))
			os.Exit(2)
		}
	}

	file.Seek(0, 0)

	output_match   := len(unique_strexp) > 0
	output_regexp  := regexp.MustCompile(unique_strexp)
	output_scanner := bufio.NewScanner(file)

	for output_scanner.Scan() {
		line := output_scanner.Text();

		output := true

		if output_match && !output_regexp.MatchString(line) {
			output = false
		} else if *sql_flag < 1 && sql_regexp.MatchString(line) {
			output = false
		} else if nltm_regexp.MatchString(line) {
			output = false
		}

		if output {
			fmt.Println(line)
		}
	}

	if err := output_scanner.Err(); err != nil {
		log.Fatal(err)
	}
}

func usage() {
	fmt.Println(fmt.Sprintf("Usage: avmlog -match=\"regexp\" -jobs=0|1 -sql=0|1 -after=\"YYYY-MM-DD HH:II::SS\" avmanager_filename.log"))
	fmt.Println("Example: avm -match=\"username|computername\" \"/path/to/manager/log/production.log\"")
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"io"
	"regexp"
	"strings"
	"flag"
	"time"
	"compress/gzip"
)

// Time layouts must use the reference time `Mon Jan 2 15:04:05 MST 2006` to
// convey the pattern with which to format/parse a given time/string
const TIME_LAYOUT string = "[2006-01-02 15:04:05 MST]"
const VERSION = "v3.0 - Carnage"

var job_regexp       *regexp.Regexp = regexp.MustCompile("^P[0-9]+(DJ|PW)[0-9]*")
var timestamp_regexp *regexp.Regexp = regexp.MustCompile("^(\\[[0-9-]+ [0-9:]+ UTC\\])")
var request_regexp   *regexp.Regexp = regexp.MustCompile("\\][[:space:]]+(P[0-9]+[A-Za-z]+[0-9]+) ")
var sql_regexp       *regexp.Regexp = regexp.MustCompile("( SQL: | SQL \\()|(EXEC sp_executesql N)|( CACHE \\()")
var ntlm_regexp      *regexp.Regexp = regexp.MustCompile(" (\\(NTLM\\)|NTLM:) ")
var debug_regexp     *regexp.Regexp = regexp.MustCompile(" DEBUG ")

func main() {
	hide_jobs_flag  := flag.Bool("hide_jobs", false, "Hide background jobs")
	hide_sql_flag   := flag.Bool("hide_sql", false, "Hide SQL statements")
	hide_ntlm_flag  := flag.Bool("hide_ntlm", false, "Hide NTLM lines")
	hide_debug_flag := flag.Bool("hide_debug", false, "Hide DEBUG lines")
	full_flag       := flag.Bool("full", false, "Show the full request/job for each found line")
	neat_flag       := flag.Bool("neat", false, "Hide clutter - equivalent to -hide_jobs -hide_sql -hide_ntlm")
	after_str       := flag.String("after", "", "Show logs after this time (YYYY-MM-DD HH:II::SS")
	find_str        := flag.String("find", "", "Find lines matching this regexp")
	hide_str        := flag.String("hide", "", "Hide lines matching this regexp")

	flag.Parse()
	args := flag.Args()

	time_after, err := time.Parse(TIME_LAYOUT, fmt.Sprintf("[%s UTC]", *after_str))
	parse_time      := false
	after_count     := 0

	if err != nil {
		if len(*after_str) > 0 {
			msg(fmt.Sprintf("Invalid time format \"%s\" - Must be YYYY-MM-DD HH::II::SS", *after_str))
			usage()
			os.Exit(2)
		}
	} else {
		parse_time = true
	}

	if len(args) < 1 {
		usage()
		os.Exit(2)
	}

	if *neat_flag {
		*hide_jobs_flag = true
		*hide_sql_flag = true
		*hide_ntlm_flag = true
	}

	msg(fmt.Sprintf("Show full requests/jobs: %t", *full_flag))
	msg(fmt.Sprintf("Show background job lines: %t", !*hide_jobs_flag))
	msg(fmt.Sprintf("Show SQL lines: %t", !*hide_sql_flag))
	msg(fmt.Sprintf("Show NTLM lines: %t", !*hide_ntlm_flag))
	msg(fmt.Sprintf("Show DEBUG lines: %t", !*hide_debug_flag))
	msg(fmt.Sprintf("Show lines after: %s", *after_str))

	filename := args[0]
	msg(fmt.Sprintf("Opening file: %s", filename))

	file := openFile(filename)
	defer file.Close()

	is_gzip      := isGzip(filename)
	file_size    := float64(fileSize(file))
	show_percent := !is_gzip
	var read_size int64 = 0

	var reader io.Reader = file
	var unique_map map[string]bool;

	find_regexp, err := regexp.Compile(*find_str)
	has_find := len(*find_str) > 0 && err == nil

	hide_regexp, err := regexp.Compile(*hide_str)
	has_hide := len(*hide_str) > 0 && err == nil

	if *full_flag && has_find {
		if is_gzip {
			// for some reason if you create a reader but don't use it,
			// an error is given when the output reader is created below
			parse_gz_reader := getGzipReader(file)
			defer parse_gz_reader.Close()

			reader = parse_gz_reader
		}

		line_count  := 0
		line_after  := !parse_time // if not parsing time, then all lines are valid
		request_ids := make([]string, 0)

		scanner := bufio.NewScanner(reader);

		for scanner.Scan() {
			line := scanner.Text();
			if find_regexp.MatchString(line) {

				if !line_after {
					if timestamp := extractTimestamp(line); len(timestamp) > 1 {
						if isAfterTime(timestamp, &time_after) {
							line_after = true
							after_count = line_count
						}
					}
				}

				if line_after {
					if request_id := extractRequestId(line); len(request_id) > 1 {
						if !*hide_jobs_flag || !isJob(request_id) {
							request_ids = append(request_ids, request_id)
						}
					}
				}
			}

			read_size += int64(len(line))

			if line_count++; line_count % 20000 == 0 {
				if show_percent {
					showPercent(line_count, float64(read_size) / file_size, line_after, len(request_ids))
				} else {
					showBytes(line_count, float64(read_size), line_after, len(request_ids))
				}
			}
		}

		file_size = float64(read_size) // set the filesize to the total known size
		msg("") // empty line

		if err := scanner.Err(); err != nil {
			log.Fatal(err)
		}

		msg(fmt.Sprintf("Found %d lines matching \"%s\"", len(request_ids), *find_str))
		unique_map = generateRequestIdMap(&request_ids)

		if len(unique_map) < 1 {
			msg(fmt.Sprintf("Found 0 request identifiers for \"%s\"", *find_str))
			os.Exit(2)
		}

		rewindFile(file)
	} else {
		msg("Not printing -full requests, skipping request collection phase")
	}

	if is_gzip {
		output_gz_reader := getGzipReader(file)
		defer output_gz_reader.Close()

		reader = output_gz_reader
	}

	show_percent = read_size > int64(0)
	read_size = 0

	line_count := 0
	line_after := !parse_time // if not parsing time, then all lines are valid
	has_requests := len(unique_map) > 0
	in_request := false

	output_scanner := bufio.NewScanner(reader);

	for output_scanner.Scan() {
		line := output_scanner.Text();

		output := false

		if !line_after {
			read_size += int64(len(line))

			if line_count++; line_count % 5000 == 0 {
				if show_percent {
					fmt.Fprintf(os.Stderr, "Reading: %.2f%%\r", (float64(read_size) / file_size) * 100)
				} else {
					fmt.Fprintf(os.Stderr, "Reading: %d lines, %0.3f GB\r", line_count, float64(read_size)/1024/1024/1024)
				}
			}

			if after_count < line_count {
				if timestamp := extractTimestamp(line); len(timestamp) > 1 {
					if isAfterTime(timestamp, &time_after) {
						msg("\n") // empty line
						line_after = true
					}
				}
			}
		}

		if line_after {
			request_id := extractRequestId(line)

			if has_requests {
				if len(request_id) > 0 {
					if unique_map[request_id] {
						if *hide_jobs_flag && isJob(request_id) {
							output = false
						} else {
							in_request = true
							output = true
						}
					} else {
						in_request = false
					}

				} else if len(request_id) < 1 && in_request {
					output = true
				}
			} else if has_find {
				output = find_regexp.MatchString(line)
			} else {
				output = true
			}
		}

		if output {
			if *hide_sql_flag && sql_regexp.MatchString(line) {
				output = false
			} else if *hide_ntlm_flag && ntlm_regexp.MatchString(line) {
				output = false
			} else if *hide_debug_flag && debug_regexp.MatchString(line) {
				output = false
			} else if has_hide && hide_regexp.MatchString(line) {
				output = false
			}
		}

		if output {
			fmt.Println(line)
		}
	}

	if err := output_scanner.Err(); err != nil {
		log.Fatal(err)
	}
}

func usage() {
	msg("AppVolumes Manager Log Tool - " + VERSION)
	msg("This tool can be used to extract the logs for specific requests from an AppVolumes Manager log")
	msg("")
	msg("Example:avmlog -after=\"2015-10-19 09:00:00\" -find \"apvuser2599\" -full -neat ~/Documents/scale.log.gz")
	msg("")
	flag.PrintDefaults()
	msg("")
}

func isAfterTime(timestamp string, time_after *time.Time) bool {
	if line_time, e := time.Parse(TIME_LAYOUT, timestamp); e != nil {
		msg(fmt.Sprintf("Got error %s", e))
		return false
	} else if line_time.Before(*time_after) {
		return false
	}

	return true
}

func isJob(request_id string) bool {
	return job_regexp.MatchString(request_id)
}

func extractTimestamp(line string) string {
	if timestamp_match := timestamp_regexp.FindStringSubmatch(line); len(timestamp_match) > 1 {
		return timestamp_match[1]
	} else {
		return ""
	}
}

func extractRequestId(line string) string {
	if request_match := request_regexp.FindStringSubmatch(line); len(request_match) > 1 {
		return request_match[1]
	} else {
		return ""
	}
}

func generateRequestIdMap(request_ids *[]string) map[string]bool {
	unique_map := make(map[string]bool, len(*request_ids))

	for _, x := range *request_ids {
		unique_map[x] = true
	}

	for k, _ := range unique_map {
		msg(fmt.Sprintf("Request ID: %s", k))
	}

	return unique_map
}

func openFile(filename string) *os.File {
	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	return file
}

func fileSize(file *os.File) int64 {
	if fi, err := file.Stat(); err != nil {
		msg("Unable to determine file size")

		return 1;
	} else {
		msg(fmt.Sprintf("The file is %d bytes long", fi.Size()))

		return fi.Size();
	}
}

func isGzip(filename string) bool {
	return strings.HasSuffix(filename, ".gz")
}

func getGzipReader(file *os.File) *gzip.Reader {
	gz_reader, err := gzip.NewReader(file)
	if err != nil {
	log.Fatal(err)
	}

	return gz_reader
}

func rewindFile(file *os.File) {
	file.Seek(0, 0)  // go back to the top (rewind)
}

func msg(output string) {
	fmt.Fprintln(os.Stderr, output)
}

func showPercent(line_count int, position float64, after bool, matches int) {
	fmt.Fprintf(
		os.Stderr,
		"Reading: %d lines, %.2f%% (after: %v, matches: %d)\r",
		line_count,
		position * 100,
		after,
		matches)
}

func showBytes(line_count int, position float64, after bool, matches int) {
	fmt.Fprintf(
		os.Stderr,
		"Reading: %d lines, %0.3f GB (after: %v, matches: %d)\r",
		line_count,
		position / 1024 / 1024 / 1024,
		after,
		matches)
}<commit_msg>Version bump to 3.0.1<commit_after>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"io"
	"regexp"
	"strings"
	"flag"
	"time"
	"compress/gzip"
)

// Time layouts must use the reference time `Mon Jan 2 15:04:05 MST 2006` to
// convey the pattern with which to format/parse a given time/string
const TIME_LAYOUT string = "[2006-01-02 15:04:05 MST]"
const VERSION = "v3.0.1 - Colossus"

var job_regexp       *regexp.Regexp = regexp.MustCompile("^P[0-9]+(DJ|PW)[0-9]*")
var timestamp_regexp *regexp.Regexp = regexp.MustCompile("^(\\[[0-9-]+ [0-9:]+ UTC\\])")
var request_regexp   *regexp.Regexp = regexp.MustCompile("\\][[:space:]]+(P[0-9]+[A-Za-z]+[0-9]+) ")
var sql_regexp       *regexp.Regexp = regexp.MustCompile("( SQL: | SQL \\()|(EXEC sp_executesql N)|( CACHE \\()")
var ntlm_regexp      *regexp.Regexp = regexp.MustCompile(" (\\(NTLM\\)|NTLM:) ")
var debug_regexp     *regexp.Regexp = regexp.MustCompile(" DEBUG ")

func main() {
	hide_jobs_flag  := flag.Bool("hide_jobs", false, "Hide background jobs")
	hide_sql_flag   := flag.Bool("hide_sql", false, "Hide SQL statements")
	hide_ntlm_flag  := flag.Bool("hide_ntlm", false, "Hide NTLM lines")
	hide_debug_flag := flag.Bool("hide_debug", false, "Hide DEBUG lines")
	full_flag       := flag.Bool("full", false, "Show the full request/job for each found line")
	neat_flag       := flag.Bool("neat", false, "Hide clutter - equivalent to -hide_jobs -hide_sql -hide_ntlm")
	after_str       := flag.String("after", "", "Show logs after this time (YYYY-MM-DD HH:II::SS")
	find_str        := flag.String("find", "", "Find lines matching this regexp")
	hide_str        := flag.String("hide", "", "Hide lines matching this regexp")

	flag.Parse()
	args := flag.Args()

	time_after, err := time.Parse(TIME_LAYOUT, fmt.Sprintf("[%s UTC]", *after_str))
	parse_time      := false
	after_count     := 0

	if err != nil {
		if len(*after_str) > 0 {
			msg(fmt.Sprintf("Invalid time format \"%s\" - Must be YYYY-MM-DD HH::II::SS", *after_str))
			usage()
			os.Exit(2)
		}
	} else {
		parse_time = true
	}

	if len(args) < 1 {
		usage()
		os.Exit(2)
	}

	if *neat_flag {
		*hide_jobs_flag = true
		*hide_sql_flag = true
		*hide_ntlm_flag = true
	}

	msg(fmt.Sprintf("Show full requests/jobs: %t", *full_flag))
	msg(fmt.Sprintf("Show background job lines: %t", !*hide_jobs_flag))
	msg(fmt.Sprintf("Show SQL lines: %t", !*hide_sql_flag))
	msg(fmt.Sprintf("Show NTLM lines: %t", !*hide_ntlm_flag))
	msg(fmt.Sprintf("Show DEBUG lines: %t", !*hide_debug_flag))
	msg(fmt.Sprintf("Show lines after: %s", *after_str))

	filename := args[0]
	msg(fmt.Sprintf("Opening file: %s", filename))

	file := openFile(filename)
	defer file.Close()

	is_gzip      := isGzip(filename)
	file_size    := float64(fileSize(file))
	show_percent := !is_gzip
	var read_size int64 = 0

	var reader io.Reader = file
	var unique_map map[string]bool;

	find_regexp, err := regexp.Compile(*find_str)
	has_find := len(*find_str) > 0 && err == nil

	hide_regexp, err := regexp.Compile(*hide_str)
	has_hide := len(*hide_str) > 0 && err == nil

	if *full_flag && has_find {
		if is_gzip {
			// for some reason if you create a reader but don't use it,
			// an error is given when the output reader is created below
			parse_gz_reader := getGzipReader(file)
			defer parse_gz_reader.Close()

			reader = parse_gz_reader
		}

		line_count  := 0
		line_after  := !parse_time // if not parsing time, then all lines are valid
		request_ids := make([]string, 0)

		scanner := bufio.NewScanner(reader);

		for scanner.Scan() {
			line := scanner.Text();
			if find_regexp.MatchString(line) {

				if !line_after {
					if timestamp := extractTimestamp(line); len(timestamp) > 1 {
						if isAfterTime(timestamp, &time_after) {
							line_after = true
							after_count = line_count
						}
					}
				}

				if line_after {
					if request_id := extractRequestId(line); len(request_id) > 1 {
						if !*hide_jobs_flag || !isJob(request_id) {
							request_ids = append(request_ids, request_id)
						}
					}
				}
			}

			read_size += int64(len(line))

			if line_count++; line_count % 20000 == 0 {
				if show_percent {
					showPercent(line_count, float64(read_size) / file_size, line_after, len(request_ids))
				} else {
					showBytes(line_count, float64(read_size), line_after, len(request_ids))
				}
			}
		}

		file_size = float64(read_size) // set the filesize to the total known size
		msg("") // empty line

		if err := scanner.Err(); err != nil {
			log.Fatal(err)
		}

		msg(fmt.Sprintf("Found %d lines matching \"%s\"", len(request_ids), *find_str))
		unique_map = generateRequestIdMap(&request_ids)

		if len(unique_map) < 1 {
			msg(fmt.Sprintf("Found 0 request identifiers for \"%s\"", *find_str))
			os.Exit(2)
		}

		rewindFile(file)
	} else {
		msg("Not printing -full requests, skipping request collection phase")
	}

	if is_gzip {
		output_gz_reader := getGzipReader(file)
		defer output_gz_reader.Close()

		reader = output_gz_reader
	}

	show_percent = read_size > int64(0)
	read_size = 0

	line_count := 0
	line_after := !parse_time // if not parsing time, then all lines are valid
	has_requests := len(unique_map) > 0
	in_request := false

	output_scanner := bufio.NewScanner(reader);

	for output_scanner.Scan() {
		line := output_scanner.Text();

		output := false

		if !line_after {
			read_size += int64(len(line))

			if line_count++; line_count % 5000 == 0 {
				if show_percent {
					fmt.Fprintf(os.Stderr, "Reading: %.2f%%\r", (float64(read_size) / file_size) * 100)
				} else {
					fmt.Fprintf(os.Stderr, "Reading: %d lines, %0.3f GB\r", line_count, float64(read_size)/1024/1024/1024)
				}
			}

			if after_count < line_count {
				if timestamp := extractTimestamp(line); len(timestamp) > 1 {
					if isAfterTime(timestamp, &time_after) {
						msg("\n") // empty line
						line_after = true
					}
				}
			}
		}

		if line_after {
			request_id := extractRequestId(line)

			if has_requests {
				if len(request_id) > 0 {
					if unique_map[request_id] {
						if *hide_jobs_flag && isJob(request_id) {
							output = false
						} else {
							in_request = true
							output = true
						}
					} else {
						in_request = false
					}

				} else if len(request_id) < 1 && in_request {
					output = true
				}
			} else if has_find {
				output = find_regexp.MatchString(line)
			} else {
				output = true
			}
		}

		if output {
			if *hide_sql_flag && sql_regexp.MatchString(line) {
				output = false
			} else if *hide_ntlm_flag && ntlm_regexp.MatchString(line) {
				output = false
			} else if *hide_debug_flag && debug_regexp.MatchString(line) {
				output = false
			} else if has_hide && hide_regexp.MatchString(line) {
				output = false
			}
		}

		if output {
			fmt.Println(line)
		}
	}

	if err := output_scanner.Err(); err != nil {
		log.Fatal(err)
	}
}

func usage() {
	msg("AppVolumes Manager Log Tool - " + VERSION)
	msg("This tool can be used to extract the logs for specific requests from an AppVolumes Manager log")
	msg("")
	msg("Example:avmlog -after=\"2015-10-19 09:00:00\" -find \"apvuser2599\" -full -neat ~/Documents/scale.log.gz")
	msg("")
	flag.PrintDefaults()
	msg("")
}

func isAfterTime(timestamp string, time_after *time.Time) bool {
	if line_time, e := time.Parse(TIME_LAYOUT, timestamp); e != nil {
		msg(fmt.Sprintf("Got error %s", e))
		return false
	} else if line_time.Before(*time_after) {
		return false
	}

	return true
}

func isJob(request_id string) bool {
	return job_regexp.MatchString(request_id)
}

func extractTimestamp(line string) string {
	if timestamp_match := timestamp_regexp.FindStringSubmatch(line); len(timestamp_match) > 1 {
		return timestamp_match[1]
	} else {
		return ""
	}
}

func extractRequestId(line string) string {
	if request_match := request_regexp.FindStringSubmatch(line); len(request_match) > 1 {
		return request_match[1]
	} else {
		return ""
	}
}

func generateRequestIdMap(request_ids *[]string) map[string]bool {
	unique_map := make(map[string]bool, len(*request_ids))

	for _, x := range *request_ids {
		unique_map[x] = true
	}

	for k, _ := range unique_map {
		msg(fmt.Sprintf("Request ID: %s", k))
	}

	return unique_map
}

func openFile(filename string) *os.File {
	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	return file
}

func fileSize(file *os.File) int64 {
	if fi, err := file.Stat(); err != nil {
		msg("Unable to determine file size")

		return 1;
	} else {
		msg(fmt.Sprintf("The file is %d bytes long", fi.Size()))

		return fi.Size();
	}
}

func isGzip(filename string) bool {
	return strings.HasSuffix(filename, ".gz")
}

func getGzipReader(file *os.File) *gzip.Reader {
	gz_reader, err := gzip.NewReader(file)
	if err != nil {
	log.Fatal(err)
	}

	return gz_reader
}

func rewindFile(file *os.File) {
	file.Seek(0, 0)  // go back to the top (rewind)
}

func msg(output string) {
	fmt.Fprintln(os.Stderr, output)
}

func showPercent(line_count int, position float64, after bool, matches int) {
	fmt.Fprintf(
		os.Stderr,
		"Reading: %d lines, %.2f%% (after: %v, matches: %d)\r",
		line_count,
		position * 100,
		after,
		matches)
}

func showBytes(line_count int, position float64, after bool, matches int) {
	fmt.Fprintf(
		os.Stderr,
		"Reading: %d lines, %0.3f GB (after: %v, matches: %d)\r",
		line_count,
		position / 1024 / 1024 / 1024,
		after,
		matches)
}<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"time"
)

func logDuration(m string, startTime time.Time) {
	log.Printf("Completed %v in %v", m, time.Since(startTime))
}

func backupTo(w io.Writer) (err error) {
	fch := make(chan *namedFile)
	ech := make(chan error)
	qch := make(chan bool)

	defer close(qch)

	defer logDuration("backup", time.Now())

	go pathGenerator("", fch, ech, qch)

	gz := gzip.NewWriter(w)
	defer func() {
		e := gz.Close()
		if err != nil {
			err = e
		}
	}()

	enc := json.NewEncoder(gz)

	for {
		select {
		case f, ok := <-fch:
			if !ok {
				return nil
			}
			log.Printf("backing up %v", f.name)
			err := enc.Encode(map[string]interface{}{
				"path": f.name,
				"meta": f.meta,
			})
			if err != nil {
				return err
			}
		case e, ok := <-ech:
			if ok {
				return e
			}
			ech = nil
		}
	}

	panic("unreachable")
}

func recordBackupObject(h string) error {
	f, err := os.Create(filepath.Join(*root, ".backup"))
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = fmt.Fprintf(f, "%v\n", h)
	return err
}

func backupToCBFS(fn string) error {
	f, err := NewHashRecord(*root, "")
	if err != nil {
		return err
	}
	defer f.Close()

	pr, pw := io.Pipe()

	go func() { pw.CloseWithError(backupTo(pw)) }()

	h, length, err := f.Process(pr)
	if err != nil {
		return err
	}

	err = recordBlobOwnership(h, length, true)
	if err != nil {
		return err
	}

	fm := fileMeta{
		OID:      h,
		Length:   length,
		Modified: time.Now().UTC(),
	}

	err = storeMeta(fn, fm, 1)
	if err != nil {
		return err
	}

	err = recordBackupObject(h)
	if err != nil {
		log.Printf("Failed to record backup OID: %v", err)
	}

	log.Printf("Replicating backup %v.", h)
	go increaseReplicaCount(h, length, globalConfig.MinReplicas-1)

	return nil
}

func doBackupDocs(w http.ResponseWriter, req *http.Request) {
	fn := req.FormValue("fn")
	if fn == "" {
		w.WriteHeader(400)
		return
	}

	if req.FormValue("bg") == "true" {
		go func() {
			err := backupToCBFS(fn)
			if err != nil {
				log.Printf("Error performing bg backup: %v", err)
			}
		}()
		w.WriteHeader(202)
		return
	}

	err := backupToCBFS(fn)
	if err != nil {
		w.WriteHeader(500)
		fmt.Fprintf(w, "Error performing backup: %v", err)
		return
	}

	w.WriteHeader(201)
}
<commit_msg>use strconv.ParseBool for bg param parsing.<commit_after>package main

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

func logDuration(m string, startTime time.Time) {
	log.Printf("Completed %v in %v", m, time.Since(startTime))
}

func backupTo(w io.Writer) (err error) {
	fch := make(chan *namedFile)
	ech := make(chan error)
	qch := make(chan bool)

	defer close(qch)

	defer logDuration("backup", time.Now())

	go pathGenerator("", fch, ech, qch)

	gz := gzip.NewWriter(w)
	defer func() {
		e := gz.Close()
		if err != nil {
			err = e
		}
	}()

	enc := json.NewEncoder(gz)

	for {
		select {
		case f, ok := <-fch:
			if !ok {
				return nil
			}
			log.Printf("backing up %v", f.name)
			err := enc.Encode(map[string]interface{}{
				"path": f.name,
				"meta": f.meta,
			})
			if err != nil {
				return err
			}
		case e, ok := <-ech:
			if ok {
				return e
			}
			ech = nil
		}
	}

	panic("unreachable")
}

func recordBackupObject(h string) error {
	f, err := os.Create(filepath.Join(*root, ".backup"))
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = fmt.Fprintf(f, "%v\n", h)
	return err
}

func backupToCBFS(fn string) error {
	f, err := NewHashRecord(*root, "")
	if err != nil {
		return err
	}
	defer f.Close()

	pr, pw := io.Pipe()

	go func() { pw.CloseWithError(backupTo(pw)) }()

	h, length, err := f.Process(pr)
	if err != nil {
		return err
	}

	err = recordBlobOwnership(h, length, true)
	if err != nil {
		return err
	}

	fm := fileMeta{
		OID:      h,
		Length:   length,
		Modified: time.Now().UTC(),
	}

	err = storeMeta(fn, fm, 1)
	if err != nil {
		return err
	}

	err = recordBackupObject(h)
	if err != nil {
		log.Printf("Failed to record backup OID: %v", err)
	}

	log.Printf("Replicating backup %v.", h)
	go increaseReplicaCount(h, length, globalConfig.MinReplicas-1)

	return nil
}

func doBackupDocs(w http.ResponseWriter, req *http.Request) {
	fn := req.FormValue("fn")
	if fn == "" {
		w.WriteHeader(400)
		return
	}

	if bg, _ := strconv.ParseBool(req.FormValue("bg")); bg {
		go func() {
			err := backupToCBFS(fn)
			if err != nil {
				log.Printf("Error performing bg backup: %v", err)
			}
		}()
		w.WriteHeader(202)
		return
	}

	err := backupToCBFS(fn)
	if err != nil {
		w.WriteHeader(500)
		fmt.Fprintf(w, "Error performing backup: %v", err)
		return
	}

	w.WriteHeader(201)
}
<|endoftext|>


<commit_before>/*

	patchtree is a simple library that knows how to interpret a folder
	structure of jd diffs and apply them on top of a base.

		test/
		  base.json
		  after_move/
		    modification.patch
		  sanitization/
		    modification.patch
		    hidden/
		      modification.patch
		    nonempty/
		      modification.patch

	Given a path relative ot the current binary, it walks backwards up the
	folder, ensuring that a modification.patch exists in each directory until
	it finds a base.json. Then it applies forward all of the
	modification.patches to give you the final composed json blob result.

*/
package patchtree

import (
	"errors"
	jd "github.com/jkomoros/jd/lib"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

const BASE_JSON = "base.json"
const PATCH = "modification.patch"
const EXPANDED_JSON_NAME = "node.expanded.json"

//JSON returns the patched json blob impplied by that directory structure or
//an error if something doesn't work. See the package doc for more.
func JSON(path string) ([]byte, error) {

	if strings.HasSuffix(path, "/") {
		path = strings.TrimSuffix(path, "/")
	}

	result, err := processDirectory(path)

	if err != nil {
		return nil, err
	}

	return []byte(result.Json()), nil

}

//MustJSON is the same as JSON, but if it would have returned an error, panics istead.
func MustJSON(path string) []byte {
	result, err := JSON(path)
	if err != nil {
		panic(err)
	}
	return result
}

type directoryFunc func(string, jd.JsonNode) (int, error)

func startDirectoryAndWalk(rootPath string, subFunc directoryFunc) (int, error) {
	baseJsonPath := filepath.Clean(rootPath + "/" + BASE_JSON)

	if _, err := os.Stat(baseJsonPath); os.IsNotExist(err) {
		return 0, errors.New("Base json file did not exist: " + err.Error())
	}

	node, err := jd.ReadJsonFile(baseJsonPath)

	if err != nil {
		return 0, errors.New("Couldn't parse base json file: " + err.Error())
	}

	return walkDirectory(rootPath, node, subFunc)
}

func walkDirectory(directory string, expandedNode jd.JsonNode, subFunc directoryFunc) (int, error) {
	files, err := ioutil.ReadDir(directory)

	if err != nil {
		return 0, errors.New("Couldn't read directory: " + err.Error())
	}

	numAffectedFiles := 0

	for _, file := range files {
		if !file.IsDir() {
			continue
		}
		subDirectory := filepath.Clean(directory + "/" + file.Name())

		subAffectedFiles, err := subFunc(subDirectory, expandedNode)
		if err != nil {
			return numAffectedFiles, err
		}
		numAffectedFiles += subAffectedFiles
	}

	return numAffectedFiles, nil
}

//ExpandTree expands all of the nodes in the patchtree, applying the chains of
//modification and created an node.expanded.json in each node. Used in a
//workflow to modify base.json: run this commeand, then modify base.json, then
//run ContractTree.
func ExpandTree(rootPath string) (affectedFiles int, err error) {
	return startDirectoryAndWalk(rootPath, expandTreeProcessDirectory)
}

func expandTreeProcessDirectory(directory string, node jd.JsonNode) (int, error) {

	diffFileName := filepath.Clean(directory + "/" + PATCH)

	if _, err := os.Stat(diffFileName); os.IsNotExist(err) {
		//TODO: it's weird to print to log when this condition is hit.
		log.Println(diffFileName + " did not exist; skipping that directory and all beneath it.")
		return 0, nil
	}

	diff, err := jd.ReadDiffFile(diffFileName)

	if err != nil {
		return 0, errors.New(diffFileName + " could not be loaded as patch file: " + err.Error())
	}

	expandedNode, err := node.Patch(diff)

	if err != nil {
		return 0, errors.New(diffFileName + " could not be applied: " + err.Error())
	}

	expandedNodeFileName := filepath.Clean(directory + "/" + EXPANDED_JSON_NAME)

	data := expandedNode.Json()

	if err := ioutil.WriteFile(expandedNodeFileName, []byte(data), 0644); err != nil {
		return 0, errors.New("Couldn't write " + expandedNodeFileName + ": " + err.Error())
	}

	numAffectedFiles, err := walkDirectory(directory, expandedNode, expandTreeProcessDirectory)

	if err != nil {
		return 0, err
	}

	return numAffectedFiles + 1, nil

}

//ContractTree goes through each node in the parse tree and where it finds a
//node.expanded,json, re-derives and overwrites the "modification.patch". Used
//as part of a workflow to modify base.json: run ExpandTree, modify base.json,
//then ContractTree.
func ContractTree(rootPath string) (numAffectedFiles int, err error) {
	return 0, errors.New("Not yet implemented")
}

func processDirectory(path string) (jd.JsonNode, error) {

	//If no more path pieces error
	if path == "" || path == "/" || path == "./" {
		return nil, errors.New("Didn't find a base.json anywhere in the given directory structure")
	}

	//TODO: check if the directory exists...

	baseJsonPath := filepath.Clean(path + "/" + BASE_JSON)

	if _, err := os.Stat(baseJsonPath); err == nil {
		//Found the directory with base.json!
		node, err := jd.ReadJsonFile(baseJsonPath)
		if err != nil {
			return nil, errors.New(path + " had error reading base.json: " + err.Error())
		}
		return node, nil
	}

	modificationPatchPath := filepath.Clean(path + "/" + PATCH)

	if _, err := os.Stat(modificationPatchPath); err == nil {

		//Recurse, with the sub-directory.
		baseJson, err := processDirectory(filepath.Dir(path))

		if err != nil {
			return nil, err
		}

		diff, err := jd.ReadDiffFile(modificationPatchPath)

		if err != nil {
			return nil, errors.New("Error reading diff file at " + modificationPatchPath + ": " + err.Error())
		}

		composed, err := baseJson.Patch(diff)

		if err != nil {
			return nil, errors.New(path + " had error diffing " + err.Error())
		}

		return composed, nil
	}

	//Path had neither base.json or modification.patch, which is an error
	return nil, errors.New("In " + path + " didn't have either " + BASE_JSON + " or " + PATCH)

}
<commit_msg>Implement ContractTree. Not tested. Part of #607.<commit_after>/*

	patchtree is a simple library that knows how to interpret a folder
	structure of jd diffs and apply them on top of a base.

		test/
		  base.json
		  after_move/
		    modification.patch
		  sanitization/
		    modification.patch
		    hidden/
		      modification.patch
		    nonempty/
		      modification.patch

	Given a path relative ot the current binary, it walks backwards up the
	folder, ensuring that a modification.patch exists in each directory until
	it finds a base.json. Then it applies forward all of the
	modification.patches to give you the final composed json blob result.

*/
package patchtree

import (
	"errors"
	jd "github.com/jkomoros/jd/lib"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

const BASE_JSON = "base.json"
const PATCH = "modification.patch"
const EXPANDED_JSON_NAME = "node.expanded.json"

//JSON returns the patched json blob impplied by that directory structure or
//an error if something doesn't work. See the package doc for more.
func JSON(path string) ([]byte, error) {

	if strings.HasSuffix(path, "/") {
		path = strings.TrimSuffix(path, "/")
	}

	result, err := processDirectory(path)

	if err != nil {
		return nil, err
	}

	return []byte(result.Json()), nil

}

//MustJSON is the same as JSON, but if it would have returned an error, panics istead.
func MustJSON(path string) []byte {
	result, err := JSON(path)
	if err != nil {
		panic(err)
	}
	return result
}

type directoryFunc func(string, jd.JsonNode) (int, error)

func startDirectoryAndWalk(rootPath string, subFunc directoryFunc) (int, error) {
	baseJsonPath := filepath.Clean(rootPath + "/" + BASE_JSON)

	if _, err := os.Stat(baseJsonPath); os.IsNotExist(err) {
		return 0, errors.New("Base json file did not exist: " + err.Error())
	}

	node, err := jd.ReadJsonFile(baseJsonPath)

	if err != nil {
		return 0, errors.New("Couldn't parse base json file: " + err.Error())
	}

	return walkDirectory(rootPath, node, subFunc)
}

func walkDirectory(directory string, expandedNode jd.JsonNode, subFunc directoryFunc) (int, error) {
	files, err := ioutil.ReadDir(directory)

	if err != nil {
		return 0, errors.New("Couldn't read directory: " + err.Error())
	}

	numAffectedFiles := 0

	for _, file := range files {
		if !file.IsDir() {
			continue
		}
		subDirectory := filepath.Clean(directory + "/" + file.Name())

		subAffectedFiles, err := subFunc(subDirectory, expandedNode)
		if err != nil {
			return numAffectedFiles, err
		}
		numAffectedFiles += subAffectedFiles
	}

	return numAffectedFiles, nil
}

//ExpandTree expands all of the nodes in the patchtree, applying the chains of
//modification and created an node.expanded.json in each node. Used in a
//workflow to modify base.json: run this commeand, then modify base.json, then
//run ContractTree.
func ExpandTree(rootPath string) (affectedFiles int, err error) {
	return startDirectoryAndWalk(rootPath, expandTreeProcessDirectory)
}

func expandTreeProcessDirectory(directory string, node jd.JsonNode) (int, error) {

	diffFileName := filepath.Clean(directory + "/" + PATCH)

	if _, err := os.Stat(diffFileName); os.IsNotExist(err) {
		//TODO: it's weird to print to log when this condition is hit.
		log.Println(diffFileName + " did not exist; skipping that directory and all beneath it.")
		return 0, nil
	}

	diff, err := jd.ReadDiffFile(diffFileName)

	if err != nil {
		return 0, errors.New(diffFileName + " could not be loaded as patch file: " + err.Error())
	}

	expandedNode, err := node.Patch(diff)

	if err != nil {
		return 0, errors.New(diffFileName + " could not be applied: " + err.Error())
	}

	expandedNodeFileName := filepath.Clean(directory + "/" + EXPANDED_JSON_NAME)

	data := expandedNode.Json()

	if err := ioutil.WriteFile(expandedNodeFileName, []byte(data), 0644); err != nil {
		return 0, errors.New("Couldn't write " + expandedNodeFileName + ": " + err.Error())
	}

	numAffectedFiles, err := walkDirectory(directory, expandedNode, expandTreeProcessDirectory)

	if err != nil {
		return 0, err
	}

	return numAffectedFiles + 1, nil

}

//ContractTree goes through each node in the parse tree and where it finds a
//node.expanded,json, re-derives and overwrites the "modification.patch". Used
//as part of a workflow to modify base.json: run ExpandTree, modify base.json,
//then ContractTree.
func ContractTree(rootPath string) (numAffectedFiles int, err error) {
	return startDirectoryAndWalk(rootPath, contractTreeProcessDirectory)
}

func contractTreeProcessDirectory(directory string, node jd.JsonNode) (int, error) {

	nodeFileName := filepath.Clean(directory + "/" + EXPANDED_JSON_NAME)

	expandedNode, err := jd.ReadJsonFile(nodeFileName)

	if err != nil {
		return 0, errors.New(nodeFileName + " could not be loaded as json file: " + err.Error())
	}

	patch := node.Diff(expandedNode)

	data := patch.Render()

	diffFileName := filepath.Clean(directory + "/" + PATCH)

	if err := ioutil.WriteFile(diffFileName, []byte(data), 0644); err != nil {
		return 0, errors.New("Couldn't write " + diffFileName + ": " + err.Error())
	}

	numAffectedFiles, err := walkDirectory(directory, expandedNode, contractTreeProcessDirectory)

	if err != nil {
		return 0, err
	}

	return numAffectedFiles + 1, nil

}

func processDirectory(path string) (jd.JsonNode, error) {

	//If no more path pieces error
	if path == "" || path == "/" || path == "./" {
		return nil, errors.New("Didn't find a base.json anywhere in the given directory structure")
	}

	//TODO: check if the directory exists...

	baseJsonPath := filepath.Clean(path + "/" + BASE_JSON)

	if _, err := os.Stat(baseJsonPath); err == nil {
		//Found the directory with base.json!
		node, err := jd.ReadJsonFile(baseJsonPath)
		if err != nil {
			return nil, errors.New(path + " had error reading base.json: " + err.Error())
		}
		return node, nil
	}

	modificationPatchPath := filepath.Clean(path + "/" + PATCH)

	if _, err := os.Stat(modificationPatchPath); err == nil {

		//Recurse, with the sub-directory.
		baseJson, err := processDirectory(filepath.Dir(path))

		if err != nil {
			return nil, err
		}

		diff, err := jd.ReadDiffFile(modificationPatchPath)

		if err != nil {
			return nil, errors.New("Error reading diff file at " + modificationPatchPath + ": " + err.Error())
		}

		composed, err := baseJson.Patch(diff)

		if err != nil {
			return nil, errors.New(path + " had error diffing " + err.Error())
		}

		return composed, nil
	}

	//Path had neither base.json or modification.patch, which is an error
	return nil, errors.New("In " + path + " didn't have either " + BASE_JSON + " or " + PATCH)

}
<|endoftext|>


<commit_before>// Copyright (c) 2018-2019 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package iptables

import (
	"bytes"
	"io"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"sync"

	version "github.com/hashicorp/go-version"
	log "github.com/sirupsen/logrus"

	"github.com/projectcalico/felix/versionparse"
)

var (
	vXDotYDotZRegexp = regexp.MustCompile(`v(\d+\.\d+\.\d+)`)

	// iptables versions:
	// v1Dot4Dot7 is the oldest version we've ever supported.
	v1Dot4Dot7 = versionparse.MustParseVersion("1.4.7")
	// v1Dot6Dot0 added --random-fully to SNAT.
	v1Dot6Dot0 = versionparse.MustParseVersion("1.6.0")
	// v1Dot6Dot2 added --random-fully to MASQUERADE and the xtables lock to iptables-restore.
	v1Dot6Dot2 = versionparse.MustParseVersion("1.6.2")

	// Linux kernel versions:
	// v3Dot10Dot0 is the oldest version we support at time of writing.
	v3Dot10Dot0 = versionparse.MustParseVersion("3.10.0")
	// v3Dot14Dot0 added the random-fully feature on the iptables interface.
	v3Dot14Dot0 = versionparse.MustParseVersion("3.14.0")
)

type Features struct {
	// SNATFullyRandom is true if --random-fully is supported by the SNAT action.
	SNATFullyRandom bool
	// MASQFullyRandom is true if --random-fully is supported by the MASQUERADE action.
	MASQFullyRandom bool
	// RestoreSupportsLock is true if the iptables-restore command supports taking the xtables lock and the
	// associated -w and -W arguments.
	RestoreSupportsLock bool
}

// Values; ""|true|false.
// "" means "detect" and is default. true|false overrides.
type FeatureDetectOverrides struct {
	SNATFullyRandom     string
	MASQFullyRandom     string
	RestoreSupportsLock string
}

type FeatureDetector struct {
	lock            sync.Mutex
	featureCache    *Features
	featureOverride *FeatureDetectOverrides

	// Path to file with kernel version
	GetKernelVersionReader func() (io.Reader, error)
	// Factory for making commands, used by UTs to shim exec.Command().
	NewCmd cmdFactory
}

func NewFeatureDetector(overrides *FeatureDetectOverrides) *FeatureDetector {
	return &FeatureDetector{
		GetKernelVersionReader: versionparse.GetKernelVersionReader,
		NewCmd:                 NewRealCmd,
		featureOverride:        overrides,
	}
}

func (d *FeatureDetector) GetFeatures() *Features {
	d.lock.Lock()
	defer d.lock.Unlock()

	if d.featureCache == nil {
		d.refreshFeaturesLockHeld()
	}

	return d.featureCache
}

func (d *FeatureDetector) RefreshFeatures() {
	d.lock.Lock()
	defer d.lock.Unlock()

	d.refreshFeaturesLockHeld()
}

func (d *FeatureDetector) refreshFeaturesLockHeld() {
	// Get the versions.  If we fail to detect a version for some reason, we use a safe default.
	log.Debug("Refreshing detected iptables features")

	// Disable checks if all features are overidden (fallback if detection
	// fails)
	if d.featureOverride != nil {
		var features Features
		var err error
		features.SNATFullyRandom, err = strconv.ParseBool(d.featureOverride.SNATFullyRandom)
		if err == nil {
			features.MASQFullyRandom, err = strconv.ParseBool(d.featureOverride.MASQFullyRandom)
		}
		if err == nil {
			features.RestoreSupportsLock, err = strconv.ParseBool(d.featureOverride.RestoreSupportsLock)
		}
		if err == nil {
			// All features overidden
			d.featureCache = &features
			log.Info("All features overidden. Feature detect disabled")
			return
		}
	}
	iptV := d.getIptablesVersion()
	kerV := d.getKernelVersion()

	// Calculate the features.
	features := Features{
		SNATFullyRandom:     iptV.Compare(v1Dot6Dot0) >= 0 && kerV.Compare(v3Dot14Dot0) >= 0,
		MASQFullyRandom:     iptV.Compare(v1Dot6Dot2) >= 0 && kerV.Compare(v3Dot14Dot0) >= 0,
		RestoreSupportsLock: iptV.Compare(v1Dot6Dot2) >= 0,
	}

	if d.featureOverride != nil {
		value, err := strconv.ParseBool(d.featureOverride.SNATFullyRandom)
		if err == nil {
			log.Info("Override feature SNATFullyRandom", value)
			features.SNATFullyRandom = value
		}
		value, err = strconv.ParseBool(d.featureOverride.MASQFullyRandom)
		if err == nil {
			log.Info("Override feature MASQFullyRandom", value)
			features.MASQFullyRandom = value
		}
		value, err = strconv.ParseBool(d.featureOverride.RestoreSupportsLock)
		if err == nil {
			log.Info("Override feature RestoreSupportsLock", value)
			features.RestoreSupportsLock = value
		}
	}

	if d.featureCache == nil || *d.featureCache != features {
		log.WithFields(log.Fields{
			"features":        features,
			"kernelVersion":   kerV,
			"iptablesVersion": iptV,
		}).Info("Updating detected iptables features")
		d.featureCache = &features
	}
}

func (d *FeatureDetector) getIptablesVersion() *version.Version {
	cmd := d.NewCmd("iptables", "--version")
	out, err := cmd.Output()
	if err != nil {
		log.WithError(err).Warn("Failed to get iptables version, assuming old version with no optional features")
		return v1Dot4Dot7
	}
	s := string(out)
	log.WithField("rawVersion", s).Debug("Ran iptables --version")
	matches := vXDotYDotZRegexp.FindStringSubmatch(s)
	if len(matches) == 0 {
		log.WithField("rawVersion", s).Warn(
			"Failed to parse iptables version, assuming old version with no optional features")
		return v1Dot4Dot7
	}
	parsedVersion, err := version.NewVersion(matches[1])
	if err != nil {
		log.WithField("rawVersion", s).WithError(err).Warn(
			"Failed to parse iptables version, assuming old version with no optional features")
		return v1Dot4Dot7
	}
	log.WithField("version", parsedVersion).Debug("Parsed iptables version")
	return parsedVersion
}

func (d *FeatureDetector) getKernelVersion() *version.Version {
	reader, err := d.GetKernelVersionReader()
	if err != nil {
		log.WithError(err).Warn("Failed to get the kernel version reader, assuming old version with no optional features")
		return v3Dot10Dot0
	}
	kernVersion, err := versionparse.GetKernelVersion(reader)
	if err != nil {
		log.WithError(err).Warn("Failed to get kernel version, assuming old version with no optional features")
		return v3Dot10Dot0
	}
	return kernVersion
}

func countRulesInIptableOutput(in []byte) int {
	count := 0
	for _, x := range bytes.Split(in, []byte("\n")) {
		if len(x) >= 1 && x[0] == '-' {
			count++
		}
	}
	return count
}

// GetIptablesBackend attempts to detect the iptables backend being used where Felix is running.
// This code is duplicating the detection method found at
// https://github.com/kubernetes/kubernetes/blob/623b6978866b5d3790d17ff13601ef9e7e4f4bf0/build/debian-iptables/iptables-wrapper#L28
// If there is a specifiedBackend then it is used but if it does not match the detected
// backend then a warning is logged.
func DetectBackend(lookPath func(file string) (string, error), newCmd cmdFactory, specifiedBackend string) string {
	ip6LgcySave := findBestBinary(lookPath, 6, "legacy", "save")
	ip4LgcySave := findBestBinary(lookPath, 4, "legacy", "save")
	ip6l, _ := newCmd(ip6LgcySave).Output()
	ip4l, _ := newCmd(ip4LgcySave).Output()
	log.WithField("ip6l", string(ip6l)).Debug("Ip6tables legacy save out")
	log.WithField("ip4l", string(ip4l)).Debug("Iptables legacy save out")
	legacyLines := countRulesInIptableOutput(ip6l) + countRulesInIptableOutput(ip4l)
	var detectedBackend string
	if legacyLines >= 10 {
		detectedBackend = "legacy"
	} else {
		ip6NftSave := findBestBinary(lookPath, 6, "nft", "save")
		ip4NftSave := findBestBinary(lookPath, 4, "nft", "save")
		ip6n, _ := newCmd(ip6NftSave).Output()
		log.WithField("ip6n", string(ip6n)).Debug("Ip6tables save out")
		ip4n, _ := newCmd(ip4NftSave).Output()
		log.WithField("ip4n", string(ip4n)).Debug("Iptables save out")
		nftLines := countRulesInIptableOutput(ip6n) + countRulesInIptableOutput(ip4n)
		if legacyLines >= nftLines {
			detectedBackend = "legacy"
		} else {
			detectedBackend = "nft"
		}
	}
	log.WithField("detectedBackend", detectedBackend).Debug("Detected Iptables backend")

	specifiedBackend = strings.ToLower(specifiedBackend)
	if specifiedBackend != "auto" {
		if specifiedBackend != detectedBackend {
			log.WithFields(log.Fields{"detectedBackend": detectedBackend, "specifiedBackend": specifiedBackend}).Warn("Iptables backend specified does not match the detected backend, using specified backend")
		}
		return specifiedBackend
	}
	return detectedBackend
}

// findBestBinary tries to find an iptables binary for the specific variant (legacy/nftables mode) and returns the name
// of the binary.  Falls back on iptables-restore/iptables-save if the specific variant isn't available.
// Panics if no binary can be found.
func findBestBinary(lookPath func(file string) (string, error), ipVersion uint8, backendMode, saveOrRestore string) string {
	if lookPath == nil {
		lookPath = exec.LookPath
	}
	verInfix := ""
	if ipVersion == 6 {
		verInfix = "6"
	}
	candidates := []string{
		"ip" + verInfix + "tables-" + backendMode + "-" + saveOrRestore,
		"ip" + verInfix + "tables-" + saveOrRestore,
	}

	logCxt := log.WithFields(log.Fields{
		"ipVersion":     ipVersion,
		"backendMode":   backendMode,
		"saveOrRestore": saveOrRestore,
		"candidates":    candidates,
	})

	for _, candidate := range candidates {
		_, err := lookPath(candidate)
		if err == nil {
			logCxt.WithField("command", candidate).Info("Looked up iptables command")
			return candidate
		}
	}

	logCxt.Panic("Failed to find iptables command")
	return ""
}

// Parse a comma separated list of overrides, example;
// "SNATFullyRandom=true,MASQFullyRandom=false"
func ParseFeatureDetectOverrides(param string) *FeatureDetectOverrides {
	if param == "" {
		return nil
	}
	var overrides FeatureDetectOverrides
	for _, s := range strings.Split(param, ",") {
		if strings.HasPrefix(s, "SNATFullyRandom=") {
			overrides.SNATFullyRandom = strings.Split(s, "=")[1]
			continue
		}
		if strings.HasPrefix(s, "MASQFullyRandom=") {
			overrides.MASQFullyRandom = strings.Split(s, "=")[1]
			continue
		}
		if strings.HasPrefix(s, "RestoreSupportsLock=") {
			overrides.RestoreSupportsLock = strings.Split(s, "=")[1]
			continue
		}
		log.Info("Unknown override", s)
	}
	return &overrides
}
<commit_msg>Removed the feature-override pre-check<commit_after>// Copyright (c) 2018-2019 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package iptables

import (
	"bytes"
	"io"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"sync"

	version "github.com/hashicorp/go-version"
	log "github.com/sirupsen/logrus"

	"github.com/projectcalico/felix/versionparse"
)

var (
	vXDotYDotZRegexp = regexp.MustCompile(`v(\d+\.\d+\.\d+)`)

	// iptables versions:
	// v1Dot4Dot7 is the oldest version we've ever supported.
	v1Dot4Dot7 = versionparse.MustParseVersion("1.4.7")
	// v1Dot6Dot0 added --random-fully to SNAT.
	v1Dot6Dot0 = versionparse.MustParseVersion("1.6.0")
	// v1Dot6Dot2 added --random-fully to MASQUERADE and the xtables lock to iptables-restore.
	v1Dot6Dot2 = versionparse.MustParseVersion("1.6.2")

	// Linux kernel versions:
	// v3Dot10Dot0 is the oldest version we support at time of writing.
	v3Dot10Dot0 = versionparse.MustParseVersion("3.10.0")
	// v3Dot14Dot0 added the random-fully feature on the iptables interface.
	v3Dot14Dot0 = versionparse.MustParseVersion("3.14.0")
)

type Features struct {
	// SNATFullyRandom is true if --random-fully is supported by the SNAT action.
	SNATFullyRandom bool
	// MASQFullyRandom is true if --random-fully is supported by the MASQUERADE action.
	MASQFullyRandom bool
	// RestoreSupportsLock is true if the iptables-restore command supports taking the xtables lock and the
	// associated -w and -W arguments.
	RestoreSupportsLock bool
}

// Values; ""|true|false.
// "" means "detect" and is default. true|false overrides.
type FeatureDetectOverrides struct {
	SNATFullyRandom     string
	MASQFullyRandom     string
	RestoreSupportsLock string
}

type FeatureDetector struct {
	lock            sync.Mutex
	featureCache    *Features
	featureOverride *FeatureDetectOverrides

	// Path to file with kernel version
	GetKernelVersionReader func() (io.Reader, error)
	// Factory for making commands, used by UTs to shim exec.Command().
	NewCmd cmdFactory
}

func NewFeatureDetector(overrides *FeatureDetectOverrides) *FeatureDetector {
	return &FeatureDetector{
		GetKernelVersionReader: versionparse.GetKernelVersionReader,
		NewCmd:                 NewRealCmd,
		featureOverride:        overrides,
	}
}

func (d *FeatureDetector) GetFeatures() *Features {
	d.lock.Lock()
	defer d.lock.Unlock()

	if d.featureCache == nil {
		d.refreshFeaturesLockHeld()
	}

	return d.featureCache
}

func (d *FeatureDetector) RefreshFeatures() {
	d.lock.Lock()
	defer d.lock.Unlock()

	d.refreshFeaturesLockHeld()
}

func (d *FeatureDetector) refreshFeaturesLockHeld() {
	// Get the versions.  If we fail to detect a version for some reason, we use a safe default.
	log.Debug("Refreshing detected iptables features")

	iptV := d.getIptablesVersion()
	kerV := d.getKernelVersion()

	// Calculate the features.
	features := Features{
		SNATFullyRandom:     iptV.Compare(v1Dot6Dot0) >= 0 && kerV.Compare(v3Dot14Dot0) >= 0,
		MASQFullyRandom:     iptV.Compare(v1Dot6Dot2) >= 0 && kerV.Compare(v3Dot14Dot0) >= 0,
		RestoreSupportsLock: iptV.Compare(v1Dot6Dot2) >= 0,
	}

	if d.featureOverride != nil {
		value, err := strconv.ParseBool(d.featureOverride.SNATFullyRandom)
		if err == nil {
			log.WithField("override", value).Info("Override feature SNATFullyRandom")
			features.SNATFullyRandom = value
		}
		value, err = strconv.ParseBool(d.featureOverride.MASQFullyRandom)
		if err == nil {
			log.WithField("override", value).Info("Override feature MASQFullyRandom")
			features.MASQFullyRandom = value
		}
		value, err = strconv.ParseBool(d.featureOverride.RestoreSupportsLock)
		if err == nil {
			log.WithField("override", value).Info("Override feature RestoreSupportsLock")
			features.RestoreSupportsLock = value
		}
	}

	if d.featureCache == nil || *d.featureCache != features {
		log.WithFields(log.Fields{
			"features":        features,
			"kernelVersion":   kerV,
			"iptablesVersion": iptV,
		}).Info("Updating detected iptables features")
		d.featureCache = &features
	}
}

func (d *FeatureDetector) getIptablesVersion() *version.Version {
	cmd := d.NewCmd("iptables", "--version")
	out, err := cmd.Output()
	if err != nil {
		log.WithError(err).Warn("Failed to get iptables version, assuming old version with no optional features")
		return v1Dot4Dot7
	}
	s := string(out)
	log.WithField("rawVersion", s).Debug("Ran iptables --version")
	matches := vXDotYDotZRegexp.FindStringSubmatch(s)
	if len(matches) == 0 {
		log.WithField("rawVersion", s).Warn(
			"Failed to parse iptables version, assuming old version with no optional features")
		return v1Dot4Dot7
	}
	parsedVersion, err := version.NewVersion(matches[1])
	if err != nil {
		log.WithField("rawVersion", s).WithError(err).Warn(
			"Failed to parse iptables version, assuming old version with no optional features")
		return v1Dot4Dot7
	}
	log.WithField("version", parsedVersion).Debug("Parsed iptables version")
	return parsedVersion
}

func (d *FeatureDetector) getKernelVersion() *version.Version {
	reader, err := d.GetKernelVersionReader()
	if err != nil {
		log.WithError(err).Warn("Failed to get the kernel version reader, assuming old version with no optional features")
		return v3Dot10Dot0
	}
	kernVersion, err := versionparse.GetKernelVersion(reader)
	if err != nil {
		log.WithError(err).Warn("Failed to get kernel version, assuming old version with no optional features")
		return v3Dot10Dot0
	}
	return kernVersion
}

func countRulesInIptableOutput(in []byte) int {
	count := 0
	for _, x := range bytes.Split(in, []byte("\n")) {
		if len(x) >= 1 && x[0] == '-' {
			count++
		}
	}
	return count
}

// GetIptablesBackend attempts to detect the iptables backend being used where Felix is running.
// This code is duplicating the detection method found at
// https://github.com/kubernetes/kubernetes/blob/623b6978866b5d3790d17ff13601ef9e7e4f4bf0/build/debian-iptables/iptables-wrapper#L28
// If there is a specifiedBackend then it is used but if it does not match the detected
// backend then a warning is logged.
func DetectBackend(lookPath func(file string) (string, error), newCmd cmdFactory, specifiedBackend string) string {
	ip6LgcySave := findBestBinary(lookPath, 6, "legacy", "save")
	ip4LgcySave := findBestBinary(lookPath, 4, "legacy", "save")
	ip6l, _ := newCmd(ip6LgcySave).Output()
	ip4l, _ := newCmd(ip4LgcySave).Output()
	log.WithField("ip6l", string(ip6l)).Debug("Ip6tables legacy save out")
	log.WithField("ip4l", string(ip4l)).Debug("Iptables legacy save out")
	legacyLines := countRulesInIptableOutput(ip6l) + countRulesInIptableOutput(ip4l)
	var detectedBackend string
	if legacyLines >= 10 {
		detectedBackend = "legacy"
	} else {
		ip6NftSave := findBestBinary(lookPath, 6, "nft", "save")
		ip4NftSave := findBestBinary(lookPath, 4, "nft", "save")
		ip6n, _ := newCmd(ip6NftSave).Output()
		log.WithField("ip6n", string(ip6n)).Debug("Ip6tables save out")
		ip4n, _ := newCmd(ip4NftSave).Output()
		log.WithField("ip4n", string(ip4n)).Debug("Iptables save out")
		nftLines := countRulesInIptableOutput(ip6n) + countRulesInIptableOutput(ip4n)
		if legacyLines >= nftLines {
			detectedBackend = "legacy"
		} else {
			detectedBackend = "nft"
		}
	}
	log.WithField("detectedBackend", detectedBackend).Debug("Detected Iptables backend")

	specifiedBackend = strings.ToLower(specifiedBackend)
	if specifiedBackend != "auto" {
		if specifiedBackend != detectedBackend {
			log.WithFields(log.Fields{"detectedBackend": detectedBackend, "specifiedBackend": specifiedBackend}).Warn("Iptables backend specified does not match the detected backend, using specified backend")
		}
		return specifiedBackend
	}
	return detectedBackend
}

// findBestBinary tries to find an iptables binary for the specific variant (legacy/nftables mode) and returns the name
// of the binary.  Falls back on iptables-restore/iptables-save if the specific variant isn't available.
// Panics if no binary can be found.
func findBestBinary(lookPath func(file string) (string, error), ipVersion uint8, backendMode, saveOrRestore string) string {
	if lookPath == nil {
		lookPath = exec.LookPath
	}
	verInfix := ""
	if ipVersion == 6 {
		verInfix = "6"
	}
	candidates := []string{
		"ip" + verInfix + "tables-" + backendMode + "-" + saveOrRestore,
		"ip" + verInfix + "tables-" + saveOrRestore,
	}

	logCxt := log.WithFields(log.Fields{
		"ipVersion":     ipVersion,
		"backendMode":   backendMode,
		"saveOrRestore": saveOrRestore,
		"candidates":    candidates,
	})

	for _, candidate := range candidates {
		_, err := lookPath(candidate)
		if err == nil {
			logCxt.WithField("command", candidate).Info("Looked up iptables command")
			return candidate
		}
	}

	logCxt.Panic("Failed to find iptables command")
	return ""
}

// Parse a comma separated list of overrides, example;
// "SNATFullyRandom=true,MASQFullyRandom=false"
func ParseFeatureDetectOverrides(param string) *FeatureDetectOverrides {
	if param == "" {
		return nil
	}
	var overrides FeatureDetectOverrides
	for _, s := range strings.Split(param, ",") {
		if strings.HasPrefix(s, "SNATFullyRandom=") {
			overrides.SNATFullyRandom = strings.Split(s, "=")[1]
			continue
		}
		if strings.HasPrefix(s, "MASQFullyRandom=") {
			overrides.MASQFullyRandom = strings.Split(s, "=")[1]
			continue
		}
		if strings.HasPrefix(s, "RestoreSupportsLock=") {
			overrides.RestoreSupportsLock = strings.Split(s, "=")[1]
			continue
		}
		log.Info("Unknown override", s)
	}
	return &overrides
}
<|endoftext|>


<commit_before>package offheap_test

import (
	"testing"

	"github.com/glycerine/go-offheap-hashtable"
	cv "github.com/smartystreets/goconvey/convey"
)

func TestCompact(t *testing.T) {

	cv.Convey("Given a big table with no values it, Compact() should re-allocate it to be smaller", t, func() {
		h := offheap.NewHashTable(128)
		cv.So(len(h.Cells), cv.ShouldEqual, 128)
		cv.So(h.Population, cv.ShouldEqual, 0)

		h.Compact()

		cv.So(len(h.Cells), cv.ShouldEqual, 1)
		cv.So(h.Population, cv.ShouldEqual, 0)
	})

}
<commit_msg>atg. more compact test<commit_after>package offheap_test

import (
	"testing"

	"github.com/glycerine/go-offheap-hashtable"
	cv "github.com/smartystreets/goconvey/convey"
)

func TestCompact(t *testing.T) {

	cv.Convey("Given a big table with no values it, Compact() should re-allocate it to be smaller", t, func() {
		h := offheap.NewHashTable(128)
		cv.So(len(h.Cells), cv.ShouldEqual, 128)
		cv.So(h.Population, cv.ShouldEqual, 0)

		h.Compact()

		cv.So(len(h.Cells), cv.ShouldEqual, 1)
		cv.So(h.Population, cv.ShouldEqual, 0)
	})

}

func TestCompatAfterDelete(t *testing.T) {

	h := offheap.NewHashTable(2)

	h.Clear()
	cv.Convey("after being filled up and then deleted down to just 2 elements, Compact() should reduce table size to 4", t, func() {
		N := 100

		// fill up a table
		cv.So(h.Population, cv.ShouldEqual, 0)
		for i := 0; i < N; i++ {
			h.Insert(uint64(i))
		}

		// now delete from it
		for i := 0; i < N-2; i++ {
			h.DeleteKey(uint64(i))
		}
		cv.So(len(h.Cells), cv.ShouldEqual, 256)
		h.Compact()
		cv.So(len(h.Cells), cv.ShouldEqual, 4)

	})
}
<|endoftext|>


<commit_before>package arn

import (
	"errors"
	"time"
)

// AnimeList ...
type AnimeList struct {
	UserID string           `json:"userId"`
	Items  []*AnimeListItem `json:"items"`

	user *User
}

// Add adds an anime to the list if it hasn't been added yet.
func (list *AnimeList) Add(id interface{}) error {
	animeID := id.(string)

	if list.Contains(animeID) {
		return errors.New("Anime " + animeID + " has already been added")
	}

	creationDate := time.Now().UTC().Format(time.RFC3339)

	newItem := &AnimeListItem{
		AnimeID: animeID,
		Status:  AnimeListStatusPlanned,
		Created: creationDate,
		Edited:  creationDate,
	}

	list.Items = append(list.Items, newItem)

	return nil
}

// Remove removes the anime ID from the list.
func (list *AnimeList) Remove(id interface{}) bool {
	animeID := id.(string)

	for index, item := range list.Items {
		if item.AnimeID == animeID {
			list.Items = append(list.Items[:index], list.Items[index+1:]...)
			return true
		}
	}

	return false
}

// Contains checks if the list contains the anime ID already.
func (list *AnimeList) Contains(id interface{}) bool {
	animeID := id.(string)

	for _, item := range list.Items {
		if item.AnimeID == animeID {
			return true
		}
	}

	return false
}

// Find returns the list item with the specified anime ID, if available.
func (list *AnimeList) Find(animeID string) *AnimeListItem {
	for _, item := range list.Items {
		if item.AnimeID == animeID {
			return item
		}
	}

	return nil
}

// User returns the user this anime list belongs to.
func (list *AnimeList) User() *User {
	if list.user == nil {
		list.user, _ = GetUser(list.UserID)
	}

	return list.user
}

// TransformBody returns an item that is passed to methods like Add, Remove, etc.
func (list *AnimeList) TransformBody(body []byte) interface{} {
	return string(body)
}

// Save saves the anime list in the database.
func (list *AnimeList) Save() error {
	return DB.Set("AnimeList", list.UserID, list)
}

// GetAnimeList ...
func GetAnimeList(userID string) (*AnimeList, error) {
	obj, err := DB.Get("AnimeList", userID)
	return obj.(*AnimeList), err
}
<commit_msg>Implemented API authorization<commit_after>package arn

import (
	"errors"
	"time"

	"github.com/aerogo/aero"
)

// AnimeList ...
type AnimeList struct {
	UserID string           `json:"userId"`
	Items  []*AnimeListItem `json:"items"`

	user *User
}

// Add adds an anime to the list if it hasn't been added yet.
func (list *AnimeList) Add(id interface{}) error {
	animeID := id.(string)

	if list.Contains(animeID) {
		return errors.New("Anime " + animeID + " has already been added")
	}

	creationDate := time.Now().UTC().Format(time.RFC3339)

	newItem := &AnimeListItem{
		AnimeID: animeID,
		Status:  AnimeListStatusPlanned,
		Created: creationDate,
		Edited:  creationDate,
	}

	list.Items = append(list.Items, newItem)

	return nil
}

// Remove removes the anime ID from the list.
func (list *AnimeList) Remove(id interface{}) bool {
	animeID := id.(string)

	for index, item := range list.Items {
		if item.AnimeID == animeID {
			list.Items = append(list.Items[:index], list.Items[index+1:]...)
			return true
		}
	}

	return false
}

// Contains checks if the list contains the anime ID already.
func (list *AnimeList) Contains(id interface{}) bool {
	animeID := id.(string)

	for _, item := range list.Items {
		if item.AnimeID == animeID {
			return true
		}
	}

	return false
}

// Find returns the list item with the specified anime ID, if available.
func (list *AnimeList) Find(animeID string) *AnimeListItem {
	for _, item := range list.Items {
		if item.AnimeID == animeID {
			return item
		}
	}

	return nil
}

// Authorize returns an error if the given API request is not authorized.
func (list *AnimeList) Authorize(ctx *aero.Context) error {
	if !ctx.HasSession() {
		return errors.New("Neither logged in nor in session")
	}

	userID, ok := ctx.Session().Get("userId").(string)

	if !ok || userID == "" {
		return errors.New("Not logged in")
	}

	if userID != ctx.Get("id") {
		return errors.New("Can not modify data from other users")
	}

	return nil
}

// User returns the user this anime list belongs to.
func (list *AnimeList) User() *User {
	if list.user == nil {
		list.user, _ = GetUser(list.UserID)
	}

	return list.user
}

// TransformBody returns an item that is passed to methods like Add, Remove, etc.
func (list *AnimeList) TransformBody(body []byte) interface{} {
	return string(body)
}

// Save saves the anime list in the database.
func (list *AnimeList) Save() error {
	return DB.Set("AnimeList", list.UserID, list)
}

// GetAnimeList ...
func GetAnimeList(userID string) (*AnimeList, error) {
	obj, err := DB.Get("AnimeList", userID)
	return obj.(*AnimeList), err
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"syscall"
	"time"

	"github.com/dickeyxxx/golock"
)

var updateTopic = &Topic{
	Name:        "update",
	Description: "update heroku-cli",
}

var updateCmd = &Command{
	Topic:       "update",
	Hidden:      true,
	Description: "updates heroku-cli",
	Args:        []Arg{{Name: "channel", Optional: true}},
	Run: func(ctx *Context) {
		channel := ctx.Args.(map[string]string)["channel"]
		if channel == "" {
			channel = Channel
		}
		Update(channel)
	},
}

var binPath = filepath.Join(AppDir, "heroku-cli")
var updateLockPath = filepath.Join(AppDir, "updating.lock")
var autoupdateFile = filepath.Join(AppDir, "autoupdate")

func init() {
	if runtime.GOOS == "windows" {
		binPath = binPath + ".exe"
	}
}

// Update updates the CLI and plugins
func Update(channel string) {
	touchAutoupdateFile()
	golock.Lock(updateLockPath)
	defer golock.Unlock(updateLockPath)
	done := make(chan bool)
	go func() {
		updateCLI(channel)
		updatePlugins()
		done <- true
	}()
	select {
	case <-time.After(time.Second * 120):
		Errln("Timed out while updating")
	case <-done:
	}
}

func updatePlugins() {
	Err("updating plugins... ")
	b, _ := node.UpdatePackages()
	Errln("done")
	if len(b) > 0 {
		Err("rebuilding plugins cache... ")
		ClearPluginCache()
		WritePluginCache(GetPlugins())
		Errln("done")
	}
}

func updateCLI(channel string) {
	Err("checking for CLI update... ")
	manifest := getUpdateManifest(channel)
	if manifest.Version == Version && manifest.Channel == Channel {
		Errf("already on latest version %s\n", Version)
		return
	}
	if !updatable() {
		Errf("Out of date: You are running %s but %s is out\n", Version, manifest.Version)
		return
	}
	Errf("updating from %s to %s (%s)... ", Version, manifest.Version, manifest.Channel)
	build := manifest.Builds[runtime.GOOS][runtime.GOARCH]
	// on windows we can't remove an existing file or remove the running binary
	// so we download the file to binName.new
	// move the running binary to binName.old (deleting any existing file first)
	// rename the downloaded file to binName
	if err := downloadBin(binPath+".new", build.URL); err != nil {
		panic(err)
	}
	if fileSha1(binPath+".new") != build.Sha1 {
		panic("SHA mismatch")
	}
	os.Remove(binPath + ".old")
	if err := os.Rename(binPath, binPath+".old"); err != nil {
		panic(err)
	}
	if err := os.Rename(binPath+".new", binPath); err != nil {
		panic(err)
	}
	Errln("done")
	golock.Unlock(updateLockPath)
	reexecBin()
}

// IsUpdateNeeded checks if an update is available
func IsUpdateNeeded(t string) bool {
	f, err := os.Stat(autoupdateFile)
	if err != nil {
		return true
	}
	if t == "soft" {
		return time.Since(f.ModTime()) > 4*time.Hour
	}
	return time.Since(f.ModTime()) > 168*time.Hour
}

func touchAutoupdateFile() {
	out, err := os.OpenFile(autoupdateFile, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		panic(err)
	}
	out.WriteString(time.Now().String())
}

type manifest struct {
	Channel, Version string
	Builds           map[string]map[string]struct {
		URL, Sha1 string
	}
}

func getUpdateManifest(channel string) manifest {
	res, err := http.Get("https://d1gvo455cekpjp.cloudfront.net/" + channel + "/manifest.json")
	if err != nil {
		panic(err)
	}
	var m manifest
	json.NewDecoder(res.Body).Decode(&m)
	return m
}

func updatable() bool {
	path, err := filepath.Abs(os.Args[0])
	if err != nil {
		Errln(err)
	}
	return path == binPath
}

func downloadBin(path, url string) error {
	out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
	if err != nil {
		return err
	}
	client := &http.Client{}
	req, err := http.NewRequest("GET", url+".gz", nil)
	if err != nil {
		return err
	}
	req.Header.Add("Accept-Encoding", "gzip")
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	uncompressed, err := gzip.NewReader(resp.Body)
	if err != nil {
		return err
	}
	_, err = io.Copy(out, uncompressed)
	if err != nil {
		return err
	}
	return out.Close()
}

func fileSha1(path string) string {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("%x", sha1.Sum(data))
}

func reexecBin() {
	if runtime.GOOS == "windows" {
		cmd := exec.Command(binPath, os.Args[1:]...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Run()
		os.Exit(0)
	} else {
		if err := syscall.Exec(binPath, os.Args, os.Environ()); err != nil {
			panic(err)
		}
		os.Exit(99) // should never happen
	}
}

// TriggerBackgroundUpdate will trigger an update to the client in the background
func TriggerBackgroundUpdate() {
	if err := exec.Command(binPath, "update").Start(); err != nil {
		panic(err)
	}
}

// WarnIfUpdating prints to stderr if the CLI is updating
func WarnIfUpdating() {
	if exists, _ := fileExists(updateLockPath); exists {
		Errln("WARNING: CLI is updating")
	}
}
<commit_msg>updater fixes<commit_after>package main

import (
	"crypto/sha1"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"syscall"
	"time"

	"github.com/dickeyxxx/golock"
	"github.com/franela/goreq"
)

var updateTopic = &Topic{
	Name:        "update",
	Description: "update heroku-cli",
}

var updateCmd = &Command{
	Topic:       "update",
	Hidden:      true,
	Description: "updates heroku-cli",
	Args:        []Arg{{Name: "channel", Optional: true}},
	Run: func(ctx *Context) {
		channel := ctx.Args.(map[string]string)["channel"]
		if channel == "" {
			channel = Channel
		}
		Update(channel)
	},
}

var binPath = filepath.Join(AppDir, "heroku-cli")
var updateLockPath = filepath.Join(AppDir, "updating.lock")
var autoupdateFile = filepath.Join(AppDir, "autoupdate")

func init() {
	if runtime.GOOS == "windows" {
		binPath = binPath + ".exe"
	}
}

// Update updates the CLI and plugins
func Update(channel string) {
	touchAutoupdateFile()
	golock.Lock(updateLockPath)
	defer golock.Unlock(updateLockPath)
	done := make(chan bool)
	go func() {
		updateCLI(channel)
		updatePlugins()
		done <- true
	}()
	select {
	case <-time.After(time.Second * 120):
		Errln("Timed out while updating")
	case <-done:
	}
}

func updatePlugins() {
	Err("updating plugins... ")
	b, _ := node.UpdatePackages()
	Errln("done")
	if len(b) > 0 {
		Err("rebuilding plugins cache... ")
		ClearPluginCache()
		WritePluginCache(GetPlugins())
		Errln("done")
	}
}

func updateCLI(channel string) {
	manifest := getUpdateManifest(channel)
	if manifest.Version == Version && manifest.Channel == Channel {
		return
	}
	if !updatable() {
		Errf("Out of date: You are running %s but %s is out\n", Version, manifest.Version)
		return
	}
	Errf("updating v4 CLI to %s (%s)... ", manifest.Version, manifest.Channel)
	build := manifest.Builds[runtime.GOOS][runtime.GOARCH]
	// on windows we can't remove an existing file or remove the running binary
	// so we download the file to binName.new
	// move the running binary to binName.old (deleting any existing file first)
	// rename the downloaded file to binName
	if err := downloadBin(binPath+".new", build.URL); err != nil {
		panic(err)
	}
	if fileSha1(binPath+".new") != build.Sha1 {
		panic("SHA mismatch")
	}
	os.Remove(binPath + ".old")
	os.Rename(binPath, binPath+".old")
	if err := os.Rename(binPath+".new", binPath); err != nil {
		panic(err)
	}
	Errln("done")
	golock.Unlock(updateLockPath)
	reexecBin()
}

// IsUpdateNeeded checks if an update is available
func IsUpdateNeeded(t string) bool {
	f, err := os.Stat(autoupdateFile)
	if err != nil {
		return true
	}
	if t == "soft" {
		return time.Since(f.ModTime()) > 4*time.Hour
	}
	return time.Since(f.ModTime()) > 168*time.Hour
}

func touchAutoupdateFile() {
	out, err := os.OpenFile(autoupdateFile, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		panic(err)
	}
	out.WriteString(time.Now().String())
}

type manifest struct {
	Channel, Version string
	Builds           map[string]map[string]struct {
		URL, Sha1 string
	}
}

func getUpdateManifest(channel string) manifest {
	res, err := goreq.Request{
		Uri: "https://d1gvo455cekpjp.cloudfront.net/" + channel + "/manifest.json",
	}.Do()
	if err != nil {
		panic(err)
	}
	var m manifest
	res.Body.FromJsonTo(&m)
	return m
}

func updatable() bool {
	path, err := filepath.Abs(os.Args[0])
	if err != nil {
		Errln(err)
	}
	return path == binPath
}

func downloadBin(path, url string) error {
	out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
	if err != nil {
		return err
	}
	res, err := goreq.Request{
		Uri: url + ".gz",
	}.Do()
	if err != nil {
		return err
	}
	defer res.Body.Close()
	_, err = io.Copy(out, res.Body)
	if err != nil {
		return err
	}
	return out.Close()
}

func fileSha1(path string) string {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("%x", sha1.Sum(data))
}

func reexecBin() {
	if runtime.GOOS == "windows" {
		cmd := exec.Command(binPath, os.Args[1:]...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Run()
		os.Exit(0)
	} else {
		if err := syscall.Exec(binPath, os.Args, os.Environ()); err != nil {
			panic(err)
		}
		os.Exit(99) // should never happen
	}
}

// TriggerBackgroundUpdate will trigger an update to the client in the background
func TriggerBackgroundUpdate() {
	if err := exec.Command(binPath, "update").Start(); err != nil {
		panic(err)
	}
}

// WarnIfUpdating prints to stderr if the CLI is updating
func WarnIfUpdating() {
	if exists, _ := fileExists(updateLockPath); exists {
		Errln("WARNING: CLI is updating")
	}
}
<|endoftext|>