<commit_before>package main

import (
	"bufio"
	"encoding/csv"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"os"
	"strings"
	"sync"
)

type Bookmark struct {
	Href        string
	Description string
}

type LookupFailure struct {
	Bookmark Bookmark
	Error    error
}

type FailureReporter func(LookupFailure)

func debug(format string, args ...interface{}) {
	if debugEnabled {
		log.Printf(format+"\n", args...)
	}
}

var debugEnabled bool

func parseJson(bookmarkJson []byte) []Bookmark {
	var bookmarks []Bookmark
	json.Unmarshal(bookmarkJson, &bookmarks)
	return bookmarks
}

func buildDownloadEndpoint(token string) string {
	endpoint, _ := url.Parse("https://api.pinboard.in/v1/posts/all")
	query := endpoint.Query()
	query.Add("auth_token", token)
	query.Add("format", "json")
	endpoint.RawQuery = query.Encode()
	return endpoint.String()
}

func buildDeleteEndpoint(token string, rawUrl string) string {
	endpoint, _ := url.Parse("https://api.pinboard.in/v1/posts/delete")
	query := endpoint.Query()
	query.Add("auth_token", token)
	query.Add("format", "json")
	query.Add("url", rawUrl)
	endpoint.RawQuery = query.Encode()
	return endpoint.String()
}

func downloadBookmarks(token string) ([]byte, error) {
	response, err := http.Get(buildDownloadEndpoint(token))
	defer response.Body.Close()

	if err != nil {
		debug("Error %s", err)
		return nil, err
	}

	return ioutil.ReadAll(response.Body)
}

// we consider HTTP 429 indicative that the resource exists
func isBadStatus(response *http.Response) bool {
	return response.StatusCode != 200 && response.StatusCode != http.StatusTooManyRequests
}

func check(bookmark Bookmark) (bool, int, error) {
	cookieJar, _ := cookiejar.New(nil)

	// TODO: Use same client in all workers
	client := &http.Client{
		Jar: cookieJar,
	}

	url := bookmark.Href
	headResponse, err := client.Head(url)
	if err != nil {
		// fmt.Printf("%s error: %s\n", url, err)
		return false, -1, err
	}

	headResponse.Body.Close()

	if isBadStatus(headResponse) {
		// fmt.Fprintf(os.Stderr, "Trying a GET request to retrieve %s\n", url)
		getResponse, err := client.Get(url)

		if err != nil {
			getResponse.Body.Close()
			// fmt.Printf("%s error: %s\n", url, err)
			return false, -1, err
		}

		getResponse.Body.Close()

		if isBadStatus(getResponse) {
			return false, getResponse.StatusCode, err
		}
	}

	return true, headResponse.StatusCode, nil
}

func worker(id int, checkJobs <-chan Bookmark, reporter FailureReporter, workgroup *sync.WaitGroup) {
	defer workgroup.Done()

	for bookmark := range checkJobs {
		debug("Worker %02d: Processing job for url %s", id, bookmark.Href)
		valid, code, err := check(bookmark)
		if !valid {
			reporter(LookupFailure{bookmark, err})
			debug("Worker %02d: ERROR: %s %d %s", id, bookmark.Href, code, err)
		} else {
			debug("Worker %02d: Success for %s\n", id, bookmark.Href)
		}
	}
}

func csvFailureReader(failure LookupFailure) {
	file, err := os.Create("failedlinks.csv")
	if err != nil {
		log.Fatal("Cannot create file", err)
	}

	defer file.Close()

	writer := csv.NewWriter(file)
	defer writer.Flush()

	var errorValue string
	if failure.Error != nil {
		errorValue = failure.Error.Error()
	}

	record := []string{
		failure.Bookmark.Description,
		failure.Bookmark.Href,
		errorValue,
	}
	writer.Write(record)
}

func stdoutFailureReporter(failure LookupFailure) {
	fmt.Fprintf(os.Stdout, "[ERR] %s\n", failure.Bookmark.Href)
}

func readUrlsFromFile(source string) []string {
	urls := make([]string, 0)

	if file, err := os.Open(source); err == nil {
		defer file.Close()

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			url := strings.TrimSpace(scanner.Text())
			urls = append(urls, url)
		}

		if err = scanner.Err(); err != nil {
			log.Fatal(err)
		}
	} else {
		log.Fatalf("ERROR: %s", err)
	}
	return urls
}

func checkAll(bookmarkJson []byte, reporter FailureReporter) {
	jobs := make(chan Bookmark, 10)
	workgroup := new(sync.WaitGroup)

	// start workers
	for w := 1; w <= 10; w++ {
		workgroup.Add(1)
		go worker(w, jobs, reporter, workgroup)
	}

	// send off URLs to check
	for _, bookmark := range parseJson(bookmarkJson) {
		jobs <- bookmark
	}

	close(jobs)
	workgroup.Wait()
}

func deleteBookmark(token string, bookmark Bookmark) {
	endpoint := buildDeleteEndpoint(token, bookmark.Href)

	debug("Deleting %s\n", bookmark.Href)

	response, err := http.Get(endpoint)
	defer response.Body.Close()

	if err != nil {
		log.Fatalf("ERROR: %s", err)
	}

	body, err := ioutil.ReadAll(response.Body)

	if err != nil {
		log.Fatalf("ERROR: %s", err)
	}

	debug("%s", body)
}

func deleteAll(token string, reader io.Reader) {
	scanner := bufio.NewScanner(reader)
	for scanner.Scan() {
		url := strings.TrimSpace(scanner.Text())
		deleteBookmark(token, Bookmark{url, ""})
	}

	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}
}

func handleDownloadAction(token string) {
	bookmarks, err := downloadBookmarks(token)
	if err != nil {
		log.Fatal(err)
	}
	debug("%s", string(bookmarks))
}

func handleDeleteAction(token string, resultsFileName string) {
	if resultsFileName == "-" {
		debug("Using stdin")
		deleteAll(token, os.Stdin)
	} else {
		debug("Using bookmarks from %s\n", resultsFileName)
		file, err := os.Open(resultsFileName)
		if err != nil {
			log.Fatal("Could not read file with bookmarks to delete")
		} else {
			deleteAll(token, file)
		}
	}
}

func handleCheckAction(token string, inputFile string, outputFile string) {
	var bookmarkJson []byte
	if len(inputFile) > 0 {
		bookmarkJson, _ = ioutil.ReadFile(inputFile)
	} else {
		bookmarkJson, _ = downloadBookmarks(token)
	}

	// different failure reporter depending on setting of outputFile, default to
	// stderr simple error printing for now
	var reporter FailureReporter
	switch {
	default:
		reporter = stdoutFailureReporter
	}

	checkAll(bookmarkJson, reporter)
}

func main() {
	var downloadAction bool
	flag.BoolVar(&downloadAction, "download", false, "Download all bookmarks, write them to stdout")

	var deleteAction bool
	flag.BoolVar(&deleteAction, "delete", false, "Use this to delete bookmarks. Requires passing a list of links to delete.")

	var token string
	flag.StringVar(&token, "token", "", "Mandatory authentication token")

	flag.BoolVar(&debugEnabled, "debug", false, "Enable debug logs, will be printed on stderr")

	var outputFile string
	flag.StringVar(&outputFile, "outputFile", "-", "File to store results of check operation in, defaults to stdout")

	var inputFile string
	flag.StringVar(&inputFile, "inputFile", "", "File containing bookmarks to check. If empty it will download all bookmarks from pinboard.")

	var inputFormat string
	flag.StringVar(&inputFormat, "inputFormat", "text", "Which format the input file is in (can be 'text', 'json')")

	var checkAction bool
	flag.BoolVar(&checkAction, "check", false, "Check the links of all bookmarks")

	flag.Parse()

	if len(token) == 0 {
		log.Fatal("-token parameter has to be set")
	}

	if downloadAction {
		handleDownloadAction(token)
	}

	if deleteAction {
		handleDeleteAction(token, outputFile)
	}

	if checkAction {
		handleCheckAction(token, inputFile, outputFile)
	}
}
<commit_msg>Print usage if no flags are set<commit_after>package main

import (
	"bufio"
	"encoding/csv"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"os"
	"strings"
	"sync"
)

type Bookmark struct {
	Href        string
	Description string
}

type LookupFailure struct {
	Bookmark Bookmark
	Error    error
}

type FailureReporter func(LookupFailure)

func debug(format string, args ...interface{}) {
	if debugEnabled {
		log.Printf(format+"\n", args...)
	}
}

var debugEnabled bool

func parseJson(bookmarkJson []byte) []Bookmark {
	var bookmarks []Bookmark
	json.Unmarshal(bookmarkJson, &bookmarks)
	return bookmarks
}

func buildDownloadEndpoint(token string) string {
	endpoint, _ := url.Parse("https://api.pinboard.in/v1/posts/all")
	query := endpoint.Query()
	query.Add("auth_token", token)
	query.Add("format", "json")
	endpoint.RawQuery = query.Encode()
	return endpoint.String()
}

func buildDeleteEndpoint(token string, rawUrl string) string {
	endpoint, _ := url.Parse("https://api.pinboard.in/v1/posts/delete")
	query := endpoint.Query()
	query.Add("auth_token", token)
	query.Add("format", "json")
	query.Add("url", rawUrl)
	endpoint.RawQuery = query.Encode()
	return endpoint.String()
}

func downloadBookmarks(token string) ([]byte, error) {
	response, err := http.Get(buildDownloadEndpoint(token))
	defer response.Body.Close()

	if err != nil {
		debug("Error %s", err)
		return nil, err
	}

	return ioutil.ReadAll(response.Body)
}

// we consider HTTP 429 indicative that the resource exists
func isBadStatus(response *http.Response) bool {
	return response.StatusCode != 200 && response.StatusCode != http.StatusTooManyRequests
}

func check(bookmark Bookmark) (bool, int, error) {
	cookieJar, _ := cookiejar.New(nil)

	// TODO: Use same client in all workers
	client := &http.Client{
		Jar: cookieJar,
	}

	url := bookmark.Href
	headResponse, err := client.Head(url)
	if err != nil {
		// fmt.Printf("%s error: %s\n", url, err)
		return false, -1, err
	}

	headResponse.Body.Close()

	if isBadStatus(headResponse) {
		// fmt.Fprintf(os.Stderr, "Trying a GET request to retrieve %s\n", url)
		getResponse, err := client.Get(url)

		if err != nil {
			getResponse.Body.Close()
			// fmt.Printf("%s error: %s\n", url, err)
			return false, -1, err
		}

		getResponse.Body.Close()

		if isBadStatus(getResponse) {
			return false, getResponse.StatusCode, err
		}
	}

	return true, headResponse.StatusCode, nil
}

func worker(id int, checkJobs <-chan Bookmark, reporter FailureReporter, workgroup *sync.WaitGroup) {
	defer workgroup.Done()

	for bookmark := range checkJobs {
		debug("Worker %02d: Processing job for url %s", id, bookmark.Href)
		valid, code, err := check(bookmark)
		if !valid {
			reporter(LookupFailure{bookmark, err})
			debug("Worker %02d: ERROR: %s %d %s", id, bookmark.Href, code, err)
		} else {
			debug("Worker %02d: Success for %s\n", id, bookmark.Href)
		}
	}
}

func csvFailureReader(failure LookupFailure) {
	file, err := os.Create("failedlinks.csv")
	if err != nil {
		log.Fatal("Cannot create file", err)
	}

	defer file.Close()

	writer := csv.NewWriter(file)
	defer writer.Flush()

	var errorValue string
	if failure.Error != nil {
		errorValue = failure.Error.Error()
	}

	record := []string{
		failure.Bookmark.Description,
		failure.Bookmark.Href,
		errorValue,
	}
	writer.Write(record)
}

func stdoutFailureReporter(failure LookupFailure) {
	fmt.Fprintf(os.Stdout, "[ERR] %s\n", failure.Bookmark.Href)
}

func readUrlsFromFile(source string) []string {
	urls := make([]string, 0)

	if file, err := os.Open(source); err == nil {
		defer file.Close()

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			url := strings.TrimSpace(scanner.Text())
			urls = append(urls, url)
		}

		if err = scanner.Err(); err != nil {
			log.Fatal(err)
		}
	} else {
		log.Fatalf("ERROR: %s", err)
	}
	return urls
}

func checkAll(bookmarkJson []byte, reporter FailureReporter) {
	jobs := make(chan Bookmark, 10)
	workgroup := new(sync.WaitGroup)

	// start workers
	for w := 1; w <= 10; w++ {
		workgroup.Add(1)
		go worker(w, jobs, reporter, workgroup)
	}

	// send off URLs to check
	for _, bookmark := range parseJson(bookmarkJson) {
		jobs <- bookmark
	}

	close(jobs)
	workgroup.Wait()
}

func deleteBookmark(token string, bookmark Bookmark) {
	endpoint := buildDeleteEndpoint(token, bookmark.Href)

	debug("Deleting %s\n", bookmark.Href)

	response, err := http.Get(endpoint)
	defer response.Body.Close()

	if err != nil {
		log.Fatalf("ERROR: %s", err)
	}

	body, err := ioutil.ReadAll(response.Body)

	if err != nil {
		log.Fatalf("ERROR: %s", err)
	}

	debug("%s", body)
}

func deleteAll(token string, reader io.Reader) {
	scanner := bufio.NewScanner(reader)
	for scanner.Scan() {
		url := strings.TrimSpace(scanner.Text())
		deleteBookmark(token, Bookmark{url, ""})
	}

	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}
}

func handleDownloadAction(token string) {
	bookmarks, err := downloadBookmarks(token)
	if err != nil {
		log.Fatal(err)
	}
	debug("%s", string(bookmarks))
}

func handleDeleteAction(token string, resultsFileName string) {
	if resultsFileName == "-" {
		debug("Using stdin")
		deleteAll(token, os.Stdin)
	} else {
		debug("Using bookmarks from %s\n", resultsFileName)
		file, err := os.Open(resultsFileName)
		if err != nil {
			log.Fatal("Could not read file with bookmarks to delete")
		} else {
			deleteAll(token, file)
		}
	}
}

func handleCheckAction(token string, inputFile string, outputFile string) {
	var bookmarkJson []byte
	if len(inputFile) > 0 {
		bookmarkJson, _ = ioutil.ReadFile(inputFile)
	} else {
		bookmarkJson, _ = downloadBookmarks(token)
	}

	// different failure reporter depending on setting of outputFile, default to
	// stderr simple error printing for now
	var reporter FailureReporter
	switch {
	default:
		reporter = stdoutFailureReporter
	}

	checkAll(bookmarkJson, reporter)
}

func main() {
	var downloadAction bool
	flag.BoolVar(&downloadAction, "download", false, "Download all bookmarks, write them to stdout")

	var deleteAction bool
	flag.BoolVar(&deleteAction, "delete", false, "Use this to delete bookmarks. Requires passing a list of links to delete.")

	var token string
	flag.StringVar(&token, "token", "", "Mandatory authentication token")

	flag.BoolVar(&debugEnabled, "debug", false, "Enable debug logs, will be printed on stderr")

	var outputFile string
	flag.StringVar(&outputFile, "outputFile", "-", "File to store results of check operation in, defaults to stdout")

	var inputFile string
	flag.StringVar(&inputFile, "inputFile", "", "File containing bookmarks to check. If empty it will download all bookmarks from pinboard.")

	var inputFormat string
	flag.StringVar(&inputFormat, "inputFormat", "text", "Which format the input file is in (can be 'text', 'json')")

	var checkAction bool
	flag.BoolVar(&checkAction, "check", false, "Check the links of all bookmarks")

	flag.Parse()

	// at least one action flag needs to be set, print usage if no flags are present
	if flag.NFlag() == 0 {
		flag.Usage()
		return
	}

	if len(token) == 0 {
		log.Fatal("-token parameter has to be set")
	}

	if downloadAction {
		handleDownloadAction(token)
	}

	if deleteAction {
		handleDeleteAction(token, outputFile)
	}

	if checkAction {
		handleCheckAction(token, inputFile, outputFile)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/fatih/color"
	"github.com/howeyc/fsnotify"
)

func handle(ev *fsnotify.FileEvent) {
	commandline := strings.Fields(flag.Args()[0])
	args := make([]string, len(commandline[1:]))

	// Figure out if we should do file name interpolation on the arguments
	for index, arg := range commandline[1:] {
		var err error
		if strings.Contains(arg, "%f") {
			arg, err = filepath.Abs(strings.Replace(arg, "%f", ev.Name, -1))
			if err != nil {
				log.Fatal(err)
			}
		}
		args[index] = arg
	}

	cmd := exec.Command(commandline[0], args...)

	pipe, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Print("\033[H\033[2J") // Clear the screen
	// Print the command in nice colors
	yellow := color.New(color.FgYellow, color.Bold).SprintfFunc()
	magenta := color.New(color.FgMagenta, color.Bold).SprintfFunc()
	out := fmt.Sprintf("Running %s %s...", yellow(commandline[0]), magenta(strings.Join(args, " ")))

	log.Println(out)
	if err = cmd.Start(); err != nil {
		log.Fatal(err)
	}

	buff := make([]byte, 1024)

	for {
		n, err := pipe.Read(buff)
		// Either if the pipe was empty or an EOF or other error was returned.
		if n == 0 && err == nil || err != nil {
			break
		}

		fmt.Print(string(buff[:n]))
	}

	// Print red error message or green success message
	if err = cmd.Wait(); err != nil {
		red := color.New(color.FgRed, color.Bold).SprintfFunc()
		log.Println(red(err.Error()))
	} else {
		green := color.New(color.FgGreen, color.Bold).SprintfFunc()
		log.Println(green("Execution successful."))
	}
}

func main() {
	dir := flag.String("d", ".", "directory to watch")
	flag.Parse()

	if flag.NArg() != 1 {
		log.Fatal("One command line argument sting required.")
	}

	watcher, err := fsnotify.NewWatcher()
	defer watcher.Close()

	if err != nil {
		log.Fatal(err)
	}

	done := make(chan bool)

	// Process events
	go func() {
		for {
			select {
			case ev := <-watcher.Event:
				handle(ev)

			case err := <-watcher.Error:
				log.Println("error:", err)

			}
		}
	}()

	log.Printf("Watching %s...", *dir)
	err = watcher.Watch(*dir)
	if err != nil {
		log.Fatal(err)
	}

	<-done
}
<commit_msg>Move commandline parsing into own function<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/fatih/color"
	"github.com/howeyc/fsnotify"
)

// parseArguments ...
func parseArguments(ev *fsnotify.FileEvent) (cmd string, args []string) {
	commandline := strings.Fields(flag.Args()[0])
	cmd = commandline[0]
	args = make([]string, len(commandline[1:]))

	// Do file name interpolation on the arguments if %f is in them
	for index, arg := range commandline[1:] {
		var err error
		if strings.Contains(arg, "%f") {
			arg, err = filepath.Abs(strings.Replace(arg, "%f", ev.Name, -1))
			if err != nil {
				log.Fatal(err)
			}
		}
		args[index] = arg
	}

	return
}

func handle(ev *fsnotify.FileEvent) {
	command, args := parseArguments(ev)
	cmd := exec.Command(command, args...)

	pipe, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Print("\033[H\033[2J") // Clear the screen
	// Print the command in nice colors
	yellow := color.New(color.FgYellow, color.Bold).SprintfFunc()
	magenta := color.New(color.FgMagenta, color.Bold).SprintfFunc()
	out := fmt.Sprintf("Running %s %s...", yellow(command), magenta(strings.Join(args, " ")))

	log.Println(out)
	if err = cmd.Start(); err != nil {
		log.Fatal(err)
	}

	buff := make([]byte, 1024)

	for {
		n, err := pipe.Read(buff)
		// Either if the pipe was empty or an EOF or other error was returned.
		if n == 0 && err == nil || err != nil {
			break
		}

		fmt.Print(string(buff[:n]))
	}

	// Print red error message or green success message
	if err = cmd.Wait(); err != nil {
		red := color.New(color.FgRed, color.Bold).SprintfFunc()
		log.Println(red(err.Error()))
	} else {
		green := color.New(color.FgGreen, color.Bold).SprintfFunc()
		log.Println(green("Execution successful."))
	}
}

func main() {
	dir := flag.String("d", ".", "directory to watch")
	flag.Parse()

	if flag.NArg() != 1 {
		log.Fatal("One command line argument sting required.")
	}

	watcher, err := fsnotify.NewWatcher()
	defer watcher.Close()

	if err != nil {
		log.Fatal(err)
	}

	done := make(chan bool)

	// Process events
	go func() {
		for {
			select {
			case ev := <-watcher.Event:
				handle(ev)

			case err := <-watcher.Error:
				log.Println("error:", err)

			}
		}
	}()

	log.Printf("Watching %s...", *dir)
	err = watcher.Watch(*dir)
	if err != nil {
		log.Fatal(err)
	}

	<-done
}
<|endoftext|>


<commit_before>package main

import (
	"./autils"
	"./config"
	"./dataProcess"
	"./middlewares"
	"./routers"
	"database/sql"
	"github.com/gin-gonic/gin"
	"net/http"
	"path/filepath"
	"regexp"
)

func main() {
	gin.SetMode(gin.ReleaseMode)

	app := gin.Default()

	// 使用中间件获取参数
	app.Use(middlewares.Params())

	cwd := autils.GetCwd()
	app.LoadHTMLGlob(filepath.Join(cwd, "views/*"))
	app.GET("/", func(c *gin.Context) {
		c.String(http.StatusOK, "Server is ok.")
	})

	//pqDB := autils.OpenDb("postgres", config.PQFlowUrl)
	pqDB := autils.OpenDb("postgres", config.PQTestUrl)
	pqDB.SetMaxOpenConns(100)
	pqDB.SetMaxIdleConns(20)

	// API路由处理
	apiRouters(app, pqDB)

	// 列表路由处理
	listRouters(app, pqDB)

	defer pqDB.Close()
	app.Run(config.Port)
}

// API路由处理
func apiRouters(router *gin.Engine, pqDB *sql.DB) {
	apis := router.Group("/api")

	apis.GET("/:type", func(c *gin.Context) {
		dataType := c.Param("type")

		token := c.Query("showx_token")
		if token != config.TokenStr {
			returnError(c, "Wrong token.")
			return
		}

		processAct(c, dataType, pqDB)
	})
}

// 列表路由处理
func listRouters(router *gin.Engine, db *sql.DB) {
	listRouters := router.Group("/list")

	listRouters.GET("/domain/:domain", func(c *gin.Context) {
		domain := c.Param("domain")
		dataProcess.RenderDomainTpl(c, domain, db)
	})

	listRouters.GET("/tags/:tagName", func(c *gin.Context) {
		tags := c.Param("tagName")
		match, err := regexp.MatchString("mip-", tags)
		autils.ErrHadle(err)

		if match {
			dataProcess.RenderTagTpl(c, tags, db)
		} else {
			dataProcess.SampleData(c, db, tags)
		}
	})
}

// 错误json信息统一处理
func returnError(c *gin.Context, msg string) {
	c.JSON(200, gin.H{
		"status": "1",
		"msg":    msg,
		"data":   nil,
	})
}

// 路径控制
func processAct(c *gin.Context, a string, pqDB *sql.DB) {
	handler := routers.RouterMap[a]

	if handler != nil {
		handler(c, pqDB)
	} else {
		returnError(c, "No such operation.")
	}
}
<commit_msg>update channel varible.<commit_after>package main

import (
	"./autils"
	"./config"
	"./dataProcess"
	"./middlewares"
	"./routers"
	"database/sql"
	"github.com/gin-gonic/gin"
	"net/http"
	"path/filepath"
	"regexp"
)

func main() {
	gin.SetMode(gin.ReleaseMode)

	app := gin.Default()

	// 使用中间件获取参数
	app.Use(middlewares.Params())

	cwd := autils.GetCwd()
	app.LoadHTMLGlob(filepath.Join(cwd, "views/*"))
	app.GET("/", func(c *gin.Context) {
		c.String(http.StatusOK, "Server is ok.")
	})

	pqDB := autils.OpenDb("postgres", config.PQFlowUrl)
	//pqDB := autils.OpenDb("postgres", config.PQTestUrl)
	pqDB.SetMaxOpenConns(100)
	pqDB.SetMaxIdleConns(20)

	// API路由处理
	apiRouters(app, pqDB)

	// 列表路由处理
	listRouters(app, pqDB)

	defer pqDB.Close()
	app.Run(config.Port)
}

// API路由处理
func apiRouters(router *gin.Engine, pqDB *sql.DB) {
	apis := router.Group("/api")

	apis.GET("/:type", func(c *gin.Context) {
		dataType := c.Param("type")

		token := c.Query("showx_token")
		if token != config.TokenStr {
			returnError(c, "Wrong token.")
			return
		}

		processAct(c, dataType, pqDB)
	})
}

// 列表路由处理
func listRouters(router *gin.Engine, db *sql.DB) {
	listRouters := router.Group("/list")

	listRouters.GET("/domain/:domain", func(c *gin.Context) {
		domain := c.Param("domain")
		dataProcess.RenderDomainTpl(c, domain, db)
	})

	listRouters.GET("/tags/:tagName", func(c *gin.Context) {
		tags := c.Param("tagName")
		match, err := regexp.MatchString("mip-", tags)
		autils.ErrHadle(err)

		if match {
			dataProcess.RenderTagTpl(c, tags, db)
		} else {
			dataProcess.SampleData(c, db, tags)
		}
	})
}

// 错误json信息统一处理
func returnError(c *gin.Context, msg string) {
	c.JSON(200, gin.H{
		"status": "1",
		"msg":    msg,
		"data":   nil,
	})
}

// 路径控制
func processAct(c *gin.Context, a string, pqDB *sql.DB) {
	handler := routers.RouterMap[a]

	if handler != nil {
		handler(c, pqDB)
	} else {
		returnError(c, "No such operation.")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"http"
	"log"
	"os"
	"time"
)

type Env map[string]interface{}
type Status int
type Headers map[string]string
type Body string

// This is the core app the user has written
type App func(Env) (Status, Headers, Body)

// These are pieces of middleware,
// which 'wrap' around the core App
// (and each other)
type Middleware func(Env, App) (Status, Headers, Body)

// Bundle a given list of Middleware pieces into a App
func bundle(r ...Middleware) App {
	if len(r) <= 1 {
		// Terminate the innermost piece of Middleware
		// Basically stops it from recursing any further.
		return func(input Env) (Status, Headers, Body) {
			return r[0](input, func(Env) (Status, Headers, Body) {
				panic("Core Mango App should never call it's upstream function.")
			})
		}
	}
	return wrap(r[0], bundle(r[1:]...))
}

// Attach a piece of Middleware to the outside
// of a App. This wraps the inner App
// inside the outer Middleware.
func wrap(middleware Middleware, app App) App {
	return func(input Env) (Status, Headers, Body) {
		return middleware(input, app)
	}
}

// Convert a App into Middleware
// We convert the core app into a Middleware
// so we can pass it to Bundle as part of the
// stack. Because the App does not call its
// upstream method, the resulting Middleware
// will just ignore any upstream passed to it.
func middlewareify(app App) Middleware {
	return func(input Env, upstream App) (Status, Headers, Body) {
		return app(input)
	}
}

type Mango struct {
	address    string
	middleware []Middleware
	app        App
}

func (this *Mango) Middleware(middleware ...Middleware) {
	this.middleware = middleware
}

func (this *Mango) buildStack() http.HandlerFunc {
	stack := this.middleware
	compiled_app := bundle(append(stack, middlewareify(this.app))...)
	return func(w http.ResponseWriter, r *http.Request) {
		env := make(map[string]interface{})
		status, headers, body := compiled_app(env)
		w.WriteHeader(int(status))
		for key, value := range headers {
			w.Header().Set(key, value)
		}
		fmt.Fprintf(w, string(body))
	}
}

func (this *Mango) Run(app App) os.Error {
	this.app = app
	if this.address == "" {
		this.address = "0.0.0.0:8000"
	}
	log.Println("Starting Mango Server On:", this.address)
	http.HandleFunc("/", this.buildStack())
	return http.ListenAndServe(this.address, nil)
}

/*************************************
 * End Mango Source
 * Begin Example Usage
 ************************************/

func Logger(env Env, app App) (Status, Headers, Body) {
	status, headers, body := app(env)
	log.Println(env["REQUEST_METHOD"], env["REQUEST_PATH"], status)
	return status, headers, body
}

func Hello(Env) (Status, Headers, Body) {
	return 200, map[string]string{"Content-Type": "text/html"}, Body(fmt.Sprintf("%d", time.Seconds()))
}

func main() {
	mango := new(Mango)
	mango.address = ":3000"
	mango.Middleware(Logger)
	mango.Run(Hello)
}
<commit_msg>renaming Env to Request which mirrors *http.Request<commit_after>package main

import (
	"fmt"
	"http"
	"log"
	"os"
	"time"
)

type Request *http.Request
type Status int
type Headers map[string]string
type Body string

// This is the core app the user has written
type App func(Request) (Status, Headers, Body)

// These are pieces of middleware,
// which 'wrap' around the core App
// (and each other)
type Middleware func(Request, App) (Status, Headers, Body)

// Bundle a given list of Middleware pieces into a App
func bundle(r ...Middleware) App {
	if len(r) <= 1 {
		// Terminate the innermost piece of Middleware
		// Basically stops it from recursing any further.
		return func(input Request) (Status, Headers, Body) {
			return r[0](input, func(Request) (Status, Headers, Body) {
				panic("Core Mango App should never call it's upstream function.")
			})
		}
	}
	return wrap(r[0], bundle(r[1:]...))
}

// Attach a piece of Middleware to the outside
// of a App. This wraps the inner App
// inside the outer Middleware.
func wrap(middleware Middleware, app App) App {
	return func(input Request) (Status, Headers, Body) {
		return middleware(input, app)
	}
}

// Convert a App into Middleware
// We convert the core app into a Middleware
// so we can pass it to Bundle as part of the
// stack. Because the App does not call its
// upstream method, the resulting Middleware
// will just ignore any upstream passed to it.
func middlewareify(app App) Middleware {
	return func(input Request, upstream App) (Status, Headers, Body) {
		return app(input)
	}
}

type Mango struct {
	address    string
	middleware []Middleware
	app        App
}

func (this *Mango) Middleware(middleware ...Middleware) {
	this.middleware = middleware
}

func (this *Mango) buildStack() http.HandlerFunc {
	stack := this.middleware
	compiled_app := bundle(append(stack, middlewareify(this.app))...)
	return func(w http.ResponseWriter, r *http.Request) {
		status, headers, body := compiled_app(r)
		w.WriteHeader(int(status))
		for key, value := range headers {
			w.Header().Set(key, value)
		}
		fmt.Fprintf(w, string(body))
	}
}

func (this *Mango) Run(app App) os.Error {
	this.app = app
	if this.address == "" {
		this.address = "0.0.0.0:8000"
	}
	log.Println("Starting Mango Server On:", this.address)
	http.HandleFunc("/", this.buildStack())
	return http.ListenAndServe(this.address, nil)
}

/*************************************
 * End Mango Source
 * Begin Example Usage
 ************************************/

func Logger(req Request, app App) (Status, Headers, Body) {
	status, headers, body := app(req)
	log.Println(req.Method, req.RawURL, status)
	return status, headers, body
}

func Hello(Request) (Status, Headers, Body) {
	return 200, map[string]string{"Content-Type": "text/html"}, Body(fmt.Sprintf("%d", time.Seconds()))
}

func main() {
	mango := new(Mango)
	mango.address = ":3000"
	mango.Middleware(Logger)
	mango.Run(Hello)
}
<|endoftext|>


<commit_before>/*
 * Copyright 2015 Fabrício Godoy
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"io/ioutil"
	"fmt"
	"github.com/skarllot/mydump/config"
)

const (
	CONFIG_FILE_NAME = "config.json"
)

func main() {
	content, err := ioutil.ReadFile(CONFIG_FILE_NAME)
	if err != nil {
		fmt.Println("Could not load configuration file:", err)
		return
	}

	cfg := &config.Config{}
	if err := json.Unmarshal(content, cfg); err != nil {
		fmt.Println("Invalid configuration file:", err)
		return
	}

	fmt.Printf("%#v\n", cfg)
}
<commit_msg>Initial destination directory creation<commit_after>/*
 * Copyright 2015 Fabrício Godoy
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"

	"github.com/skarllot/mydump/config"
)

const (
	CONFIG_FILE_NAME = "config.json"
)

func main() {
	content, err := ioutil.ReadFile(CONFIG_FILE_NAME)
	if err != nil {
		fmt.Println("Could not load configuration file:", err)
		return
	}

	cfg := &config.Config{}
	if err := json.Unmarshal(content, cfg); err != nil {
		fmt.Println("Invalid configuration file:", err)
		return
	}

	if b, err := pathExists(cfg.Destination); !b {
		if err != nil {
			fmt.Println("Could not stat destination directory:", err)
			return
		}
		if err := os.MkdirAll(cfg.Destination, 0750); err != nil {
			fmt.Println("Could not create destination directory:", err)
			return
		}
	}

	for _, job := range cfg.Jobs {
		dstPath := path.Join(cfg.Destination, job.Database.Hostname)
		if b, err := pathExists(dstPath); !b {
			if err != nil {
				fmt.Println("Could not stat destination directory:", err)
				return
			}
			if err := os.Mkdir(dstPath, 0750); err != nil {
				fmt.Println("Could not create destination directory:", err)
				return
			}
		}
	}

	fmt.Println("End")
}

func pathExists(p string) (bool, error) {
	_, err := os.Stat(p)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}

	return false, err
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"github.com/peterbourgon/mergemap"
	"github.com/russross/blackfriday"
	"html/template"
	"os"
	"path/filepath"
)

var (
	FrontSeparator = []byte("---\n")
)

var (
	debug     = flag.Bool("debug", false, "print debug information (implies verbose)")
	sourceDir = flag.String("source", "src", "path to site source (input)")
	targetDir = flag.String("target", "tgt", "path to site target (output)")
	globalKey = flag.String("global.key", "files", "template node name for per-file metadata")
)

func init() {
	flag.Parse()

	var err error
	for _, s := range []*string{sourceDir, targetDir} {
		if *s, err = filepath.Abs(*s); err != nil {
			Fatalf("%s", err)
		}
	}
}

func main() {
	m := map[string]interface{}{}
	s := NewStack()
	filepath.Walk(*sourceDir, GatherJSON(s))
	filepath.Walk(*sourceDir, GatherSource(s, m))
	s.Add("", map[string]interface{}{*globalKey: m})
	filepath.Walk(*sourceDir, Transform(s))
}

// splitMetadata splits the input buffer on FrontSeparator. It returns a byte-
// slice suitable for unmarshaling into metadata, if it exists, and the
// remainder of the input buffer.
func splitMetadata(buf []byte) ([]byte, []byte) {
	split := bytes.SplitN(buf, FrontSeparator, 2)
	if len(split) == 2 {
		return split[0], split[1]
	}
	return []byte{}, buf
}

func GatherJSON(s StackReadWriter) filepath.WalkFunc {
	Debugf("gathering JSON")
	return func(path string, info os.FileInfo, _ error) error {
		if info.IsDir() {
			return nil // descend
		}
		switch filepath.Ext(path) {
		case ".json":
			metadata := ParseJSON(Read(path))
			s.Add(filepath.Dir(path), metadata)
			Debugf("%s gathered (%d element(s))", path, len(metadata))
		}
		return nil
	}
}

func GatherSource(s StackReadWriter, m map[string]interface{}) filepath.WalkFunc {
	Debugf("gathering source")
	return func(path string, info os.FileInfo, _ error) error {
		if info.IsDir() {
			return nil // descend
		}
		switch filepath.Ext(path) {
		case ".html":
			fullMetadata := map[string]interface{}{
				"source":  Relative(*sourceDir, path),
				"target":  Relative(*targetDir, TargetFor(path, filepath.Ext(path))),
				"url":     "/" + Relative(*targetDir, TargetFor(path, filepath.Ext(path))),
				"sortkey": filepath.Base(path),
			}
			metadataBuf, _ := splitMetadata(Read(path))
			if len(metadataBuf) > 0 {
				fileMetadata := ParseJSON(metadataBuf)
				s.Add(path, fileMetadata)
			}
			fullMetadata = mergemap.Merge(fullMetadata, s.Get(path))
			SplatInto(m, Relative(*sourceDir, path), fullMetadata)
			Debugf("%s gathered (%d element(s))", path, len(fullMetadata))

		case ".md":
			fullMetadata := map[string]interface{}{
				"source":  Relative(*sourceDir, path),
				"target":  Relative(*targetDir, TargetFor(path, ".html")),
				"url":     "/" + Relative(*targetDir, TargetFor(path, ".html")),
				"sortkey": filepath.Base(path),
			}
			metadataBuf, _ := splitMetadata(Read(path))
			if len(metadataBuf) > 0 {
				fileMetadata := ParseJSON(metadataBuf)
				s.Add(path, fileMetadata)
			}
			fullMetadata = mergemap.Merge(fullMetadata, s.Get(path))
			SplatInto(m, Relative(*sourceDir, path), fullMetadata)
			Debugf("%s gathered (%d element(s))", path, len(fullMetadata))
		}
		return nil
	}
}

func Transform(s StackReader) filepath.WalkFunc {
	Debugf("transforming")
	return func(path string, info os.FileInfo, _ error) error {
		if info.IsDir() {
			Debugf("descending into %s", path)
			return nil // descend
		}

		Debugf("processing %s", path)
		switch filepath.Ext(path) {
		case ".json":
			Debugf("%s ignored for transformation", path)

		case ".html":
			// read
			_, contentBuf := splitMetadata(Read(path))

			// render
			outputBuf := RenderTemplate(path, contentBuf, s.Get(path))

			// write
			dst := TargetFor(path, filepath.Ext(path))
			Write(dst, outputBuf)
			Debugf("%s transformed to %s", path, dst)

		case ".md":
			// read
			_, contentBuf := splitMetadata(Read(path))

			// render
			metadata := mergemap.Merge(s.Get(path), map[string]interface{}{
				"content": template.HTML(RenderMarkdown(contentBuf)),
			})
			templatePath, templateBuf := Template(s, path)
			outputBuf := RenderTemplate(templatePath, templateBuf, metadata)

			// write
			dst := TargetFor(path, ".html")
			Write(dst, outputBuf)
			Debugf("%s transformed to %s", path, dst)

		case ".source", ".template":
			Debugf("%s ignored for transformation", path)

		default:
			dst := TargetFor(path, filepath.Ext(path))
			Copy(dst, path)
			Debugf("%s transformed to %s verbatim", path, dst)
		}
		return nil
	}
}

func RenderTemplate(path string, input []byte, metadata map[string]interface{}) []byte {
	R := func(relativeFilename string) string {
		filename := filepath.Join(filepath.Dir(path), relativeFilename)
		return string(RenderTemplate(filename, Read(filename), metadata))
	}
	importhtml := func(relativeFilename string) template.HTML {
		return template.HTML(R(relativeFilename))
	}
	importcss := func(relativeFilename string) template.CSS {
		return template.CSS(R(relativeFilename))
	}
	importjs := func(relativeFilename string) template.JS {
		return template.JS(R(relativeFilename))
	}

	templateName := Relative(*sourceDir, path)
	funcMap := template.FuncMap{
		"importhtml": importhtml,
		"importcss":  importcss,
		"importjs":   importjs,
		"sorted":     SortedValues,
	}

	tmpl, err := template.New(templateName).Funcs(funcMap).Parse(string(input))
	if err != nil {
		Fatalf("Render Template %s: Parse: %s", path, err)
	}

	output := bytes.Buffer{}
	if err = tmpl.Execute(&output, metadata); err != nil {
		Fatalf("Render Template %s: Execute: %s", path, err)
	}

	return output.Bytes()
}

func RenderMarkdown(input []byte) []byte {
	Debugf("rendering %d byte(s) of Markdown", len(input))
	htmlOptions := 0
	htmlOptions = htmlOptions | blackfriday.HTML_GITHUB_BLOCKCODE
	htmlOptions = htmlOptions | blackfriday.HTML_USE_SMARTYPANTS
	title, css := "", ""
	htmlRenderer := blackfriday.HtmlRenderer(htmlOptions, title, css)

	mdOptions := 0
	mdOptions = mdOptions | blackfriday.EXTENSION_FENCED_CODE

	return blackfriday.Markdown(input, htmlRenderer, mdOptions)
}
<commit_msg>Fix help flag<commit_after>package main

import (
	"bytes"
	"flag"
	"github.com/peterbourgon/mergemap"
	"github.com/russross/blackfriday"
	"html/template"
	"os"
	"path/filepath"
)

var (
	FrontSeparator = []byte("---\n")
)

var (
	debug     = flag.Bool("debug", false, "print debug information")
	sourceDir = flag.String("source", "src", "path to site source (input)")
	targetDir = flag.String("target", "tgt", "path to site target (output)")
	globalKey = flag.String("global.key", "files", "template node name for per-file metadata")
)

func init() {
	flag.Parse()

	var err error
	for _, s := range []*string{sourceDir, targetDir} {
		if *s, err = filepath.Abs(*s); err != nil {
			Fatalf("%s", err)
		}
	}
}

func main() {
	m := map[string]interface{}{}
	s := NewStack()
	filepath.Walk(*sourceDir, GatherJSON(s))
	filepath.Walk(*sourceDir, GatherSource(s, m))
	s.Add("", map[string]interface{}{*globalKey: m})
	filepath.Walk(*sourceDir, Transform(s))
}

// splitMetadata splits the input buffer on FrontSeparator. It returns a byte-
// slice suitable for unmarshaling into metadata, if it exists, and the
// remainder of the input buffer.
func splitMetadata(buf []byte) ([]byte, []byte) {
	split := bytes.SplitN(buf, FrontSeparator, 2)
	if len(split) == 2 {
		return split[0], split[1]
	}
	return []byte{}, buf
}

func GatherJSON(s StackReadWriter) filepath.WalkFunc {
	Debugf("gathering JSON")
	return func(path string, info os.FileInfo, _ error) error {
		if info.IsDir() {
			return nil // descend
		}
		switch filepath.Ext(path) {
		case ".json":
			metadata := ParseJSON(Read(path))
			s.Add(filepath.Dir(path), metadata)
			Debugf("%s gathered (%d element(s))", path, len(metadata))
		}
		return nil
	}
}

func GatherSource(s StackReadWriter, m map[string]interface{}) filepath.WalkFunc {
	Debugf("gathering source")
	return func(path string, info os.FileInfo, _ error) error {
		if info.IsDir() {
			return nil // descend
		}
		switch filepath.Ext(path) {
		case ".html":
			fullMetadata := map[string]interface{}{
				"source":  Relative(*sourceDir, path),
				"target":  Relative(*targetDir, TargetFor(path, filepath.Ext(path))),
				"url":     "/" + Relative(*targetDir, TargetFor(path, filepath.Ext(path))),
				"sortkey": filepath.Base(path),
			}
			metadataBuf, _ := splitMetadata(Read(path))
			if len(metadataBuf) > 0 {
				fileMetadata := ParseJSON(metadataBuf)
				s.Add(path, fileMetadata)
			}
			fullMetadata = mergemap.Merge(fullMetadata, s.Get(path))
			SplatInto(m, Relative(*sourceDir, path), fullMetadata)
			Debugf("%s gathered (%d element(s))", path, len(fullMetadata))

		case ".md":
			fullMetadata := map[string]interface{}{
				"source":  Relative(*sourceDir, path),
				"target":  Relative(*targetDir, TargetFor(path, ".html")),
				"url":     "/" + Relative(*targetDir, TargetFor(path, ".html")),
				"sortkey": filepath.Base(path),
			}
			metadataBuf, _ := splitMetadata(Read(path))
			if len(metadataBuf) > 0 {
				fileMetadata := ParseJSON(metadataBuf)
				s.Add(path, fileMetadata)
			}
			fullMetadata = mergemap.Merge(fullMetadata, s.Get(path))
			SplatInto(m, Relative(*sourceDir, path), fullMetadata)
			Debugf("%s gathered (%d element(s))", path, len(fullMetadata))
		}
		return nil
	}
}

func Transform(s StackReader) filepath.WalkFunc {
	Debugf("transforming")
	return func(path string, info os.FileInfo, _ error) error {
		if info.IsDir() {
			Debugf("descending into %s", path)
			return nil // descend
		}

		Debugf("processing %s", path)
		switch filepath.Ext(path) {
		case ".json":
			Debugf("%s ignored for transformation", path)

		case ".html":
			// read
			_, contentBuf := splitMetadata(Read(path))

			// render
			outputBuf := RenderTemplate(path, contentBuf, s.Get(path))

			// write
			dst := TargetFor(path, filepath.Ext(path))
			Write(dst, outputBuf)
			Debugf("%s transformed to %s", path, dst)

		case ".md":
			// read
			_, contentBuf := splitMetadata(Read(path))

			// render
			metadata := mergemap.Merge(s.Get(path), map[string]interface{}{
				"content": template.HTML(RenderMarkdown(contentBuf)),
			})
			templatePath, templateBuf := Template(s, path)
			outputBuf := RenderTemplate(templatePath, templateBuf, metadata)

			// write
			dst := TargetFor(path, ".html")
			Write(dst, outputBuf)
			Debugf("%s transformed to %s", path, dst)

		case ".source", ".template":
			Debugf("%s ignored for transformation", path)

		default:
			dst := TargetFor(path, filepath.Ext(path))
			Copy(dst, path)
			Debugf("%s transformed to %s verbatim", path, dst)
		}
		return nil
	}
}

func RenderTemplate(path string, input []byte, metadata map[string]interface{}) []byte {
	R := func(relativeFilename string) string {
		filename := filepath.Join(filepath.Dir(path), relativeFilename)
		return string(RenderTemplate(filename, Read(filename), metadata))
	}
	importhtml := func(relativeFilename string) template.HTML {
		return template.HTML(R(relativeFilename))
	}
	importcss := func(relativeFilename string) template.CSS {
		return template.CSS(R(relativeFilename))
	}
	importjs := func(relativeFilename string) template.JS {
		return template.JS(R(relativeFilename))
	}

	templateName := Relative(*sourceDir, path)
	funcMap := template.FuncMap{
		"importhtml": importhtml,
		"importcss":  importcss,
		"importjs":   importjs,
		"sorted":     SortedValues,
	}

	tmpl, err := template.New(templateName).Funcs(funcMap).Parse(string(input))
	if err != nil {
		Fatalf("Render Template %s: Parse: %s", path, err)
	}

	output := bytes.Buffer{}
	if err = tmpl.Execute(&output, metadata); err != nil {
		Fatalf("Render Template %s: Execute: %s", path, err)
	}

	return output.Bytes()
}

func RenderMarkdown(input []byte) []byte {
	Debugf("rendering %d byte(s) of Markdown", len(input))
	htmlOptions := 0
	htmlOptions = htmlOptions | blackfriday.HTML_GITHUB_BLOCKCODE
	htmlOptions = htmlOptions | blackfriday.HTML_USE_SMARTYPANTS
	title, css := "", ""
	htmlRenderer := blackfriday.HtmlRenderer(htmlOptions, title, css)

	mdOptions := 0
	mdOptions = mdOptions | blackfriday.EXTENSION_FENCED_CODE

	return blackfriday.Markdown(input, htmlRenderer, mdOptions)
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "os"
    "strings"
)

type MetricConfig struct {
    id string
    label string
    metric_type string
    draw_type string
    color string
}

type Widget struct {
    graph_title string
    graph_category string
    graph_info string
    graph_vlabel string
    graph_period string
    graph_args string
    metrics []MetricConfig
}

var availableWidgets = map[string]Widget{
    "queries": Widget{
        graph_title: "ClickHouse queries",
        graph_category: "clickhouse",
        graph_info: "Values received from ClickHouse system.events table",
        graph_vlabel: "queries / second",
        graph_period: "second",
        graph_args: "--lower-limit 0",
        metrics: []MetricConfig{
            MetricConfig{
                id: "select",
                label: "Selects",
                metric_type: "DERIVE",
                draw_type: "AREA",
                color: "COLOUR0",
            },
            MetricConfig{
                id: "insert",
                label: "Inserts",
                metric_type: "DERIVE",
                draw_type: "STACK",
                color: "COLOUR1",
            },
        },
    },
}

func main() {
    widgetName, action := parseOptions()
    widget, ok := availableWidgets[widgetName]
    if !ok {
        fmt.Printf("Invalid widget name: %s", widgetName)
        os.Exit(1)
    }

    if (action == "config") {
        renderWidgetConfig(widget)
    } else {
        renderWidgetData(widget)
    }
}

func renderWidgetConfig(w Widget) {
    fmt.Printf(`graph_title %s
graph_category %s
graph_info %s
graph_vlabel %s
graph_period %s
graph_args %s
`, w.graph_title, w.graph_category, w.graph_info, w.graph_vlabel, w.graph_period, w.graph_args)
    for _, m := range w.metrics {
        fmt.Printf(`
%s.label %s
%s.type %s
%s.min 0
%s.draw %s
%s.colour %s
`, m.id, m.label, m.id, m.metric_type, m.id, m.id, m.draw_type, m.id, m.color)
    }
}

func renderWidgetData(w Widget) {
    stats := loadClickHouseStats()
    fmt.Printf("RenderWidgetData: %o %d\n", w, stats["Query"])

}

func loadClickHouseStats() map[string]int {
    return map[string]int{
        "Query": 748356,
        "SelectQuery": 289681,
        "InsertQuery": 1038037,
    }
}

func parseOptions() (string, string) {
    widgetName := "queries"
    action := "data"
    args := os.Args
    if (len(args) > 1) {
        action = args[1]
    }
    nameParts := strings.Split(args[0], "_")
    if (len(nameParts) > 1) {
        widgetName = nameParts[1]
    }
    return widgetName, action
}<commit_msg>render data<commit_after>package main

import (
    "fmt"
    "os"
    "strings"
)

type MetricConfig struct {
    id string
    label string
    metric_type string
    draw_type string
    color string
    clickhouseEvent string
}

type Widget struct {
    graph_title string
    graph_category string
    graph_info string
    graph_vlabel string
    graph_period string
    graph_args string
    metrics []MetricConfig
}

var availableWidgets = map[string]Widget{
    "queries": Widget{
        graph_title: "ClickHouse queries",
        graph_category: "clickhouse",
        graph_info: "Values received from ClickHouse system.events table",
        graph_vlabel: "queries / second",
        graph_period: "second",
        graph_args: "--lower-limit 0",
        metrics: []MetricConfig{
            MetricConfig{
                id: "select",
                label: "Selects",
                metric_type: "DERIVE",
                draw_type: "AREA",
                color: "COLOUR0",
                clickhouseEvent: "SelectQuery",
            },
            MetricConfig{
                id: "insert",
                label: "Inserts",
                metric_type: "DERIVE",
                draw_type: "STACK",
                color: "COLOUR1",
                clickhouseEvent: "InsertQuery",
            },
        },
    },
}

func main() {
    widgetName, action := parseOptions()
    widget, ok := availableWidgets[widgetName]
    if !ok {
        fmt.Printf("Invalid widget name: %s", widgetName)
        os.Exit(1)
    }

    if (action == "config") {
        renderWidgetConfig(widget)
    } else {
        renderWidgetData(widget)
    }
}

func renderWidgetConfig(w Widget) {
    fmt.Printf(`graph_title %s
graph_category %s
graph_info %s
graph_vlabel %s
graph_period %s
graph_args %s
`, w.graph_title, w.graph_category, w.graph_info, w.graph_vlabel, w.graph_period, w.graph_args)
    for _, m := range w.metrics {
        fmt.Printf(`
%s.label %s
%s.type %s
%s.min 0
%s.draw %s
%s.colour %s
`, m.id, m.label, m.id, m.metric_type, m.id, m.id, m.draw_type, m.id, m.color)
    }
}

func renderWidgetData(w Widget) {
    stats := loadClickHouseStats()
    for _, m := range w.metrics {
        fmt.Printf("%s.value %d\n", m.id, stats[m.clickhouseEvent])
    }

}

func loadClickHouseStats() map[string]int {
    return map[string]int{
        "Query": 748356,
        "SelectQuery": 289681,
        "InsertQuery": 1038037,
    }
}

func parseOptions() (string, string) {
    widgetName := "queries"
    action := "data"
    args := os.Args
    if (len(args) > 1) {
        action = args[1]
    }
    nameParts := strings.Split(args[0], "_")
    if (len(nameParts) > 1) {
        widgetName = nameParts[1]
    }
    return widgetName, action
}<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/ConSol/sakuli-go-wrapper/execute"
	"github.com/ConSol/sakuli-go-wrapper/helper"
	"github.com/ConSol/sakuli-go-wrapper/input"
	"os"
	"path/filepath"
	"time"
)

func main() {
	var loop int
	var javaHome string
	var javaProperties input.StringSlice
	var preHooks input.StringSlice
	var postHooks input.StringSlice
	var browser string
	var inter string
	var sahiHome string
	var version bool

	sakuliJars := filepath.Join(helper.GetSahiHome(), "libs", "java")
	myFlagSet := flag.NewFlagSet("", flag.ExitOnError)
	input.MyFlagSet = myFlagSet
	myFlagSet.Usage = func() {
		fmt.Fprintf(os.Stderr, `Generic Sakuli test starter.
%d - The Sakuli team / Philip Griesbacher.
http://www.sakuli.org
https://github.com/ConSol/sakuli

Usage:   sakuli[.exe] COMMAND ARGUMENT [OPTIONS]
         sakuli -help
         sakuli -version
         sakuli -run <sakuli suite> [OPTIONS]
         sakuli -encrypt <secret> [OPTIONS]

Commands:
         run <sakuli suite>
         encrypt <secret>

Options:
         -loop=<minutes>           Loop this suite, wait n seconds between
                                   executions, 0 means no loops (default: 0)
         -javaHome=<folder>        Java bin dir (overrides PATH)
         -preHook=<programpath>    A programm which will be executed before
                                   sakuli (Can be added multiple times)
         -postHook=<programpath>   A programm which will be executed after
                                   sakuli (Can be added multiple times)
         -D=<JVM option>           JVM option to set a property on runtime,
                                   overrides the 'sakuli.properties'
         -browser=<browser>        Browser for the test execution
                                   (default: Firefox)
         -interface=<interface>    Network interface used for encryption
         -sahiHome=<folder>        Sahi installation folder
         -version                  Version info
         -help                     This help text

`, time.Now().Year())
	}

	myFlagSet.IntVar(&loop, "loop", 0, "loop this suite, wait n seconds between executions, 0 means no loops (default: 0)")
	myFlagSet.StringVar(&javaHome, "javahome", "", "Java bin dir (overrides PATH)")
	myFlagSet.Var(&preHooks, "preHook", "A programm which will be executed before sakuli (Can be added multiple times)")
	myFlagSet.Var(&postHooks, "postHook", "A programm which will be executed after sakuli (Can be added multiple times)")

	myFlagSet.Var(&javaProperties, "D", "JVM option to set a property on runtime, overrides the 'sakuli.properties'")
	myFlagSet.StringVar(&browser, "browser", "", "browser for the test execution (default: Firefox)")
	myFlagSet.StringVar(&inter, "interface", "", "network interface used for encryption")
	myFlagSet.StringVar(&sahiHome, "sahi_home", "", "Sahi installation folder")
	myFlagSet.BoolVar(&version, "version", false, "version info")

	if len(os.Args) > 2 {
		myFlagSet.Parse(os.Args[3:])
	} else {
		myFlagSet.Parse(os.Args[1:])
		if version {
			input.PrintVersion()
		}
		input.ExitWithHelp("")
	}

	sakuliProperties := map[string]string{"sakuli_home": helper.GetSahiHome()}
	typ, argument := input.ParseArgs(append(os.Args[1:3],myFlagSet.Args()...))
	switch typ {
	case input.RunMode:
		input.TestRun(argument)
		sakuliProperties[input.RunMode] = argument
	case input.EncryptMode:
		sakuliProperties[input.EncryptMode] = argument
	case input.Error:
		panic("can't pars args")
	}

	javaExecutable := input.TestJavaHome(javaHome)
	javaProperties = javaProperties.AddPrefix("-D")

	if browser != "" {
		sakuliProperties["browser"] = browser
	}
	if inter != "" {
		sakuliProperties["interface"] = inter
	}
	if sahiHome != "" {
		sakuliProperties["sahiHome"] = sahiHome
	}
	joinedSakuliProperties := genSakuliPropertiesList(sakuliProperties)

	if len(preHooks) > 0 {
		fmt.Println("=========== Starting Pre-Hooks ===========")
		for _, pre := range preHooks {
			execute.RunHandler(pre)
		}
		fmt.Println("=========== Finished Pre-Hooks ===========")
	}

	sakuliReturnCode := execute.RunSakuli(javaExecutable, sakuliJars, javaProperties, joinedSakuliProperties)
	for loop > 0 {
		fmt.Printf("*** Loop mode - sleeping for %d seconds... ***\n", loop)
		time.Sleep(time.Duration(loop) * time.Second)
		execute.RunSakuli(javaExecutable, sakuliJars, javaProperties, joinedSakuliProperties)
	}

	if len(postHooks) > 0 {
		fmt.Println("=========== Starting Post-Hooks ===========")
		for _, post := range postHooks {
			execute.RunHandler(post)
		}
		fmt.Println("=========== Finished Post-Hooks ===========")
	}
	os.Exit(sakuliReturnCode)
}

func genSakuliPropertiesList(properties map[string]string) input.StringSlice {
	propertiesString := []string{}
	for k, v := range properties {
		propertiesString = append(propertiesString, fmt.Sprintf("--%s", k))
		propertiesString = append(propertiesString, v)
	}
	return propertiesString
}
<commit_msg>main.go: edited help<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/ConSol/sakuli-go-wrapper/execute"
	"github.com/ConSol/sakuli-go-wrapper/helper"
	"github.com/ConSol/sakuli-go-wrapper/input"
	"os"
	"path/filepath"
	"time"
)

func main() {
	var loop int
	var javaHome string
	var javaProperties input.StringSlice
	var preHooks input.StringSlice
	var postHooks input.StringSlice
	var browser string
	var inter string
	var sahiHome string
	var version bool

	sakuliJars := filepath.Join(helper.GetSahiHome(), "libs", "java")
	myFlagSet := flag.NewFlagSet("", flag.ExitOnError)
	input.MyFlagSet = myFlagSet
	myFlagSet.Usage = func() {
		fmt.Fprintf(os.Stderr, `Generic Sakuli test starter.
%d - The Sakuli team / Philip Griesbacher.
http://www.sakuli.org
https://github.com/ConSol/sakuli

Usage:   sakuli[.exe] COMMAND ARGUMENT [OPTIONS]
         sakuli -help
         sakuli -version
         sakuli -run <sakuli suite path> [OPTIONS]
         sakuli -encrypt <secret> [OPTIONS]

Commands:
         run <sakuli suite path>
         encrypt <secret>

Options:
         -loop=<minutes>           Loop this suite, wait n seconds between
                                   executions, 0 means no loops (default: 0)
         -javaHome=<folder>        Java bin dir (overrides PATH)
         -preHook=<programpath>    A program which will be executed before a
                                   suite run (can be added multiple times)
         -postHook=<programpath>   A program which will be executed after a
                                   suite run (can be added multiple times)
         -D=<JVM option>           JVM option to set a property at runtime,
                                   overrides file based properties
         -browser=<browser>        Browser for the test execution
                                   (default: Firefox)
         -interface=<interface>    Network interface card name, used by
                                   command 'encrypt' as salt
         -sahiHome=<folder>        Sahi installation folder
         -version                  Version info
         -help                     This help text

`, time.Now().Year())
	}

	myFlagSet.IntVar(&loop, "loop", 0, "loop this suite, wait n seconds between executions, 0 means no loops (default: 0)")
	myFlagSet.StringVar(&javaHome, "javahome", "", "Java bin dir (overrides PATH)")
	myFlagSet.Var(&preHooks, "preHook", "A program which will be executed before a suite run (can be added multiple times)")
	myFlagSet.Var(&postHooks, "postHook", "A program which will be executed after a suite run (can be added multiple times)")

	myFlagSet.Var(&javaProperties, "D", "JVM option to set a property at runtime, overrides file based properties")
	myFlagSet.StringVar(&browser, "browser", "", "browser for the test execution (default: Firefox)")
	myFlagSet.StringVar(&inter, "interface", "", "network interface icaed name, used by command 'encrypt' as salt")
	myFlagSet.StringVar(&sahiHome, "sahi_home", "", "Sahi installation folder")
	myFlagSet.BoolVar(&version, "version", false, "version info")

	if len(os.Args) > 2 {
		myFlagSet.Parse(os.Args[3:])
	} else {
		myFlagSet.Parse(os.Args[1:])
		if version {
			input.PrintVersion()
		}
		input.ExitWithHelp("")
	}

	sakuliProperties := map[string]string{"sakuli_home": helper.GetSahiHome()}
	typ, argument := input.ParseArgs(append(os.Args[1:3],myFlagSet.Args()...))
	switch typ {
	case input.RunMode:
		input.TestRun(argument)
		sakuliProperties[input.RunMode] = argument
	case input.EncryptMode:
		sakuliProperties[input.EncryptMode] = argument
	case input.Error:
		panic("can't pars args")
	}

	javaExecutable := input.TestJavaHome(javaHome)
	javaProperties = javaProperties.AddPrefix("-D")

	if browser != "" {
		sakuliProperties["browser"] = browser
	}
	if inter != "" {
		sakuliProperties["interface"] = inter
	}
	if sahiHome != "" {
		sakuliProperties["sahiHome"] = sahiHome
	}
	joinedSakuliProperties := genSakuliPropertiesList(sakuliProperties)

	if len(preHooks) > 0 {
		fmt.Println("=========== Starting Pre-Hooks ===========")
		for _, pre := range preHooks {
			execute.RunHandler(pre)
		}
		fmt.Println("=========== Finished Pre-Hooks ===========")
	}

	sakuliReturnCode := execute.RunSakuli(javaExecutable, sakuliJars, javaProperties, joinedSakuliProperties)
	for loop > 0 {
		fmt.Printf("*** Loop mode - sleeping for %d seconds... ***\n", loop)
		time.Sleep(time.Duration(loop) * time.Second)
		execute.RunSakuli(javaExecutable, sakuliJars, javaProperties, joinedSakuliProperties)
	}

	if len(postHooks) > 0 {
		fmt.Println("=========== Starting Post-Hooks ===========")
		for _, post := range postHooks {
			execute.RunHandler(post)
		}
		fmt.Println("=========== Finished Post-Hooks ===========")
	}
	os.Exit(sakuliReturnCode)
}

func genSakuliPropertiesList(properties map[string]string) input.StringSlice {
	propertiesString := []string{}
	for k, v := range properties {
		propertiesString = append(propertiesString, fmt.Sprintf("--%s", k))
		propertiesString = append(propertiesString, v)
	}
	return propertiesString
}
<|endoftext|>


<commit_before>// Package main is a script that reads a filesystem full of dcm files and
// generates a json report.
package main

import (
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	// "strconv"
	"log"
	"strings"

	"github.com/davidgamba/go-dicom/dcmdump/tag"
	"github.com/davidgamba/go-dicom/dcmdump/ts"
	vri "github.com/davidgamba/go-dicom/dcmdump/vr"
	"github.com/davidgamba/go-getoptions"
)

var debug bool

func debugf(format string, a ...interface{}) (n int, err error) {
	if debug {
		return fmt.Printf(format, a...)
	}
	return 0, nil
}
func debugln(a ...interface{}) (n int, err error) {
	if debug {
		return fmt.Println(a...)
	}
	return 0, nil
}

type stringSlice []string

func (s stringSlice) contains(a string) bool {
	for _, b := range s {
		if a == b {
			return true
		}
	}
	return false
}

type dicomqr struct {
	Empty [128]byte
	DICM  [4]byte
	Rest  []byte
}

// DataElement -
type DataElement struct {
	N        int
	TagGroup []byte // [2]byte
	TagElem  []byte // [2]byte
	TagStr   string
	VR       []byte // [2]byte
	VRStr    string
	VRLen    int
	Len      uint32
	Data     []byte
	PartOfSQ bool
}

// String -
func (de *DataElement) String() string {
	tn := tag.Tag[de.TagStr]["name"]
	if _, ok := tag.Tag[de.TagStr]; !ok {
		tn = "MISSING"
	}
	padding := ""
	if de.PartOfSQ {
		padding = "    "
	}
	if de.Len < 128 {
		return fmt.Sprintf("%s%04d (%s) %s %d %d %s %s", padding, de.N, de.TagStr, de.VRStr, de.VRLen, de.Len, tn, de.stringData())
	}
	return fmt.Sprintf("%s%04d (%s) %s %d %d %s %s", padding, de.N, de.TagStr, de.VRStr, de.VRLen, de.Len, tn, "...")
}

type fh os.File

func readNBytes(f *os.File, size int) ([]byte, error) {
	data := make([]byte, size)
	for {
		data = data[:cap(data)]
		n, err := f.Read(data)
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, err
		}
		data = data[:n]
	}
	return data, nil
}

// http://rosettacode.org/wiki/Strip_control_codes_and_extended_characters_from_a_string#Go
// two UTF-8 functions identical except for operator comparing c to 127
func stripCtlFromUTF8(str string) string {
	return strings.Map(func(r rune) rune {
		if r >= 32 && r != 127 {
			return r
		}
		return '.'
	}, str)
}

func tagString(b []byte) string {
	tag := strings.ToUpper(fmt.Sprintf("%02x%02x%02x%02x", b[1], b[0], b[3], b[2]))
	return tag
}

func printBytes(b []byte) {
	if !debug {
		return
	}
	l := len(b)
	var s string
	for i := 0; i < l; i++ {
		s += stripCtlFromUTF8(string(b[i]))
		if i != 0 && i%8 == 0 {
			if i%16 == 0 {
				fmt.Printf(" - %s\n", s)
				s = ""
			} else {
				fmt.Printf(" - ")
			}
		}
		fmt.Printf("%2x ", b[i])
		if i == l-1 {
			if 15-i%16 > 7 {
				fmt.Printf(" - ")
			}
			for j := 0; j < 15-i%16; j++ {
				// fmt.Printf("   ")
				fmt.Printf("   ")
			}
			fmt.Printf(" - %s\n", s)
			s = ""
		}
	}
	fmt.Printf("\n")
}

func (de *DataElement) stringData() string {
	if de.TagStr == "00020010" {
		if tsStr, ok := ts.TS[string(de.Data)]; ok {
			return string(de.Data) + " " + tsStr["name"].(string)
		}
	}
	if _, ok := vri.VR[de.VRStr]["fixed"]; ok && vri.VR[de.VRStr]["fixed"].(bool) {
		s := ""
		l := len(de.Data)
		n := 0
		vrl := vri.VR[de.VRStr]["len"].(int)
		switch vrl {
		case 1:
			for n+1 <= l {
				s += fmt.Sprintf("%d ", de.Data[n])
				n++
			}
			return s
		case 2:
			for n+2 <= l {
				e := binary.LittleEndian.Uint16(de.Data[n : n+2])
				s += fmt.Sprintf("%d ", e)
				n += 2
			}
			return s
		case 4:
			for n+4 <= l {
				e := binary.LittleEndian.Uint32(de.Data[n : n+4])
				s += fmt.Sprintf("%d ", e)
				n += 4
			}
			return s
		default:
			return string(de.Data)
		}
	} else {
		if _, ok := vri.VR[de.VRStr]["padded"]; ok && vri.VR[de.VRStr]["padded"].(bool) {
			l := len(de.Data)
			if de.Data[l-1] == 0x0 {
				return string(de.Data[:l-1])
			}
			return string(de.Data)
		}
		return string(de.Data)
	}
}

func parseDataElement(bytes []byte, n int, explicit bool) {
	log.Printf("parseDataElement")
	l := len(bytes)
	// Data element
	m := n
	for n <= l && m+4 <= l {
		undefinedLen := false
		de := DataElement{N: n}
		m += 4
		t := bytes[n:m]
		de.TagGroup = bytes[n : n+2]
		de.TagElem = bytes[n+2 : n+4]
		de.TagStr = tagString(t)
		// TODO: Clean up tagString
		tagStr := tagString(t)
		log.Printf("n: %d, Tag: %X -> %s\n", n, t, tagStr)
		printBytes(bytes[n:m])
		n = m
		if tagStr == "" {
			log.Printf("%d Empty Tag: %s\n", n, tagStr)
		} else if _, ok := tag.Tag[tagStr]; !ok {
			fmt.Fprintf(os.Stderr, "ERROR: %d Missing tag '%s'\n", n, tagStr)
		} else {
			log.Printf("Tag Name: %s\n", tag.Tag[tagStr]["name"])
		}
		var len uint32
		var vr string
		if explicit {
			debugf("%d VR\n", n)
			m += 2
			printBytes(bytes[n:m])
			de.VR = bytes[n:m]
			de.VRStr = string(bytes[n:m])
			vr = string(bytes[n:m])
			if _, ok := vri.VR[vr]; !ok {
				// if bytes[n] == 0x0 && bytes[n+1] == 0x0 {
				// 	fmt.Fprintf(os.Stderr, "ERROR: Blank VR\n")
				// } else {
				fmt.Fprintf(os.Stderr, "ERROR: %d Missing VR '%s'\n", n, vr)
				printBytes(bytes[n:])
				return
				// }
			}
			n = m
			if vr == "OB" ||
				vr == "OD" ||
				vr == "OF" ||
				vr == "OL" ||
				vr == "OW" ||
				vr == "SQ" ||
				vr == "UC" ||
				vr == "UR" ||
				vr == "UT" ||
				vr == "UN" {
				debugln("Reserved")
				m += 2
				printBytes(bytes[n:m])
				n = m
				debugln("Lenght")
				m += 4
				printBytes(bytes[n:m])
				len = binary.LittleEndian.Uint32(bytes[n:m])
				n = m
			} else {
				debugln("Lenght")
				m += 2
				printBytes(bytes[n:m])
				len16 := binary.LittleEndian.Uint16(bytes[n:m])
				len = uint32(len16)
				n = m
			}
		} else {
			debugln("Lenght")
			m += 4
			printBytes(bytes[n:m])
			len = binary.LittleEndian.Uint32(bytes[n:m])
			n = m
		}
		if len == 0xFFFFFFFF {
			undefinedLen = true
			for {
				// Find FFFEE0DD: SequenceDelimitationItem
				endTag := bytes[m : m+4]
				endTagStr := tagString(endTag)
				if endTagStr == "FFFEE00D" || endTagStr == "FFFEE0DD" {
					log.Printf("found SequenceDelimitationItem")
					len = uint32(m - n)
					m = n
					break
				} else {
					m++
					if m >= l {
						fmt.Fprintf(os.Stderr, "ERROR: Couldn't find SequenceDelimitationItem\n")
						printBytes(bytes[n:l])
						return
					}
				}
			}
		}
		de.Len = len
		debugf("Lenght: %d\n", len)
		m += int(len)
		printBytes(bytes[n:m])
		if vr == "SQ" || vr == "OW" {
			de.Data = []byte{}
			log.Printf("parseDataElement SQ")
			parseDataElement(bytes[n:m], n, explicit)
		} else {
			de.Data = bytes[n:m]
			fmt.Println(de.String())
		}
		if undefinedLen {
			m += 8
		}
		n = m
	}
	log.Printf("parseDataElement Complete")
}

func parseSQDataElements(bytes []byte, n int, explicit bool) int {
	log.Printf("parseSQDataElements")
	l := len(bytes)
	m := n
	for n <= l && m+4 <= l {
		de := DataElement{N: n}
		m := n + 4
		printBytes(bytes[n:m])
		t := bytes[n:m]
		tagStr := tagString(t)
		de.TagGroup = bytes[n : n+2]
		de.TagElem = bytes[n+2 : n+4]
		de.TagStr = tagString(t)
		log.Printf("n: %d, Tag: %X -> %s\n", n, t, tagStr)
		if _, ok := tag.Tag[tagStr]; !ok {
			fmt.Fprintf(os.Stderr, "ERROR: %d Missing tag '%s'\n", n, tagStr)
		}
		// if _, ok := tag.Tag[tagStr]; ok && tag.Tag[tagStr]["name"] == "ItemDelimitationItem" {
		// 	sequenceDelimitationItem = true
		// }
		for m <= l {
			// Find FFFEE00D: ItemDelimitationItem
			endTag := bytes[m : m+4]
			endTagStr := tagString(endTag)
			if endTagStr == "FFFEE00D" {
				debugln("Item Delim found")
				de.Data = bytes[n:m]
				printBytes(bytes[n:m])
				log.Printf("Tag: %X -> %s\n", endTag, endTagStr)
				m += 4
				n = m
				// m += 4
				// printBytes(bytes[n:m])
				// n = m
				break
			} else {
				m++
			}
		}
		fmt.Println(de.String())
	}
	log.Printf("parseSQDataElement Complete")
	return n
}

func synopsis() {
	synopsis := `dcmdump <dcm_file> [--debug]
`
	fmt.Fprintln(os.Stderr, synopsis)
}

func main() {

	var file string
	opt := getoptions.New()
	opt.Bool("help", false)
	opt.BoolVar(&debug, "debug", false)
	remaining, err := opt.Parse(os.Args[1:])
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
		os.Exit(1)
	}
	if opt.Called("help") {
		synopsis()
		os.Exit(1)
	}
	if len(remaining) < 1 {
		fmt.Fprintf(os.Stderr, "ERROR: Missing file\n")
		synopsis()
		os.Exit(1)
	}
	file = remaining[0]
	if !debug {
		log.SetOutput(ioutil.Discard)
	}
	bytes, err := ioutil.ReadFile(file)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: failed to read file: '%s'\n", err)
		os.Exit(1)
	}

	// Intro
	n := 128
	printBytes(bytes[0:n])
	// DICM
	m := n + 4
	printBytes(bytes[n:m])
	n = m

	explicit := true

	parseDataElement(bytes, n, explicit)
}
<commit_msg>dcmdump: Somewhat handle blank VR<commit_after>// Package main is a script that reads a filesystem full of dcm files and
// generates a json report.
package main

import (
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	// "strconv"
	"log"
	"strings"

	"github.com/davidgamba/go-dicom/dcmdump/tag"
	"github.com/davidgamba/go-dicom/dcmdump/ts"
	vri "github.com/davidgamba/go-dicom/dcmdump/vr"
	"github.com/davidgamba/go-getoptions"
)

var debug bool

func debugf(format string, a ...interface{}) (n int, err error) {
	if debug {
		return fmt.Printf(format, a...)
	}
	return 0, nil
}
func debugln(a ...interface{}) (n int, err error) {
	if debug {
		return fmt.Println(a...)
	}
	return 0, nil
}

type stringSlice []string

func (s stringSlice) contains(a string) bool {
	for _, b := range s {
		if a == b {
			return true
		}
	}
	return false
}

type dicomqr struct {
	Empty [128]byte
	DICM  [4]byte
	Rest  []byte
}

// DataElement -
type DataElement struct {
	N        int
	TagGroup []byte // [2]byte
	TagElem  []byte // [2]byte
	TagStr   string
	VR       []byte // [2]byte
	VRStr    string
	VRLen    int
	Len      uint32
	Data     []byte
	PartOfSQ bool
}

// String -
func (de *DataElement) String() string {
	tn := tag.Tag[de.TagStr]["name"]
	if _, ok := tag.Tag[de.TagStr]; !ok {
		tn = "MISSING"
	}
	padding := ""
	if de.PartOfSQ {
		padding = "    "
	}
	if de.Len < 128 {
		return fmt.Sprintf("%s%04d (%s) %s %d %d %s %s", padding, de.N, de.TagStr, de.VRStr, de.VRLen, de.Len, tn, de.stringData())
	}
	return fmt.Sprintf("%s%04d (%s) %s %d %d %s %s", padding, de.N, de.TagStr, de.VRStr, de.VRLen, de.Len, tn, "...")
}

type fh os.File

func readNBytes(f *os.File, size int) ([]byte, error) {
	data := make([]byte, size)
	for {
		data = data[:cap(data)]
		n, err := f.Read(data)
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, err
		}
		data = data[:n]
	}
	return data, nil
}

// http://rosettacode.org/wiki/Strip_control_codes_and_extended_characters_from_a_string#Go
// two UTF-8 functions identical except for operator comparing c to 127
func stripCtlFromUTF8(str string) string {
	return strings.Map(func(r rune) rune {
		if r >= 32 && r != 127 {
			return r
		}
		return '.'
	}, str)
}

func tagString(b []byte) string {
	tag := strings.ToUpper(fmt.Sprintf("%02x%02x%02x%02x", b[1], b[0], b[3], b[2]))
	return tag
}

func printBytes(b []byte) {
	if !debug {
		return
	}
	l := len(b)
	var s string
	for i := 0; i < l; i++ {
		s += stripCtlFromUTF8(string(b[i]))
		if i != 0 && i%8 == 0 {
			if i%16 == 0 {
				fmt.Printf(" - %s\n", s)
				s = ""
			} else {
				fmt.Printf(" - ")
			}
		}
		fmt.Printf("%2x ", b[i])
		if i == l-1 {
			if 15-i%16 > 7 {
				fmt.Printf(" - ")
			}
			for j := 0; j < 15-i%16; j++ {
				// fmt.Printf("   ")
				fmt.Printf("   ")
			}
			fmt.Printf(" - %s\n", s)
			s = ""
		}
	}
	fmt.Printf("\n")
}

func (de *DataElement) stringData() string {
	if de.TagStr == "00020010" {
		if tsStr, ok := ts.TS[string(de.Data)]; ok {
			return string(de.Data) + " " + tsStr["name"].(string)
		}
	}
	if _, ok := vri.VR[de.VRStr]["fixed"]; ok && vri.VR[de.VRStr]["fixed"].(bool) {
		s := ""
		l := len(de.Data)
		n := 0
		vrl := vri.VR[de.VRStr]["len"].(int)
		switch vrl {
		case 1:
			for n+1 <= l {
				s += fmt.Sprintf("%d ", de.Data[n])
				n++
			}
			return s
		case 2:
			for n+2 <= l {
				e := binary.LittleEndian.Uint16(de.Data[n : n+2])
				s += fmt.Sprintf("%d ", e)
				n += 2
			}
			return s
		case 4:
			for n+4 <= l {
				e := binary.LittleEndian.Uint32(de.Data[n : n+4])
				s += fmt.Sprintf("%d ", e)
				n += 4
			}
			return s
		default:
			return string(de.Data)
		}
	} else {
		if _, ok := vri.VR[de.VRStr]["padded"]; ok && vri.VR[de.VRStr]["padded"].(bool) {
			l := len(de.Data)
			if de.Data[l-1] == 0x0 {
				return string(de.Data[:l-1])
			}
			return string(de.Data)
		}
		return string(de.Data)
	}
}

func parseDataElement(bytes []byte, n int, explicit bool) {
	log.Printf("parseDataElement")
	l := len(bytes)
	// Data element
	m := n
	for n <= l && m+4 <= l {
		undefinedLen := false
		de := DataElement{N: n}
		m += 4
		t := bytes[n:m]
		de.TagGroup = bytes[n : n+2]
		de.TagElem = bytes[n+2 : n+4]
		de.TagStr = tagString(t)
		// TODO: Clean up tagString
		tagStr := tagString(t)
		log.Printf("n: %d, Tag: %X -> %s\n", n, t, tagStr)
		printBytes(bytes[n:m])
		n = m
		if tagStr == "" {
			log.Printf("%d Empty Tag: %s\n", n, tagStr)
		} else if _, ok := tag.Tag[tagStr]; !ok {
			fmt.Fprintf(os.Stderr, "INFO: %d Missing tag '%s'\n", n, tagStr)
		} else {
			log.Printf("Tag Name: %s\n", tag.Tag[tagStr]["name"])
		}
		var len uint32
		var vr string
		if explicit {
			debugf("%d VR\n", n)
			m += 2
			printBytes(bytes[n:m])
			de.VR = bytes[n:m]
			de.VRStr = string(bytes[n:m])
			vr = string(bytes[n:m])
			if _, ok := vri.VR[vr]; !ok {
				if bytes[n] == 0x0 && bytes[n+1] == 0x0 {
					fmt.Fprintf(os.Stderr, "INFO: Blank VR\n")
					vr = "00"
					de.VRStr = "00"
				} else {
					fmt.Fprintf(os.Stderr, "ERROR: %d Missing VR '%s'\n", n, vr)
					printBytes(bytes[n:])
					return
				}
			}
			n = m
			if vr == "OB" ||
				vr == "OD" ||
				vr == "OF" ||
				vr == "OL" ||
				vr == "OW" ||
				vr == "SQ" ||
				vr == "UC" ||
				vr == "UR" ||
				vr == "UT" ||
				vr == "UN" {
				debugln("Reserved")
				m += 2
				printBytes(bytes[n:m])
				n = m
				debugln("Lenght")
				m += 4
				printBytes(bytes[n:m])
				len = binary.LittleEndian.Uint32(bytes[n:m])
				n = m
			} else {
				debugln("Lenght")
				m += 2
				printBytes(bytes[n:m])
				len16 := binary.LittleEndian.Uint16(bytes[n:m])
				len = uint32(len16)
				n = m
			}
		} else {
			debugln("Lenght")
			m += 4
			printBytes(bytes[n:m])
			len = binary.LittleEndian.Uint32(bytes[n:m])
			n = m
		}
		if len == 0xFFFFFFFF {
			undefinedLen = true
			for {
				// Find FFFEE0DD: SequenceDelimitationItem
				endTag := bytes[m : m+4]
				endTagStr := tagString(endTag)
				if endTagStr == "FFFEE00D" || endTagStr == "FFFEE0DD" {
					log.Printf("found SequenceDelimitationItem")
					len = uint32(m - n)
					m = n
					break
				} else {
					m++
					if m >= l {
						fmt.Fprintf(os.Stderr, "ERROR: Couldn't find SequenceDelimitationItem\n")
						printBytes(bytes[n:l])
						return
					}
				}
			}
		}
		de.Len = len
		debugf("Lenght: %d\n", len)
		m += int(len)
		printBytes(bytes[n:m])
		if vr == "SQ" || vr == "OW" {
			de.Data = []byte{}
			log.Printf("parseDataElement SQ")
			parseDataElement(bytes[n:m], n, explicit)
		} else {
			de.Data = bytes[n:m]
			fmt.Println(de.String())
		}
		if undefinedLen {
			m += 8
		}
		n = m
	}
	log.Printf("parseDataElement Complete")
}

func parseSQDataElements(bytes []byte, n int, explicit bool) int {
	log.Printf("parseSQDataElements")
	l := len(bytes)
	m := n
	for n <= l && m+4 <= l {
		de := DataElement{N: n}
		m := n + 4
		printBytes(bytes[n:m])
		t := bytes[n:m]
		tagStr := tagString(t)
		de.TagGroup = bytes[n : n+2]
		de.TagElem = bytes[n+2 : n+4]
		de.TagStr = tagString(t)
		log.Printf("n: %d, Tag: %X -> %s\n", n, t, tagStr)
		if _, ok := tag.Tag[tagStr]; !ok {
			fmt.Fprintf(os.Stderr, "ERROR: %d Missing tag '%s'\n", n, tagStr)
		}
		// if _, ok := tag.Tag[tagStr]; ok && tag.Tag[tagStr]["name"] == "ItemDelimitationItem" {
		// 	sequenceDelimitationItem = true
		// }
		for m <= l {
			// Find FFFEE00D: ItemDelimitationItem
			endTag := bytes[m : m+4]
			endTagStr := tagString(endTag)
			if endTagStr == "FFFEE00D" {
				debugln("Item Delim found")
				de.Data = bytes[n:m]
				printBytes(bytes[n:m])
				log.Printf("Tag: %X -> %s\n", endTag, endTagStr)
				m += 4
				n = m
				// m += 4
				// printBytes(bytes[n:m])
				// n = m
				break
			} else {
				m++
			}
		}
		fmt.Println(de.String())
	}
	log.Printf("parseSQDataElement Complete")
	return n
}

func synopsis() {
	synopsis := `dcmdump <dcm_file> [--debug]
`
	fmt.Fprintln(os.Stderr, synopsis)
}

func main() {

	var file string
	opt := getoptions.New()
	opt.Bool("help", false)
	opt.BoolVar(&debug, "debug", false)
	remaining, err := opt.Parse(os.Args[1:])
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
		os.Exit(1)
	}
	if opt.Called("help") {
		synopsis()
		os.Exit(1)
	}
	if len(remaining) < 1 {
		fmt.Fprintf(os.Stderr, "ERROR: Missing file\n")
		synopsis()
		os.Exit(1)
	}
	file = remaining[0]
	if !debug {
		log.SetOutput(ioutil.Discard)
	}
	bytes, err := ioutil.ReadFile(file)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: failed to read file: '%s'\n", err)
		os.Exit(1)
	}

	// Intro
	n := 128
	printBytes(bytes[0:n])
	// DICM
	m := n + 4
	printBytes(bytes[n:m])
	n = m

	explicit := true

	parseDataElement(bytes, n, explicit)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file is a modified version of net/hosts.go from the golang repo

package hosts

import (
	"bufio"
	"bytes"
	"io"
	"net"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/coredns/coredns/plugin"
)

const cacheMaxAge = 5 * time.Second

func parseLiteralIP(addr string) net.IP {
	if i := strings.Index(addr, "%"); i >= 0 {
		// discard ipv6 zone
		addr = addr[0:i]
	}

	return net.ParseIP(addr)
}

func absDomainName(b string) string {
	return plugin.Name(b).Normalize()
}

// Hostsfile contains known host entries.
type Hostsfile struct {
	sync.Mutex

	// list of zones we are authoritive for
	Origins []string

	// Key for the list of literal IP addresses must be a host
	// name. It would be part of DNS labels, a FQDN or an absolute
	// FQDN.
	// For now the key is converted to lower case for convenience.
	byNameV4 map[string][]net.IP
	byNameV6 map[string][]net.IP

	// Key for the list of host names must be a literal IP address
	// including IPv6 address with zone identifier.
	// We don't support old-classful IP address notation.
	byAddr map[string][]string

	// inline saves the hosts file is inlined in Corefile
	// We need a copy here as we want to use inline to override
	// the default /etc/hosts
	inline []string

	expire time.Time
	path   string
	mtime  time.Time
	size   int64
}

// ReadHosts determines if the cached data needs to be updated based on the size and modification time of the hostsfile.
func (h *Hostsfile) ReadHosts() {
	now := time.Now()

	if now.Before(h.expire) && len(h.byAddr) > 0 {
		return
	}
	stat, err := os.Stat(h.path)
	if err == nil && h.mtime.Equal(stat.ModTime()) && h.size == stat.Size() {
		h.expire = now.Add(cacheMaxAge)
		return
	}

	var file *os.File
	if file, _ = os.Open(h.path); file == nil {
		// If this is the first time then we will try to parse inline
		if len(h.byAddr) == 0 && len(h.inline) > 0 {
			h.Parse(nil)
		}
		return
	}
	defer file.Close()

	h.Parse(file)

	// Update the data cache.
	h.expire = now.Add(cacheMaxAge)
	h.mtime = stat.ModTime()
	h.size = stat.Size()
}

// Parse reads the hostsfile and populates the byName and byAddr maps.
func (h *Hostsfile) Parse(file io.Reader) {
	hsv4 := make(map[string][]net.IP)
	hsv6 := make(map[string][]net.IP)
	is := make(map[string][]string)

	var readers []io.Reader
	if file != nil {
		readers = append(readers, file)
	}
	readers = append(readers, strings.NewReader(strings.Join(h.inline, "\n")))
	scanner := bufio.NewScanner(io.MultiReader(readers...))
	for scanner.Scan() {
		line := scanner.Bytes()
		if i := bytes.Index(line, []byte{'#'}); i >= 0 {
			// Discard comments.
			line = line[0:i]
		}
		f := bytes.Fields(line)
		if len(f) < 2 {
			continue
		}
		addr := parseLiteralIP(string(f[0]))
		if addr == nil {
			continue
		}
		ver := ipVersion(string(f[0]))
		for i := 1; i < len(f); i++ {
			name := absDomainName(string(f[i]))
			if plugin.Zones(h.Origins).Matches(name) == "" {
				// name is not in Origins
				continue
			}
			switch ver {
			case 4:
				hsv4[name] = append(hsv4[name], addr)
			case 6:
				hsv6[name] = append(hsv6[name], addr)
			default:
				continue
			}
			is[addr.String()] = append(is[addr.String()], name)
		}
	}
	h.byNameV4 = hsv4
	h.byNameV6 = hsv6
	h.byAddr = is
}

// ipVersion returns what IP version was used textually
func ipVersion(s string) int {
	for i := 0; i < len(s); i++ {
		switch s[i] {
		case '.':
			return 4
		case ':':
			return 6
		}
	}
	return 0
}

// LookupStaticHostV4 looks up the IPv4 addresses for the given host from the hosts file.
func (h *Hostsfile) LookupStaticHostV4(host string) []net.IP {
	h.Lock()
	defer h.Unlock()
	h.ReadHosts()
	if len(h.byNameV4) != 0 {
		if ips, ok := h.byNameV4[absDomainName(host)]; ok {
			ipsCp := make([]net.IP, len(ips))
			copy(ipsCp, ips)
			return ipsCp
		}
	}
	return nil
}

// LookupStaticHostV6 looks up the IPv6 addresses for the given host from the hosts file.
func (h *Hostsfile) LookupStaticHostV6(host string) []net.IP {
	h.Lock()
	defer h.Unlock()
	h.ReadHosts()
	if len(h.byNameV6) != 0 {
		if ips, ok := h.byNameV6[absDomainName(host)]; ok {
			ipsCp := make([]net.IP, len(ips))
			copy(ipsCp, ips)
			return ipsCp
		}
	}
	return nil
}

// LookupStaticAddr looks up the hosts for the given address from the hosts file.
func (h *Hostsfile) LookupStaticAddr(addr string) []string {
	h.Lock()
	defer h.Unlock()
	h.ReadHosts()
	addr = parseLiteralIP(addr).String()
	if addr == "" {
		return nil
	}
	if len(h.byAddr) != 0 {
		if hosts, ok := h.byAddr[addr]; ok {
			hostsCp := make([]string, len(hosts))
			copy(hostsCp, hosts)
			return hostsCp
		}
	}
	return nil
}
<commit_msg>Fix locking for hosts plugin<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file is a modified version of net/hosts.go from the golang repo

package hosts

import (
	"bufio"
	"bytes"
	"io"
	"net"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/coredns/coredns/plugin"
)

const cacheMaxAge = 5 * time.Second

func parseLiteralIP(addr string) net.IP {
	if i := strings.Index(addr, "%"); i >= 0 {
		// discard ipv6 zone
		addr = addr[0:i]
	}

	return net.ParseIP(addr)
}

func absDomainName(b string) string {
	return plugin.Name(b).Normalize()
}

// Hostsfile contains known host entries.
type Hostsfile struct {
	sync.RWMutex

	// list of zones we are authoritive for
	Origins []string

	// Key for the list of literal IP addresses must be a host
	// name. It would be part of DNS labels, a FQDN or an absolute
	// FQDN.
	// For now the key is converted to lower case for convenience.
	byNameV4 map[string][]net.IP
	byNameV6 map[string][]net.IP

	// Key for the list of host names must be a literal IP address
	// including IPv6 address with zone identifier.
	// We don't support old-classful IP address notation.
	byAddr map[string][]string

	// inline saves the hosts file is inlined in Corefile
	// We need a copy here as we want to use inline to override
	// the default /etc/hosts
	inline []string

	expire time.Time
	path   string
	mtime  time.Time
	size   int64
}

// ReadHosts determines if the cached data needs to be updated based on the size and modification time of the hostsfile.
func (h *Hostsfile) ReadHosts() {
	now := time.Now()

	if now.Before(h.expire) && len(h.byAddr) > 0 {
		return
	}
	stat, err := os.Stat(h.path)
	if err == nil && h.mtime.Equal(stat.ModTime()) && h.size == stat.Size() {
		h.expire = now.Add(cacheMaxAge)
		return
	}

	var file *os.File
	if file, _ = os.Open(h.path); file == nil {
		// If this is the first time then we will try to parse inline
		if len(h.byAddr) == 0 && len(h.inline) > 0 {
			h.Parse(nil)
		}
		return
	}
	defer file.Close()

	h.Parse(file)

	// Update the data cache.
	h.expire = now.Add(cacheMaxAge)
	h.mtime = stat.ModTime()
	h.size = stat.Size()
}

// Parse reads the hostsfile and populates the byName and byAddr maps.
func (h *Hostsfile) Parse(file io.Reader) {
	hsv4 := make(map[string][]net.IP)
	hsv6 := make(map[string][]net.IP)
	is := make(map[string][]string)

	var readers []io.Reader
	if file != nil {
		readers = append(readers, file)
	}
	readers = append(readers, strings.NewReader(strings.Join(h.inline, "\n")))
	scanner := bufio.NewScanner(io.MultiReader(readers...))
	for scanner.Scan() {
		line := scanner.Bytes()
		if i := bytes.Index(line, []byte{'#'}); i >= 0 {
			// Discard comments.
			line = line[0:i]
		}
		f := bytes.Fields(line)
		if len(f) < 2 {
			continue
		}
		addr := parseLiteralIP(string(f[0]))
		if addr == nil {
			continue
		}
		ver := ipVersion(string(f[0]))
		for i := 1; i < len(f); i++ {
			name := absDomainName(string(f[i]))
			if plugin.Zones(h.Origins).Matches(name) == "" {
				// name is not in Origins
				continue
			}
			switch ver {
			case 4:
				hsv4[name] = append(hsv4[name], addr)
			case 6:
				hsv6[name] = append(hsv6[name], addr)
			default:
				continue
			}
			is[addr.String()] = append(is[addr.String()], name)
		}
	}
	h.Lock()
	defer h.Unlock()
	h.byNameV4 = hsv4
	h.byNameV6 = hsv6
	h.byAddr = is
}

// ipVersion returns what IP version was used textually
func ipVersion(s string) int {
	for i := 0; i < len(s); i++ {
		switch s[i] {
		case '.':
			return 4
		case ':':
			return 6
		}
	}
	return 0
}

// LookupStaticHostV4 looks up the IPv4 addresses for the given host from the hosts file.
func (h *Hostsfile) LookupStaticHostV4(host string) []net.IP {
	h.RLock()
	defer h.RUnlock()
	h.ReadHosts()
	if len(h.byNameV4) != 0 {
		if ips, ok := h.byNameV4[absDomainName(host)]; ok {
			ipsCp := make([]net.IP, len(ips))
			copy(ipsCp, ips)
			return ipsCp
		}
	}
	return nil
}

// LookupStaticHostV6 looks up the IPv6 addresses for the given host from the hosts file.
func (h *Hostsfile) LookupStaticHostV6(host string) []net.IP {
	h.RLock()
	defer h.RUnlock()
	h.ReadHosts()
	if len(h.byNameV6) != 0 {
		if ips, ok := h.byNameV6[absDomainName(host)]; ok {
			ipsCp := make([]net.IP, len(ips))
			copy(ipsCp, ips)
			return ipsCp
		}
	}
	return nil
}

// LookupStaticAddr looks up the hosts for the given address from the hosts file.
func (h *Hostsfile) LookupStaticAddr(addr string) []string {
	h.RLock()
	defer h.RUnlock()
	h.ReadHosts()
	addr = parseLiteralIP(addr).String()
	if addr == "" {
		return nil
	}
	if len(h.byAddr) != 0 {
		if hosts, ok := h.byAddr[addr]; ok {
			hostsCp := make([]string, len(hosts))
			copy(hostsCp, hosts)
			return hostsCp
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package odb

import (
	"bufio"
	"bytes"
	"encoding/hex"
	"fmt"
	"io"
	"strings"
	"time"
)

// Signature represents a commit signature, which can represent either
// committership or authorship of the commit that this signature belongs to. It
// specifies a name, email, and time that the signature was created.
//
// NOTE: this type is _not_ used by the `*Commit` instance, as it does not
// preserve cruft bytes. It is kept as a convenience type to test with.
type Signature struct {
	// Name is the first and last name of the individual holding this
	// signature.
	Name string
	// Email is the email address of the individual holding this signature.
	Email string
	// When is the instant in time when the signature was created.
	When time.Time
}

const (
	formatTimeZoneOnly = "-0700"
)

// String implements the fmt.Stringer interface and formats a Signature as
// expected in the Git commit internal object format. For instance:
//
//  Taylor Blau <ttaylorr@github.com> 1494258422 -0600
func (s *Signature) String() string {
	at := s.When.Unix()
	zone := s.When.Format(formatTimeZoneOnly)

	return fmt.Sprintf("%s <%s> %d %s", s.Name, s.Email, at, zone)
}

// ExtraHeader encapsulates a key-value pairing of header key to header value.
// It is stored as a struct{string, string} in memory as opposed to a
// map[string]string to maintain ordering in a byte-for-byte encode/decode round
// trip.
type ExtraHeader struct {
	// K is the header key, or the first run of bytes up until a ' ' (\x20)
	// character.
	K string
	// V is the header value, or the remaining run of bytes in the line,
	// stripping off the above "K" field as a prefix.
	V string
}

// Commit encapsulates a Git commit entry.
type Commit struct {
	// Author is the Author this commit, or the original writer of the
	// contents.
	//
	// NOTE: this field is stored as a string to ensure any extra "cruft"
	// bytes are preserved through migration.
	Author string
	// Committer is the individual or entity that added this commit to the
	// history.
	//
	// NOTE: this field is stored as a string to ensure any extra "cruft"
	// bytes are preserved through migration.
	Committer string
	// ParentIDs are the IDs of all parents for which this commit is a
	// linear child.
	ParentIDs [][]byte
	// TreeID is the root Tree associated with this commit.
	TreeID []byte
	// ExtraHeaders stores headers not listed above, for instance
	// "encoding", "gpgsig", or "mergetag" (among others).
	ExtraHeaders []*ExtraHeader
	// Message is the commit message, including any signing information
	// associated with this commit.
	Message string
}

// Type implements Object.ObjectType by returning the correct object type for
// Commits, CommitObjectType.
func (c *Commit) Type() ObjectType { return CommitObjectType }

// Decode implements Object.Decode and decodes the uncompressed commit being
// read. It returns the number of uncompressed bytes being consumed off of the
// stream, which should be strictly equal to the size given.
//
// If any error was encountered along the way, that will be returned, along with
// the number of bytes read up to that point.
func (c *Commit) Decode(from io.Reader, size int64) (n int, err error) {
	var finishedHeaders bool
	var messageParts []string

	s := bufio.NewScanner(from)
	for s.Scan() {
		text := s.Text()
		n = n + len(text+"\n")

		if len(s.Text()) == 0 {
			finishedHeaders = true
			continue
		}

		if fields := strings.Fields(text); len(fields) > 0 && !finishedHeaders {
			switch fields[0] {
			case "tree":
				id, err := hex.DecodeString(fields[1])
				if err != nil {
					return n, err
				}
				c.TreeID = id
			case "parent":
				id, err := hex.DecodeString(fields[1])
				if err != nil {
					return n, err
				}
				c.ParentIDs = append(c.ParentIDs, id)
			case "author":
				c.Author = strings.Join(fields[1:], " ")
			case "committer":
				c.Committer = strings.Join(fields[1:], " ")
			default:
				c.ExtraHeaders = append(c.ExtraHeaders, &ExtraHeader{
					K: fields[0],
					V: strings.Join(fields[1:], " "),
				})
			}
		} else {
			messageParts = append(messageParts, s.Text())
		}
	}

	c.Message = strings.Join(messageParts, "\n")

	if err = s.Err(); err != nil {
		return n, err
	}
	return n, err
}

// Encode encodes the commit's contents to the given io.Writer, "w". If there was
// any error copying the commit's contents, that error will be returned.
//
// Otherwise, the number of bytes written will be returned.
func (c *Commit) Encode(to io.Writer) (n int, err error) {
	n, err = fmt.Fprintf(to, "tree %s\n", hex.EncodeToString(c.TreeID))
	if err != nil {
		return n, err
	}

	for _, pid := range c.ParentIDs {
		n1, err := fmt.Fprintf(to, "parent %s\n", hex.EncodeToString(pid))
		if err != nil {
			return n, err
		}

		n = n + n1
	}

	n2, err := fmt.Fprintf(to, "author %s\ncommitter %s\n", c.Author, c.Committer)
	if err != nil {
		return n, err
	}

	n = n + n2

	for _, hdr := range c.ExtraHeaders {
		n3, err := fmt.Fprintf(to, "%s %s\n", hdr.K, hdr.V)
		if err != nil {
			return n, err
		}

		n = n + n3
	}

	n4, err := fmt.Fprintf(to, "\n%s\n", c.Message)
	if err != nil {
		return n, err
	}

	return n + n4, err
}

// Equal returns whether the receiving and given commits are equal, or in other
// words, whether they are represented by the same SHA-1 when saved to the
// object database.
func (c *Commit) Equal(other *Commit) bool {
	if (c == nil) != (other == nil) {
		return false
	}

	if c != nil {
		if len(c.ParentIDs) != len(other.ParentIDs) {
			return false
		}
		for i := 0; i < len(c.ParentIDs); i++ {
			p1 := c.ParentIDs[i]
			p2 := other.ParentIDs[i]

			if !bytes.Equal(p1, p2) {
				return false
			}
		}

		if len(c.ExtraHeaders) != len(other.ExtraHeaders) {
			return false
		}
		for i := 0; i < len(c.ExtraHeaders); i++ {
			e1 := c.ExtraHeaders[i]
			e2 := other.ExtraHeaders[i]

			if e1.K != e2.K || e1.V != e2.V {
				return false
			}
		}

		return c.Author == other.Author &&
			c.Committer == other.Committer &&
			c.Message == other.Message &&
			bytes.Equal(c.TreeID, other.TreeID)
	}
	return true
}
<commit_msg>odb: extract commit author/committer idents directly<commit_after>package odb

import (
	"bufio"
	"bytes"
	"encoding/hex"
	"fmt"
	"io"
	"strings"
	"time"
)

// Signature represents a commit signature, which can represent either
// committership or authorship of the commit that this signature belongs to. It
// specifies a name, email, and time that the signature was created.
//
// NOTE: this type is _not_ used by the `*Commit` instance, as it does not
// preserve cruft bytes. It is kept as a convenience type to test with.
type Signature struct {
	// Name is the first and last name of the individual holding this
	// signature.
	Name string
	// Email is the email address of the individual holding this signature.
	Email string
	// When is the instant in time when the signature was created.
	When time.Time
}

const (
	formatTimeZoneOnly = "-0700"
)

// String implements the fmt.Stringer interface and formats a Signature as
// expected in the Git commit internal object format. For instance:
//
//  Taylor Blau <ttaylorr@github.com> 1494258422 -0600
func (s *Signature) String() string {
	at := s.When.Unix()
	zone := s.When.Format(formatTimeZoneOnly)

	return fmt.Sprintf("%s <%s> %d %s", s.Name, s.Email, at, zone)
}

// ExtraHeader encapsulates a key-value pairing of header key to header value.
// It is stored as a struct{string, string} in memory as opposed to a
// map[string]string to maintain ordering in a byte-for-byte encode/decode round
// trip.
type ExtraHeader struct {
	// K is the header key, or the first run of bytes up until a ' ' (\x20)
	// character.
	K string
	// V is the header value, or the remaining run of bytes in the line,
	// stripping off the above "K" field as a prefix.
	V string
}

// Commit encapsulates a Git commit entry.
type Commit struct {
	// Author is the Author this commit, or the original writer of the
	// contents.
	//
	// NOTE: this field is stored as a string to ensure any extra "cruft"
	// bytes are preserved through migration.
	Author string
	// Committer is the individual or entity that added this commit to the
	// history.
	//
	// NOTE: this field is stored as a string to ensure any extra "cruft"
	// bytes are preserved through migration.
	Committer string
	// ParentIDs are the IDs of all parents for which this commit is a
	// linear child.
	ParentIDs [][]byte
	// TreeID is the root Tree associated with this commit.
	TreeID []byte
	// ExtraHeaders stores headers not listed above, for instance
	// "encoding", "gpgsig", or "mergetag" (among others).
	ExtraHeaders []*ExtraHeader
	// Message is the commit message, including any signing information
	// associated with this commit.
	Message string
}

// Type implements Object.ObjectType by returning the correct object type for
// Commits, CommitObjectType.
func (c *Commit) Type() ObjectType { return CommitObjectType }

// Decode implements Object.Decode and decodes the uncompressed commit being
// read. It returns the number of uncompressed bytes being consumed off of the
// stream, which should be strictly equal to the size given.
//
// If any error was encountered along the way, that will be returned, along with
// the number of bytes read up to that point.
func (c *Commit) Decode(from io.Reader, size int64) (n int, err error) {
	var finishedHeaders bool
	var messageParts []string

	s := bufio.NewScanner(from)
	for s.Scan() {
		text := s.Text()
		n = n + len(text+"\n")

		if len(s.Text()) == 0 {
			finishedHeaders = true
			continue
		}

		if fields := strings.Fields(text); len(fields) > 0 && !finishedHeaders {
			switch fields[0] {
			case "tree":
				id, err := hex.DecodeString(fields[1])
				if err != nil {
					return n, err
				}
				c.TreeID = id
			case "parent":
				id, err := hex.DecodeString(fields[1])
				if err != nil {
					return n, err
				}
				c.ParentIDs = append(c.ParentIDs, id)
			case "author":
				if len(text) >= 7 {
					c.Author = text[7:]
				} else {
					c.Author = ""
				}
			case "committer":
				if len(text) >= 10 {
					c.Committer = text[10:]
				} else {
					c.Committer = ""
				}
			default:
				c.ExtraHeaders = append(c.ExtraHeaders, &ExtraHeader{
					K: fields[0],
					V: strings.Join(fields[1:], " "),
				})
			}
		} else {
			messageParts = append(messageParts, s.Text())
		}
	}

	c.Message = strings.Join(messageParts, "\n")

	if err = s.Err(); err != nil {
		return n, err
	}
	return n, err
}

// Encode encodes the commit's contents to the given io.Writer, "w". If there was
// any error copying the commit's contents, that error will be returned.
//
// Otherwise, the number of bytes written will be returned.
func (c *Commit) Encode(to io.Writer) (n int, err error) {
	n, err = fmt.Fprintf(to, "tree %s\n", hex.EncodeToString(c.TreeID))
	if err != nil {
		return n, err
	}

	for _, pid := range c.ParentIDs {
		n1, err := fmt.Fprintf(to, "parent %s\n", hex.EncodeToString(pid))
		if err != nil {
			return n, err
		}

		n = n + n1
	}

	n2, err := fmt.Fprintf(to, "author %s\ncommitter %s\n", c.Author, c.Committer)
	if err != nil {
		return n, err
	}

	n = n + n2

	for _, hdr := range c.ExtraHeaders {
		n3, err := fmt.Fprintf(to, "%s %s\n", hdr.K, hdr.V)
		if err != nil {
			return n, err
		}

		n = n + n3
	}

	n4, err := fmt.Fprintf(to, "\n%s\n", c.Message)
	if err != nil {
		return n, err
	}

	return n + n4, err
}

// Equal returns whether the receiving and given commits are equal, or in other
// words, whether they are represented by the same SHA-1 when saved to the
// object database.
func (c *Commit) Equal(other *Commit) bool {
	if (c == nil) != (other == nil) {
		return false
	}

	if c != nil {
		if len(c.ParentIDs) != len(other.ParentIDs) {
			return false
		}
		for i := 0; i < len(c.ParentIDs); i++ {
			p1 := c.ParentIDs[i]
			p2 := other.ParentIDs[i]

			if !bytes.Equal(p1, p2) {
				return false
			}
		}

		if len(c.ExtraHeaders) != len(other.ExtraHeaders) {
			return false
		}
		for i := 0; i < len(c.ExtraHeaders); i++ {
			e1 := c.ExtraHeaders[i]
			e2 := other.ExtraHeaders[i]

			if e1.K != e2.K || e1.V != e2.V {
				return false
			}
		}

		return c.Author == other.Author &&
			c.Committer == other.Committer &&
			c.Message == other.Message &&
			bytes.Equal(c.TreeID, other.TreeID)
	}
	return true
}
<|endoftext|>