<commit_before>/*
Copyright (c) 2017, AverageSecurityGuy
# All rights reserved.

Demonstrate code flow in Go

Example usage:

$ go run flow.go
*/

package main

import (
    "fmt"
    "time"
)

func main() {
    str := "This is a string."
    ba := []byte("This is another string.")

    // A string is a collection of immutable, UTF-8 encode, bytes. We can
    // iterate over the items in the string and print their index and their
    // value.

    fmt.Println("Print each index and char of a string with for loop.")
    for i:=0; i<len(str); i++ {
        fmt.Printf("%d: %c\n", i, str[i])
    }
    fmt.Println()

    // The more idiomatic way to do this would be to use the range keyword.
    fmt.Println("Print each index and char of a string with range loop.")
    for i, c := range(str) {
        fmt.Printf("%d: %c\n", i, c)
    }
    fmt.Println()

    // Either method could be used for byte arrays as well.
    fmt.Println("Use the range method to print each index and byte value.")
    for i, b := range(ba) {
        fmt.Printf("%d: %d\n", i, b)
    }
    fmt.Println()

    // If you only want the index or the value you can use an underscore to
    // discard the value you don't want.
    fmt.Println("Only get the characters in the string.")
    for _, c := range(str) {
        fmt.Println(c)
    }
    fmt.Println()

    fmt.Println("Only get the index values in the string.")
    for i, _ := range(str) {
        fmt.Println(i)
    }
    fmt.Println()

    // Loops can be broken with the break keyword and processing can continue
    // using the continue keyword.
    fmt.Println("Print only the odd characters up to 12.")
    for i, c := range(str) {
        if (i % 2) == 0 {
            continue
        } else if i >= 12 {
            break
        } else {
            fmt.Println(i, string(c))
        }
    }
    fmt.Println()

    // Go does not have a while loop. Use a for loop with appropriate index
    // values or use the range keyword. If you need an infinite loop you can
    // use the following syntax.
    fmt.Println("Infinite Loop")
    for {
        fmt.Println("Hit Ctrl-C to quit.")
        time.Sleep(2 * time.Second)
    }

}
<commit_msg>Day 2<commit_after>/*
Copyright (c) 2017, AverageSecurityGuy
# All rights reserved.

Demonstrate code flow in Go

Example usage:

$ go run flow.go
*/

package main

import (
	"fmt"
	"time"
)

func main() {
	str := "This is a string."
	ba := []byte("This is another string.")

	// A string is a collection of immutable, UTF-8 encode, bytes. We can
	// iterate over the items in the string and print their index and their
	// value.

	fmt.Println("Print each index and char of a string with for loop.")
	for i := 0; i < len(str); i++ {
		fmt.Printf("%d: %c\n", i, str[i])
	}
	fmt.Println()

	// The more idiomatic way to do this would be to use the range keyword.
	fmt.Println("Print each index and char of a string with range loop.")
	for i, c := range str {
		fmt.Printf("%d: %c\n", i, c)
	}
	fmt.Println()

	// Either method could be used for byte arrays as well.
	fmt.Println("Use the range method to print each index and byte value.")
	for i, b := range ba {
		fmt.Printf("%d: %d\n", i, b)
	}
	fmt.Println()

	// If you only want the index or the value you can use an underscore to
	// discard the value you don't want.
	fmt.Println("Only get the characters in the string.")
	for _, c := range str {
		fmt.Println(c)
	}
	fmt.Println()

	fmt.Println("Only get the index values in the string.")
	for i, _ := range str {
		fmt.Println(i)
	}
	fmt.Println()

	// Loops can be broken with the break keyword and processing can continue
	// using the continue keyword.
	fmt.Println("Print only the odd characters up to 12.")
	for i, c := range str {
		if (i % 2) == 0 {
			continue
		} else if i >= 12 {
			break
		} else {
			fmt.Println(i, string(c))
		}
	}
	fmt.Println()

	// Go does not have a while loop. Use a for loop with appropriate index
	// values or use the range keyword. If you need an infinite loop you can
	// use the following syntax.
	fmt.Println("Infinite Loop")
	for {
		fmt.Println("Hit Ctrl-C to quit.")
		time.Sleep(2 * time.Second)
	}

}
<|endoftext|>


<commit_before>package pick

import (
	"io"
	"strings"
)

// input can be input/select/textarea
// TODO: now only get 1st value, need []string as result
// TODO: pick select option that has no value attr
// TODO: radio
func PickForm(reader io.Reader, attr *Attr) (input map[string][]string) {
	htmlArr := PickHtml(&Option{
		PageSource: reader,
		TagName:    "form",
		Attr:       attr,
	}, 1)
	if len(htmlArr) == 0 {
		return
	}

	html := htmlArr[0]
	input = make(map[string][]string)

	// pick input
	inputNameArr := PickAttr(&Option{
		PageSource: strings.NewReader(html),
		TagName:    "input",
		Attr:       nil,
	}, "name", 0)

	for i := 0; i < len(inputNameArr); i++ {
		values := PickAttr(&Option{
			PageSource: strings.NewReader(html),
			TagName:    "input",
			Attr: &Attr{
				Label: "name",
				Value: inputNameArr[i],
			},
		}, "value", 1)
		if len(values) == 0 {
			continue
		}

		input[inputNameArr[i]] = []string{values[0]}
	}

	// pick textarea
	textNameArr := PickAttr(&Option{
		PageSource: strings.NewReader(html),
		TagName:    "textarea",
		Attr:       nil,
	}, "name", 0)

	for i := 0; i < len(textNameArr); i++ {
		values := PickText(&Option{
			PageSource: strings.NewReader(html),
			TagName:    "textarea",
			Attr: &Attr{
				Label: "name",
				Value: textNameArr[i],
			},
		}, 1)
		if len(values) == 0 {
			continue
		}

		input[textNameArr[i]] = []string{values[0]}
	}

	// pick select
	selectNameArr := PickAttr(&Option{
		PageSource: strings.NewReader(html),
		TagName:    "select",
		Attr:       nil,
	}, "name", 0)

	for i := 0; i < len(selectNameArr); i++ {
		selectHtmlArr := PickHtml(&Option{
			PageSource: strings.NewReader(html),
			TagName:    "select",
			Attr: &Attr{
				Label: "name",
				Value: selectNameArr[i],
			},
		}, 1)
		if len(selectHtmlArr) == 0 {
			continue
		}

		// get selected option
		values := PickAttr(&Option{
			PageSource: strings.NewReader(selectHtmlArr[0]),
			TagName:    "option",
			Attr: &Attr{
				Label: "selected",
				Value: "",
			},
		}, "value", 1)
		if len(values) != 0 {
			input[selectNameArr[i]] = []string{values[0]}
			continue
		}

		// if no select get the 1st option value
		values = PickAttr(&Option{
			PageSource: strings.NewReader(selectHtmlArr[0]),
			TagName:    "option",
			Attr:       nil,
		}, "value", 1)
		if len(values) == 0 {
			continue
		}

		input[selectNameArr[i]] = []string{values[0]}
	}

	return
}
<commit_msg>fix #PickForm no input values now as empty string as value<commit_after>package pick

import (
	"io"
	"strings"
)

// input can be input/select/textarea
// TODO: now only get 1st value, need []string as result
// TODO: pick select option that has no value attr
// TODO: radio
func PickForm(reader io.Reader, attr *Attr) (input map[string][]string) {
	htmlArr := PickHtml(&Option{
		PageSource: reader,
		TagName:    "form",
		Attr:       attr,
	}, 1)
	if len(htmlArr) == 0 {
		return
	}

	html := htmlArr[0]
	input = make(map[string][]string)

	// pick input
	inputNameArr := PickAttr(&Option{
		PageSource: strings.NewReader(html),
		TagName:    "input",
		Attr:       nil,
	}, "name", 0)

	for i := 0; i < len(inputNameArr); i++ {
		values := PickAttr(&Option{
			PageSource: strings.NewReader(html),
			TagName:    "input",
			Attr: &Attr{
				Label: "name",
				Value: inputNameArr[i],
			},
		}, "value", 1)
		if len(values) == 0 {
			input[inputNameArr[i]] = []string{""}
			continue
		}

		input[inputNameArr[i]] = []string{values[0]}
	}

	// pick textarea
	textNameArr := PickAttr(&Option{
		PageSource: strings.NewReader(html),
		TagName:    "textarea",
		Attr:       nil,
	}, "name", 0)

	for i := 0; i < len(textNameArr); i++ {
		values := PickText(&Option{
			PageSource: strings.NewReader(html),
			TagName:    "textarea",
			Attr: &Attr{
				Label: "name",
				Value: textNameArr[i],
			},
		}, 1)
		if len(values) == 0 {
			input[textNameArr[i]] = []string{""}
			continue
		}

		input[textNameArr[i]] = []string{values[0]}
	}

	// pick select
	selectNameArr := PickAttr(&Option{
		PageSource: strings.NewReader(html),
		TagName:    "select",
		Attr:       nil,
	}, "name", 0)

	for i := 0; i < len(selectNameArr); i++ {
		selectHtmlArr := PickHtml(&Option{
			PageSource: strings.NewReader(html),
			TagName:    "select",
			Attr: &Attr{
				Label: "name",
				Value: selectNameArr[i],
			},
		}, 1)
		if len(selectHtmlArr) == 0 {
			continue
		}

		// get selected option
		values := PickAttr(&Option{
			PageSource: strings.NewReader(selectHtmlArr[0]),
			TagName:    "option",
			Attr: &Attr{
				Label: "selected",
				Value: "",
			},
		}, "value", 1)
		if len(values) != 0 {
			input[selectNameArr[i]] = []string{values[0]}
			continue
		}

		// if no select get the 1st option value
		values = PickAttr(&Option{
			PageSource: strings.NewReader(selectHtmlArr[0]),
			TagName:    "option",
			Attr:       nil,
		}, "value", 1)
		if len(values) == 0 {
			input[selectNameArr[i]] = []string{""}
			continue
		}

		input[selectNameArr[i]] = []string{values[0]}
	}

	return
}
<|endoftext|>


<commit_before>package controllers

import (
	"encoding/json"
	"github.com/gojp/kana"
	"github.com/gojp/nihongo/app/helpers"
	"github.com/gojp/nihongo/app/models"
	"github.com/gojp/nihongo/app/routes"
	"github.com/robfig/revel"
	"log"
	"regexp"
	"strings"
)

type App struct {
	*revel.Controller
}

type Word struct {
	*models.Word
}

type PopularSearch struct {
	Term string
}

// wrap a string in strong tags
func makeStrong(query string) string {
	return "" + query + ""
}

// convert the query to hiragana and katakana. if it's already in
// hiragana or katakana, it will just be the same.
func convertQueryToKana(query string) (hiragana, katakana string) {
	h := kana.RomajiToHiragana(query)
	k := kana.RomajiToKatakana(query)
	return h, k
}

// Wrap the query in  tags so that we can highlight it in the results
func highlightQuery(w Word, query string) {
	// make regular expression that matches the original query
	re := regexp.MustCompile(`\b` + regexp.QuoteMeta(query) + `\b`)
	// convert original query to kana
	h, k := convertQueryToKana(query)
	// wrap the query in strong tags
	queryHighlighted := makeStrong(query)
	hiraganaHighlighted := makeStrong(h)
	katakanaHighlighted := makeStrong(k)

	// if the original input is Japanese, then the original input converted
	// to hiragana and katakana will be equal, so just choose one
	// to highlight so that we only end up with one pair of strong tags
	if hiraganaHighlighted == katakanaHighlighted {
		w.JapaneseHL = strings.Replace(w.Japanese, h, hiraganaHighlighted, -1)
	} else {
		// The original input is romaji, so we convert it to hiragana and katakana
		// and highlight both.
		w.JapaneseHL = strings.Replace(w.Japanese, h, hiraganaHighlighted, -1)
		w.JapaneseHL = strings.Replace(w.JapaneseHL, k, katakanaHighlighted, -1)
	}

	// highlight the furigana too, same as above
	w.FuriganaHL = strings.Replace(w.Furigana, h, hiraganaHighlighted, -1)
	w.FuriganaHL = strings.Replace(w.FuriganaHL, k, katakanaHighlighted, -1)
	// highlight the query inside the list of English definitions
	w.EnglishHL = []string{}
	for _, e := range w.English {
		e = re.ReplaceAllString(e, queryHighlighted)
		w.EnglishHL = append(w.EnglishHL, e)
	}
}

func getWordList(hits [][]byte, query string) (wordList []Word) {
	// highlight queries and build Word object
	for _, hit := range hits {
		w := Word{}
		err := json.Unmarshal(hit, &w)
		if err != nil {
			log.Println(err)
		}
		highlightQuery(w, query)
		wordList = append(wordList, w)
	}
	return wordList
}

func (a App) Search(query string) revel.Result {
	if len(query) == 0 {
		return a.Redirect(routes.App.Index())
	}
	hits := helpers.Search(query)
	wordList := getWordList(hits, query)
	return a.Render(wordList)
}

func (c App) Details(query string) revel.Result {
	println(query)
	if len(query) == 0 {
		return c.Redirect(routes.App.Index())
	}
	if strings.Contains(query, " ") {
		return c.Redirect(routes.App.Details(strings.Replace(query, " ", "_", -1)))
	}

	query = strings.Replace(query, "_", " ", -1)
	hits := helpers.Search(query)
	wordList := getWordList(hits, query)
	pageTitle := query + " in Japanese"

	return c.Render(wordList, query, pageTitle)
}

func (c App) SearchGet() revel.Result {
	if query, ok := c.Params.Values["q"]; ok && len(query) > 0 {
		return c.Redirect(routes.App.Details(query[0]))
	}
	return c.Redirect(routes.App.Index())
}

func (c App) About() revel.Result {
	return c.Render()
}

func (c App) Index() revel.Result {

	// get the popular searches
	// collection := c.MongoSession.DB("greenbook").C("hits")
	// q := collection.Find(nil).Sort("-count")

	// termList := []models.SearchTerm{}
	// iter := q.Limit(10).Iter()
	// iter.All(&termList)

	termList := []PopularSearch{
		PopularSearch{"今日は"},
		PopularSearch{"kanji"},
		PopularSearch{"amazing"},
		PopularSearch{"かんじ"},
		PopularSearch{"莞爾"},
		PopularSearch{"天真流露"},
		PopularSearch{"funny"},
		PopularSearch{"にほんご"},
	}

	return c.Render(termList)
}
<commit_msg>remove println<commit_after>package controllers

import (
	"encoding/json"
	"github.com/gojp/kana"
	"github.com/gojp/nihongo/app/helpers"
	"github.com/gojp/nihongo/app/models"
	"github.com/gojp/nihongo/app/routes"
	"github.com/robfig/revel"
	"log"
	"regexp"
	"strings"
)

type App struct {
	*revel.Controller
}

type Word struct {
	*models.Word
}

type PopularSearch struct {
	Term string
}

// wrap a string in strong tags
func makeStrong(query string) string {
	return "" + query + ""
}

// convert the query to hiragana and katakana. if it's already in
// hiragana or katakana, it will just be the same.
func convertQueryToKana(query string) (hiragana, katakana string) {
	h := kana.RomajiToHiragana(query)
	k := kana.RomajiToKatakana(query)
	return h, k
}

// Wrap the query in  tags so that we can highlight it in the results
func highlightQuery(w Word, query string) {
	// make regular expression that matches the original query
	re := regexp.MustCompile(`\b` + regexp.QuoteMeta(query) + `\b`)
	// convert original query to kana
	h, k := convertQueryToKana(query)
	// wrap the query in strong tags
	queryHighlighted := makeStrong(query)
	hiraganaHighlighted := makeStrong(h)
	katakanaHighlighted := makeStrong(k)

	// if the original input is Japanese, then the original input converted
	// to hiragana and katakana will be equal, so just choose one
	// to highlight so that we only end up with one pair of strong tags
	if hiraganaHighlighted == katakanaHighlighted {
		w.JapaneseHL = strings.Replace(w.Japanese, h, hiraganaHighlighted, -1)
	} else {
		// The original input is romaji, so we convert it to hiragana and katakana
		// and highlight both.
		w.JapaneseHL = strings.Replace(w.Japanese, h, hiraganaHighlighted, -1)
		w.JapaneseHL = strings.Replace(w.JapaneseHL, k, katakanaHighlighted, -1)
	}

	// highlight the furigana too, same as above
	w.FuriganaHL = strings.Replace(w.Furigana, h, hiraganaHighlighted, -1)
	w.FuriganaHL = strings.Replace(w.FuriganaHL, k, katakanaHighlighted, -1)
	// highlight the query inside the list of English definitions
	w.EnglishHL = []string{}
	for _, e := range w.English {
		e = re.ReplaceAllString(e, queryHighlighted)
		w.EnglishHL = append(w.EnglishHL, e)
	}
}

func getWordList(hits [][]byte, query string) (wordList []Word) {
	// highlight queries and build Word object
	for _, hit := range hits {
		w := Word{}
		err := json.Unmarshal(hit, &w)
		if err != nil {
			log.Println(err)
		}
		highlightQuery(w, query)
		wordList = append(wordList, w)
	}
	return wordList
}

func (a App) Search(query string) revel.Result {
	if len(query) == 0 {
		return a.Redirect(routes.App.Index())
	}
	hits := helpers.Search(query)
	wordList := getWordList(hits, query)
	return a.Render(wordList)
}

func (c App) Details(query string) revel.Result {
	if len(query) == 0 {
		return c.Redirect(routes.App.Index())
	}
	if strings.Contains(query, " ") {
		return c.Redirect(routes.App.Details(strings.Replace(query, " ", "_", -1)))
	}

	query = strings.Replace(query, "_", " ", -1)
	hits := helpers.Search(query)
	wordList := getWordList(hits, query)
	pageTitle := query + " in Japanese"

	return c.Render(wordList, query, pageTitle)
}

func (c App) SearchGet() revel.Result {
	if query, ok := c.Params.Values["q"]; ok && len(query) > 0 {
		return c.Redirect(routes.App.Details(query[0]))
	}
	return c.Redirect(routes.App.Index())
}

func (c App) About() revel.Result {
	return c.Render()
}

func (c App) Index() revel.Result {

	// get the popular searches
	// collection := c.MongoSession.DB("greenbook").C("hits")
	// q := collection.Find(nil).Sort("-count")

	// termList := []models.SearchTerm{}
	// iter := q.Limit(10).Iter()
	// iter.All(&termList)

	termList := []PopularSearch{
		PopularSearch{"今日は"},
		PopularSearch{"kanji"},
		PopularSearch{"amazing"},
		PopularSearch{"かんじ"},
		PopularSearch{"莞爾"},
		PopularSearch{"天真流露"},
		PopularSearch{"funny"},
		PopularSearch{"にほんご"},
	}

	return c.Render(termList)
}
<|endoftext|>


<commit_before>package controllers

import "github.com/revel/revel"
import "database/sql"
import _ "github.com/go-sql-driver/mysql"
import "fmt"
import "os"
import "encoding/json"
import "io/ioutil"

type App struct {
	*revel.Controller
}

type Example struct {
    text  []string
 }

func (c App) Index() revel.Result {

	db, err := sql.Open("mysql", "root:kazuki69@/golang")

    if err != nil {
        panic(err.Error())  // Just for example purpose. You should use proper error handling instead of panic
    }
    defer db.Close()

    // Prepare statement for inserting data
    stmtIns, err := db.Prepare("INSERT INTO squareNum VALUES( ?, ? )") // ? = placeholder
    if err != nil {
    	fmt.Printf("%s", err.Error());
    }
    defer stmtIns.Close() // Close the statement when we leave main() / the program terminates

    // Prepare statement for reading data
    stmtOut, err := db.Prepare("SELECT squareNumber FROM squareNum WHERE number = ?")
    if err != nil {
    	fmt.Printf("%s", err.Error());
    }
    defer stmtOut.Close()

    // Insert square numbers for 0-24 in the database
    for i := 0; i < 25; i++ {
        _, err = stmtIns.Exec(i, (i * i)) // Insert tuples (i, i^2)
        if err != nil {
    		fmt.Printf("%s", err.Error());
        }
    }

    var squareNum int // we "scan" the result in here

    // Query the square-number of 13
    err = stmtOut.QueryRow(13).Scan(&squareNum) // WHERE number = 13
    if err != nil {
    	fmt.Printf("%s", err.Error());
    }
    fmt.Printf("The square number of 13 is: %d", squareNum)

    // Query another number.. 1 maybe?
    err = stmtOut.QueryRow(1).Scan(&squareNum) // WHERE number = 1
    if err != nil {
    	fmt.Printf("%s", err.Error());
    }
    fmt.Printf("The square number of 1 is: %d", squareNum)

	greeting := "test json"

    type jsonobject struct {
    Object ObjectType
}

type ObjectType struct {
    Buffer_size int
    Databases   []DatabasesType
}

type DatabasesType struct {
    Host   string
    User   string
    Pass   string
    Type   string
    Name   string
    Tables []TablesType
}

type TablesType struct {
    Name     string
    Statment string
    Regex    string
    Types    []TypesType
}

type TypesType struct {
    Id    string
    Value string
}

// Main function
// I realize this function is much too simple I am simply at a loss to

func main() {
    file, e := ioutil.ReadFile("./config.json")
    if e != nil {
        fmt.Printf("File error: %v\n", e)
        os.Exit(1)
    }
    fmt.Printf("%s\n", string(file))

    //m := new(Dispatch)
    //var m interface{}
    var jsontype jsonobject
    json.Unmarshal(file, &jsontype)
    fmt.Printf("Results: %v\n", jsontype)
}

	return c.Render(greeting)
}

func (c App) Hello(myName string) revel.Result {

	c.Validation.Required(myName).Message("Your name is required!")
	c.Validation.MinSize(myName, 3).Message("Your name is not long enough!")

	if c.Validation.HasErrors() {
		c.Validation.Keep()
		c.FlashParams()
		return c.Redirect(App.Index)
	}

	return c.Render(myName)

}

func (c App) TestJson() revel.Result {

	//TODO

	return c.Render(App.Index)
}<commit_msg>Changing name policy<commit_after>package controllers

import "github.com/revel/revel"
import "database/sql"
import _ "github.com/go-sql-driver/mysql"
import "fmt"
import "os"
import "encoding/json"
import "io/ioutil"

type App struct {
	*revel.Controller
}

type Example struct {
    text  []string
 }

func (c App) Index() revel.Result {

	db, err := sql.Open("mysql", "root:kazuki69@/golang")

    if err != nil {
        panic(err.Error())  // Just for example purpose. You should use proper error handling instead of panic
    }
    defer db.Close()

    // Prepare statement for inserting data
    stmtIns, err := db.Prepare("INSERT INTO squareNum VALUES( ?, ? )") // ? = placeholder
    if err != nil {
    	fmt.Printf("%s", err.Error());
    }
    defer stmtIns.Close() // Close the statement when we leave main() / the program terminates

    // Prepare statement for reading data
    stmtOut, err := db.Prepare("SELECT squareNumber FROM squareNum WHERE number = ?")
    if err != nil {
    	fmt.Printf("%s", err.Error());
    }
    defer stmtOut.Close()

    // Insert square numbers for 0-24 in the database
    for i := 0; i < 25; i++ {
        _, err = stmtIns.Exec(i, (i * i)) // Insert tuples (i, i^2)
        if err != nil {
    		fmt.Printf("%s", err.Error());
        }
    }

    var squareNum int // we "scan" the result in here

    // Query the square-number of 13
    err = stmtOut.QueryRow(13).Scan(&squareNum) // WHERE number = 13
    if err != nil {
    	fmt.Printf("%s", err.Error());
    }
    fmt.Printf("The square number of 13 is: %d", squareNum)

    // Query another number.. 1 maybe?
    err = stmtOut.QueryRow(1).Scan(&squareNum) // WHERE number = 1
    if err != nil {
    	fmt.Printf("%s", err.Error());
    }
    fmt.Printf("The square number of 1 is: %d", squareNum)

	greeting := "test json"

    type jsonobject struct {
    Object ObjectType
}

type ObjectType struct {
    Buffer_size int
    Databases   []DatabasesType
}

type DatabasesType struct {
    Host   string
    User   string
    Pass   string
    Type   string
    Name   string
    Tables []TablesType
}

type TablesType struct {
    Name     string
    Statment string
    Regex    string
    Types    []TypesType
}

type TypesType struct {
    Id    string
    Value string
}

// Main function
// I realize this function is much too simple I am simply at a loss to

func main() {
    file, e := ioutil.ReadFile("./config.json")
    if e != nil {
        fmt.Printf("File error: %v\n", e)
        os.Exit(1)
    }
    fmt.Printf("%s\n", string(file))

    //m := new(Dispatch)
    //var m interface{}
    var jsontype jsonobject
    json.Unmarshal(file, &jsontype)
    fmt.Printf("Results: %v\n", jsontype)
}

	return c.Render(greeting)
}

func (c App) Hello(myName string) revel.Result {

	c.Validation.Required(myName).Message("Your name is required!")
	c.Validation.MinSize(myName, 10).Message("Your name is not long enough!")

	if c.Validation.HasErrors() {
		c.Validation.Keep()
		c.FlashParams()
		return c.Redirect(App.Index)
	}

	return c.Render(myName)

}

func (c App) TestJson() revel.Result {

	//TODO

	return c.Render(App.Index)
}<|endoftext|>


<commit_before>//
//  trayicon.go
//
//  Created by Martino Facchin
//  Copyright (c) 2015 Arduino LLC
//
//  Permission is hereby granted, free of charge, to any person
//  obtaining a copy of this software and associated documentation
//  files (the "Software"), to deal in the Software without
//  restriction, including without limitation the rights to use,
//  copy, modify, merge, publish, distribute, sublicense, and/or sell
//  copies of the Software, and to permit persons to whom the
//  Software is furnished to do so, subject to the following
//  conditions:
//
//  The above copyright notice and this permission notice shall be
//  included in all copies or substantial portions of the Software.
//
//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
//  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
//  OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
//  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
//  HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
//  WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
//  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
//  OTHER DEALINGS IN THE SOFTWARE.
//

// +build !cli

package main

import (
	"os"
	"path/filepath"
	"runtime"

	"github.com/arduino/arduino-create-agent/icon"
	"github.com/getlantern/systray"
	"github.com/go-ini/ini"
	"github.com/kardianos/osext"
	log "github.com/sirupsen/logrus"
	"github.com/skratchdot/open-golang/open"
	"go.bug.st/serial.v1"
)

func setupSysTray() {
	runtime.LockOSThread()
	if *hibernate == true {
		systray.Run(setupSysTrayHibernate,nil)
	} else {
		systray.Run(setupSysTrayReal,nil)
	}
}

func addRebootTrayElement() {
	reboot_tray := systray.AddMenuItem("Reboot to update", "")

	go func() {
		<-reboot_tray.ClickedCh
		systray.Quit()
		log.Println("Restarting now...")
		log.Println("Restart because addReebotTrayElement")
		restart("")
	}()
}

type ConfigIni struct {
	Name      string
	Localtion string
}

func getConfigs() []ConfigIni {
	// parse all configs in executable folder
	// config.ini must be there, so call it Default
	src, _ := osext.Executable()
	dest := filepath.Dir(src)

	var configs []ConfigIni

	filepath.Walk(dest, func(path string, f os.FileInfo, _ error) error {
		if !f.IsDir() {
			if filepath.Ext(path) == ".ini" {
				cfg, err := ini.LoadSources(ini.LoadOptions{IgnoreInlineComment: true}, filepath.Join(dest, f.Name()))
				if err != nil {
					return err
				}
				defaultSection, err := cfg.GetSection("")
				name := defaultSection.Key("name").String()
				if name == "" || err != nil {
					name = "Default config"
				}
				conf := ConfigIni{Name: name, Localtion: f.Name()}
				configs = append(configs, conf)
			}
		}
		return nil
	})
	return configs
}

func setupSysTrayReal() {

	systray.SetIcon(icon.GetIcon())
	mUrl := systray.AddMenuItem("Go to Arduino Create", "Arduino Create")
	mDebug := systray.AddMenuItem("Open debug console", "Debug console")
	menuVer := systray.AddMenuItem("Agent version "+version+"-"+git_revision, "")
	mPause := systray.AddMenuItem("Pause Plugin", "")
	var mConfigCheckbox []*systray.MenuItem

	configs := getConfigs()

	if len(configs) > 1 {
		for _, config := range configs {
			entry := systray.AddMenuItem(config.Name, "")
			mConfigCheckbox = append(mConfigCheckbox, entry)
			// decorate configs
			gliph := " ☐ "
			if *additionalConfig == config.Localtion {
				gliph = " 🗹 "
			}
			entry.SetTitle(gliph + config.Name)
		}
	}
	//mQuit := systray.AddMenuItem("Quit Plugin", "")

	menuVer.Disable()

	for i, _ := range mConfigCheckbox {
		go func(v int) {
			for {
				<-mConfigCheckbox[v].ClickedCh

				restart("", "-additional-config", configs[v].Localtion)
			}
		}(i)
	}

	go func() {
		<-mPause.ClickedCh
		ports, _ := serial.GetPortsList()
		for _, element := range ports {
			spClose(element)
		}
		systray.Quit()
		*hibernate = true
		log.Println("Restart because setup went wrong?")
		restart("")
	}()

	// go func() {
	// 	<-mQuit.ClickedCh
	// 	systray.Quit()
	// 	exit()
	// }()

	go func() {
		for {
			<-mDebug.ClickedCh
			logAction("log on")
			open.Start("http://127.0.0.1" + port)
		}
	}()

	// We can manipulate the systray in other goroutines
	go func() {
		for {
			<-mUrl.ClickedCh
			open.Start("https://create.arduino.cc")
		}
	}()
}

func setupSysTrayHibernate() {

	systray.SetIcon(icon.GetIconHiber())
	mOpen := systray.AddMenuItem("Open Plugin", "")
	mQuit := systray.AddMenuItem("Kill Plugin", "")

	go func() {
		<-mOpen.ClickedCh
		*hibernate = false
		log.Println("Restart for hibernation")
		systray.Quit()
		restart("")
	}()

	go func() {
		<-mQuit.ClickedCh
		systray.Quit()
		exit()
	}()
}

func quitSysTray() {
	systray.Quit()
}
<commit_msg>Reword menu items<commit_after>//
//  trayicon.go
//
//  Created by Martino Facchin
//  Copyright (c) 2015 Arduino LLC
//
//  Permission is hereby granted, free of charge, to any person
//  obtaining a copy of this software and associated documentation
//  files (the "Software"), to deal in the Software without
//  restriction, including without limitation the rights to use,
//  copy, modify, merge, publish, distribute, sublicense, and/or sell
//  copies of the Software, and to permit persons to whom the
//  Software is furnished to do so, subject to the following
//  conditions:
//
//  The above copyright notice and this permission notice shall be
//  included in all copies or substantial portions of the Software.
//
//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
//  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
//  OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
//  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
//  HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
//  WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
//  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
//  OTHER DEALINGS IN THE SOFTWARE.
//

// +build !cli

package main

import (
	"os"
	"path/filepath"
	"runtime"

	"github.com/arduino/arduino-create-agent/icon"
	"github.com/getlantern/systray"
	"github.com/go-ini/ini"
	"github.com/kardianos/osext"
	log "github.com/sirupsen/logrus"
	"github.com/skratchdot/open-golang/open"
	"go.bug.st/serial.v1"
)

func setupSysTray() {
	runtime.LockOSThread()
	if *hibernate == true {
		systray.Run(setupSysTrayHibernate,nil)
	} else {
		systray.Run(setupSysTrayReal,nil)
	}
}

func addRebootTrayElement() {
	reboot_tray := systray.AddMenuItem("Reboot to update", "")

	go func() {
		<-reboot_tray.ClickedCh
		systray.Quit()
		log.Println("Restarting now...")
		log.Println("Restart because addReebotTrayElement")
		restart("")
	}()
}

type ConfigIni struct {
	Name      string
	Localtion string
}

func getConfigs() []ConfigIni {
	// parse all configs in executable folder
	// config.ini must be there, so call it Default
	src, _ := osext.Executable()
	dest := filepath.Dir(src)

	var configs []ConfigIni

	filepath.Walk(dest, func(path string, f os.FileInfo, _ error) error {
		if !f.IsDir() {
			if filepath.Ext(path) == ".ini" {
				cfg, err := ini.LoadSources(ini.LoadOptions{IgnoreInlineComment: true}, filepath.Join(dest, f.Name()))
				if err != nil {
					return err
				}
				defaultSection, err := cfg.GetSection("")
				name := defaultSection.Key("name").String()
				if name == "" || err != nil {
					name = "Default config"
				}
				conf := ConfigIni{Name: name, Localtion: f.Name()}
				configs = append(configs, conf)
			}
		}
		return nil
	})
	return configs
}

func setupSysTrayReal() {

	systray.SetIcon(icon.GetIcon())
	menuVer := systray.AddMenuItem("Agent version "+version+"-"+git_revision, "")
	systray.AddSeparator()
	mUrl := systray.AddMenuItem("Go to Arduino Create", "Arduino Create")
	mDebug := systray.AddMenuItem("Open debug console", "Debug console")
	systray.AddSeparator()
	mPause := systray.AddMenuItem("Pause Plugin", "")
	var mConfigCheckbox []*systray.MenuItem

	configs := getConfigs()

	if len(configs) > 1 {
		for _, config := range configs {
			entry := systray.AddMenuItem(config.Name, "")
			mConfigCheckbox = append(mConfigCheckbox, entry)
			// decorate configs
			gliph := " ☐ "
			if *additionalConfig == config.Localtion {
				gliph = " 🗹 "
			}
			entry.SetTitle(gliph + config.Name)
		}
	}
	//mQuit := systray.AddMenuItem("Quit Plugin", "")

	menuVer.Disable()

	for i, _ := range mConfigCheckbox {
		go func(v int) {
			for {
				<-mConfigCheckbox[v].ClickedCh

				restart("", "-additional-config", configs[v].Localtion)
			}
		}(i)
	}

	go func() {
		<-mPause.ClickedCh
		ports, _ := serial.GetPortsList()
		for _, element := range ports {
			spClose(element)
		}
		systray.Quit()
		*hibernate = true
		log.Println("Restart because setup went wrong?")
		restart("")
	}()

	// go func() {
	// 	<-mQuit.ClickedCh
	// 	systray.Quit()
	// 	exit()
	// }()

	go func() {
		for {
			<-mDebug.ClickedCh
			logAction("log on")
			open.Start("http://127.0.0.1" + port)
		}
	}()

	// We can manipulate the systray in other goroutines
	go func() {
		for {
			<-mUrl.ClickedCh
			open.Start("https://create.arduino.cc")
		}
	}()
}

func setupSysTrayHibernate() {

	systray.SetIcon(icon.GetIconHiber())
	mOpen := systray.AddMenuItem("Resume Plugin", "")
	systray.AddSeparator()
	mQuit := systray.AddMenuItem("Quit Plugin", "")

	go func() {
		<-mOpen.ClickedCh
		*hibernate = false
		log.Println("Restart for hibernation")
		systray.Quit()
		restart("")
	}()

	go func() {
		<-mQuit.ClickedCh
		systray.Quit()
		exit()
	}()
}

func quitSysTray() {
	systray.Quit()
}
<|endoftext|>


<commit_before>package whatlanggo

import (
	"sort"
	"unicode"
)

type trigram struct {
	trigram string
	count   int
}

//convert punctuations and digits to space.
func toTrigramChar(ch rune) rune {
	if isStopChar(ch) {
		return ' '
	}
	return ch
}

func getTrigramsWithPositions(text string) map[string]int {
	counterMap := count(text)
	trigrams := make([]trigram, len(counterMap))

	i := 0
	for tg, count := range counterMap {
		trigrams[i] = trigram{tg, count}
		i++
	}

	sort.SliceStable(trigrams, func(i, j int) bool { return trigrams[i].count < trigrams[j].count })

	trigramsWithPositions := map[string]int{}

	j := 0
	for i := len(trigrams) - 1; i >= 0; i-- {
		trigramsWithPositions[trigrams[i].trigram] = j
		j++
	}
	return trigramsWithPositions
}

func count(text string) map[string]int {
	var r1, r2, r3 rune
	trigrams := map[string]int{}
	var txt []rune

	for _, r := range text {
		txt = append(txt, unicode.ToLower(toTrigramChar(r)))
	}
	txt = append(txt, ' ')

	r1 = ' '
	r2 = txt[0]
	for i := 1; i < len(txt); i++ {
		r3 = txt[i]
		if !(r2 == ' ' && (r1 == ' ' || r3 == ' ')) {
			trigram := []rune{}
			trigram = append(trigram, r1)
			trigram = append(trigram, r2)
			trigram = append(trigram, r3)
			if trigrams[string(trigram)] == 0 {
				trigrams[string(trigram)] = 1
			} else {
				trigrams[string(trigram)]++
			}
		}
		r1 = r2
		r2 = r3
	}

	return trigrams
}
<commit_msg>Generate trigrams with a consistent ordering<commit_after>package whatlanggo

import (
	"sort"
	"strings"
	"unicode"
)

type trigram struct {
	trigram string
	count   int
}

//convert punctuations and digits to space.
func toTrigramChar(ch rune) rune {
	if isStopChar(ch) {
		return ' '
	}
	return ch
}

func getTrigramsWithPositions(text string) map[string]int {
	counterMap := count(text)
	trigrams := make([]trigram, len(counterMap))

	i := 0
	for tg, count := range counterMap {
		trigrams[i] = trigram{tg, count}
		i++
	}

	sort.SliceStable(trigrams, func(i, j int) bool {
		if trigrams[i].count == trigrams[j].count {
			return strings.Compare(trigrams[i].trigram, trigrams[j].trigram) < 0
		}
		return trigrams[i].count < trigrams[j].count
	})

	trigramsWithPositions := map[string]int{}

	j := 0
	for i := len(trigrams) - 1; i >= 0; i-- {
		trigramsWithPositions[trigrams[i].trigram] = j
		j++
	}
	return trigramsWithPositions
}

func count(text string) map[string]int {
	var r1, r2, r3 rune
	trigrams := map[string]int{}
	var txt []rune

	for _, r := range text {
		txt = append(txt, unicode.ToLower(toTrigramChar(r)))
	}
	txt = append(txt, ' ')

	r1 = ' '
	r2 = txt[0]
	for i := 1; i < len(txt); i++ {
		r3 = txt[i]
		if !(r2 == ' ' && (r1 == ' ' || r3 == ' ')) {
			trigram := []rune{}
			trigram = append(trigram, r1)
			trigram = append(trigram, r2)
			trigram = append(trigram, r3)
			if trigrams[string(trigram)] == 0 {
				trigrams[string(trigram)] = 1
			} else {
				trigrams[string(trigram)]++
			}
		}
		r1 = r2
		r2 = r3
	}

	return trigrams
}
<|endoftext|>


<commit_before>// +build linux

package sandbox

import (
	"crypto/rand"
	"errors"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"sync"

	"github.com/containernetworking/plugins/pkg/ns"
	"github.com/docker/docker/pkg/mount"
	"github.com/docker/docker/pkg/symlink"
	"golang.org/x/sys/unix"
)

// Get returns the NetNs for a given NetNsIface
func (n *NetNs) Get() *NetNs {
	return n
}

// Initialized returns true if the NetNs is already initialized
func (n *NetNs) Initialized() bool {
	return n.initialized
}

// Initialize does the necessary setup for a NetNs
func (n *NetNs) Initialize() (NetNsIface, error) {
	netNS, err := NewNS()
	if err != nil {
		return nil, err
	}
	n.netNS = netNS
	n.closed = false
	n.initialized = true
	return n, nil
}

func getNetNs(nsPath string) (*NetNs, error) {
	netNS, err := ns.GetNS(nsPath)
	if err != nil {
		return nil, err
	}

	return &NetNs{netNS: netNS, closed: false, restored: true}, nil
}

// NetNs handles data pertaining a network namespace
type NetNs struct {
	sync.Mutex
	netNS       ns.NetNS
	symlink     *os.File
	closed      bool
	restored    bool
	initialized bool
}

// SymlinkCreate creates the necessary symlinks for the NetNs
func (n *NetNs) SymlinkCreate(name string) error {
	if n.netNS == nil {
		return errors.New("no netns set up")
	}
	b := make([]byte, 4)
	_, randErr := rand.Reader.Read(b)
	if randErr != nil {
		return randErr
	}

	nsName := fmt.Sprintf("%s-%x", name, b)
	symlinkPath := filepath.Join(NsRunDir, nsName)

	if err := os.Symlink(n.Path(), symlinkPath); err != nil {
		return err
	}

	fd, err := os.Open(symlinkPath)
	if err != nil {
		if removeErr := os.RemoveAll(symlinkPath); removeErr != nil {
			return removeErr
		}

		return err
	}

	n.symlink = fd

	return nil
}

// Path returns the path of the network namespace handle
func (n *NetNs) Path() string {
	if n == nil || n.netNS == nil {
		return ""
	}
	return n.netNS.Path()
}

// Close closes this network namespace
func (n *NetNs) Close() error {
	if n == nil || n.netNS == nil {
		return nil
	}
	return n.netNS.Close()
}

// Remove ensures this network namespace handle is closed and removed
func (n *NetNs) Remove() error {
	n.Lock()
	defer n.Unlock()

	if n.closed {
		// netNsRemove() can be called multiple
		// times without returning an error.
		return nil
	}

	if err := n.symlinkRemove(); err != nil {
		return err
	}

	if err := n.Close(); err != nil {
		return err
	}

	n.closed = true

	if n.restored {
		// we got namespaces in the form of
		// /var/run/netns/cni-0d08effa-06eb-a963-f51a-e2b0eceffc5d
		// but /var/run on most system is symlinked to /run so we first resolve
		// the symlink and then try and see if it's mounted
		fp, err := symlink.FollowSymlinkInScope(n.Path(), "/")
		if err != nil {
			return err
		}
		if mounted, err := mount.Mounted(fp); err == nil && mounted {
			if err := unix.Unmount(fp, unix.MNT_DETACH); err != nil {
				return err
			}
		}

		if n.Path() != "" {
			if err := os.RemoveAll(n.Path()); err != nil {
				return err
			}
		}
	}

	return nil
}

func (n *NetNs) symlinkRemove() error {
	if err := n.symlink.Close(); err != nil {
		return fmt.Errorf("failed to close net ns symlink: %v", err)
	}

	if err := os.RemoveAll(n.symlink.Name()); err != nil {
		return fmt.Errorf("failed to remove net ns symlink: %v", err)
	}

	return nil
}

func hostNetNsPath() (string, error) {
	netNS, err := ns.GetCurrentNS()
	if err != nil {
		return "", err
	}

	defer netNS.Close()
	return netNS.Path(), nil
}

// Creates a new persistent network namespace and returns an object
// representing that namespace, without switching to it
func NewNS() (ns.NetNS, error) {
	const nsRunDir = "/var/run/netns"

	b := make([]byte, 16)
	_, err := rand.Reader.Read(b)
	if err != nil {
		return nil, fmt.Errorf("failed to generate random netns name: %v", err)
	}

	err = os.MkdirAll(nsRunDir, 0755)
	if err != nil {
		return nil, err
	}

	// create an empty file at the mount point
	nsName := fmt.Sprintf("cni-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
	nsPath := path.Join(nsRunDir, nsName)
	mountPointFd, err := os.Create(nsPath)
	if err != nil {
		return nil, err
	}
	mountPointFd.Close()

	// Ensure the mount point is cleaned up on errors; if the namespace
	// was successfully mounted this will have no effect because the file
	// is in-use
	defer os.RemoveAll(nsPath)

	var wg sync.WaitGroup
	wg.Add(1)

	// do namespace work in a dedicated goroutine, so that we can safely
	// Lock/Unlock OSThread without upsetting the lock/unlock state of
	// the caller of this function
	go (func() {
		defer wg.Done()
		runtime.LockOSThread()

		var origNS ns.NetNS
		origNS, err = ns.GetNS(getCurrentThreadNetNSPath())
		if err != nil {
			return
		}
		defer origNS.Close()

		// create a new netns on the current thread
		err = unix.Unshare(unix.CLONE_NEWNET)
		if err != nil {
			return
		}
		defer origNS.Set()

		// bind mount the new netns from the current thread onto the mount point
		err = unix.Mount(getCurrentThreadNetNSPath(), nsPath, "none", unix.MS_BIND, "")
		if err != nil {
			return
		}

		_, err = os.Open(nsPath)
		if err != nil {
			return
		}
	})()
	wg.Wait()

	if err != nil {
		unix.Unmount(nsPath, unix.MNT_DETACH)
		return nil, fmt.Errorf("failed to create namespace: %v", err)
	}

	return ns.GetNS(nsPath)
}

func getCurrentThreadNetNSPath() string {
	// /proc/self/ns/net returns the namespace of the main thread, not
	// of whatever thread this goroutine is running on.  Make sure we
	// use the thread's net namespace since the thread is switching around
	return fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid())
}
<commit_msg>Fix errcheck lint for network namespace creation<commit_after>// +build linux

package sandbox

import (
	"crypto/rand"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"sync"

	"github.com/containernetworking/plugins/pkg/ns"
	"github.com/docker/docker/pkg/mount"
	"github.com/docker/docker/pkg/symlink"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/sys/unix"
)

// Get returns the NetNs for a given NetNsIface
func (n *NetNs) Get() *NetNs {
	return n
}

// Initialized returns true if the NetNs is already initialized
func (n *NetNs) Initialized() bool {
	return n.initialized
}

// Initialize does the necessary setup for a NetNs
func (n *NetNs) Initialize() (NetNsIface, error) {
	netNS, err := NewNS()
	if err != nil {
		return nil, err
	}
	n.netNS = netNS
	n.closed = false
	n.initialized = true
	return n, nil
}

func getNetNs(nsPath string) (*NetNs, error) {
	netNS, err := ns.GetNS(nsPath)
	if err != nil {
		return nil, err
	}

	return &NetNs{netNS: netNS, closed: false, restored: true}, nil
}

// NetNs handles data pertaining a network namespace
type NetNs struct {
	sync.Mutex
	netNS       ns.NetNS
	symlink     *os.File
	closed      bool
	restored    bool
	initialized bool
}

// SymlinkCreate creates the necessary symlinks for the NetNs
func (n *NetNs) SymlinkCreate(name string) error {
	if n.netNS == nil {
		return errors.New("no netns set up")
	}
	b := make([]byte, 4)
	_, randErr := rand.Reader.Read(b)
	if randErr != nil {
		return randErr
	}

	nsName := fmt.Sprintf("%s-%x", name, b)
	symlinkPath := filepath.Join(NsRunDir, nsName)

	if err := os.Symlink(n.Path(), symlinkPath); err != nil {
		return err
	}

	fd, err := os.Open(symlinkPath)
	if err != nil {
		if removeErr := os.RemoveAll(symlinkPath); removeErr != nil {
			return removeErr
		}

		return err
	}

	n.symlink = fd

	return nil
}

// Path returns the path of the network namespace handle
func (n *NetNs) Path() string {
	if n == nil || n.netNS == nil {
		return ""
	}
	return n.netNS.Path()
}

// Close closes this network namespace
func (n *NetNs) Close() error {
	if n == nil || n.netNS == nil {
		return nil
	}
	return n.netNS.Close()
}

// Remove ensures this network namespace handle is closed and removed
func (n *NetNs) Remove() error {
	n.Lock()
	defer n.Unlock()

	if n.closed {
		// netNsRemove() can be called multiple
		// times without returning an error.
		return nil
	}

	if err := n.symlinkRemove(); err != nil {
		return err
	}

	if err := n.Close(); err != nil {
		return err
	}

	n.closed = true

	if n.restored {
		// we got namespaces in the form of
		// /var/run/netns/cni-0d08effa-06eb-a963-f51a-e2b0eceffc5d
		// but /var/run on most system is symlinked to /run so we first resolve
		// the symlink and then try and see if it's mounted
		fp, err := symlink.FollowSymlinkInScope(n.Path(), "/")
		if err != nil {
			return err
		}
		if mounted, err := mount.Mounted(fp); err == nil && mounted {
			if err := unix.Unmount(fp, unix.MNT_DETACH); err != nil {
				return err
			}
		}

		if n.Path() != "" {
			if err := os.RemoveAll(n.Path()); err != nil {
				return err
			}
		}
	}

	return nil
}

func (n *NetNs) symlinkRemove() error {
	if err := n.symlink.Close(); err != nil {
		return fmt.Errorf("failed to close net ns symlink: %v", err)
	}

	if err := os.RemoveAll(n.symlink.Name()); err != nil {
		return fmt.Errorf("failed to remove net ns symlink: %v", err)
	}

	return nil
}

func hostNetNsPath() (string, error) {
	netNS, err := ns.GetCurrentNS()
	if err != nil {
		return "", err
	}

	defer netNS.Close()
	return netNS.Path(), nil
}

// Creates a new persistent network namespace and returns an object
// representing that namespace, without switching to it
func NewNS() (ns.NetNS, error) {
	const nsRunDir = "/var/run/netns"

	b := make([]byte, 16)
	_, err := rand.Reader.Read(b)
	if err != nil {
		return nil, fmt.Errorf("failed to generate random netns name: %v", err)
	}

	err = os.MkdirAll(nsRunDir, 0755)
	if err != nil {
		return nil, err
	}

	// create an empty file at the mount point
	nsName := fmt.Sprintf("cni-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
	nsPath := path.Join(nsRunDir, nsName)
	mountPointFd, err := os.Create(nsPath)
	if err != nil {
		return nil, err
	}
	mountPointFd.Close()

	// Ensure the mount point is cleaned up on errors; if the namespace
	// was successfully mounted this will have no effect because the file
	// is in-use
	defer os.RemoveAll(nsPath)

	var wg sync.WaitGroup
	wg.Add(1)

	// do namespace work in a dedicated goroutine, so that we can safely
	// Lock/Unlock OSThread without upsetting the lock/unlock state of
	// the caller of this function
	go (func() {
		defer wg.Done()
		runtime.LockOSThread()

		var origNS ns.NetNS
		origNS, err = ns.GetNS(getCurrentThreadNetNSPath())
		if err != nil {
			return
		}
		defer origNS.Close()

		// create a new netns on the current thread
		err = unix.Unshare(unix.CLONE_NEWNET)
		if err != nil {
			return
		}
		defer func() {
			if err := origNS.Set(); err != nil {
				logrus.Warnf("unable to set network namespace: %v", err)
			}
		}()

		// bind mount the new netns from the current thread onto the mount point
		err = unix.Mount(getCurrentThreadNetNSPath(), nsPath, "none", unix.MS_BIND, "")
		if err != nil {
			return
		}

		_, err = os.Open(nsPath)
		if err != nil {
			return
		}
	})()
	wg.Wait()

	if err != nil {
		if unmountErr := unix.Unmount(nsPath, unix.MNT_DETACH); err != nil {
			return nil, errors.Wrapf(err, "unable to unmount %v: %v",
				nsPath, unmountErr)
		}
		return nil, fmt.Errorf("failed to create namespace: %v", err)
	}

	return ns.GetNS(nsPath)
}

func getCurrentThreadNetNSPath() string {
	// /proc/self/ns/net returns the namespace of the main thread, not
	// of whatever thread this goroutine is running on.  Make sure we
	// use the thread's net namespace since the thread is switching around
	return fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid())
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"sigint.ca/fuse"
)

const (
	tUnknown = iota
	tPrefix
	tObject
)

type s3fs struct {
	bucket string
	key    string
	kind   int
	size   uint64
	body   io.ReadCloser
}

func (ctx s3fs) String() string {
	var kind string
	switch ctx.kind {
	case tUnknown:
		kind = "unknown"
	case tPrefix:
		kind = "prefix"
	case tObject:
		kind = "object"
	}
	return fmt.Sprintf("{key: %#v, kind: %s}", ctx.key, kind)
}

func (ctx s3fs) Root() (fuse.Node, fuse.Error) {
	dprintf("Root: %v", ctx)
	ctx.key = ""
	ctx.kind = tPrefix
	return ctx, nil
}

func (ctx s3fs) Attr() fuse.Attr {
	dprintf("Attr: %v", ctx)
	switch ctx.kind {
	case tPrefix:
		dprintf("Attr: type: directory")
		return fuse.Attr{
			Mode: os.ModeDir | 0755,
			Uid:  serverUid,
			Gid:  serverGid,
		}
	case tObject:
		dprintf("Attr: type: regular file")
		return fuse.Attr{
			Mode: 0644,
			Size: ctx.size,
			Uid:  serverUid,
			Gid:  serverGid,
		}
	default:
		dprintf("Attr: type: unsupported")
		return fuse.Attr{}
	}
}

func (ctx s3fs) Lookup(name string, intr fuse.Intr) (fuse.Node, fuse.Error) {
	dprintf("Lookup: %#v (ctx.key: %#v)", name, ctx.key)
	key := ctx.key + "/" + name
	if ctx.key == "" {
		key = name
	}
	kind, obj, ok := s3Lookup(key)
	if !ok {
		dprintf("Lookup: returned ENOENT for %#v / %#v", ctx.key, name)
		return nil, fuse.ENOENT
	}
	ctx.key = key
	ctx.kind = kind
	if obj != nil {
		ctx.size = uint64(*obj.ContentLength)
		ctx.body = obj.Body
	} else {
		ctx.size = 0
		ctx.body = nil
	}
	dprintf("Lookup: got %v", ctx)
	return ctx, nil
}

func (ctx s3fs) ReadDir(intr fuse.Intr) ([]fuse.Dirent, fuse.Error) {
	dprintf("ReadDir: %v", ctx)
	if ctx.kind != tPrefix {
		return nil, fuse.EIO
	}

	objs, tPrefixes, err := s3ListDir(ctx.key)
	if err != nil {
		dprintf("ReadDir: %v", err)
	}

	out := make([]fuse.Dirent, 0, len(objs)+len(tPrefixes))
	for _, obj := range objs {
		out = append(out, fuse.Dirent{Name: filepath.Base(obj)})
	}
	for _, pre := range tPrefixes {
		out = append(out, fuse.Dirent{Name: filepath.Base(pre)})
	}

	dprintf("ReadDir: got %d dirents: %v", len(out), out)
	return out, nil
}

func (ctx s3fs) ReadAll(intr fuse.Intr) ([]byte, fuse.Error) {
	dprintf("ReadAll: %v", ctx)
	defer ctx.body.Close()
	buf, err := ioutil.ReadAll(ctx.body)
	if err != nil {
		dprintf("ReadAll: error reading %#v: %v", ctx.key, err)
		return nil, fuse.EIO
	}
	return buf, nil
}

func (ctx s3fs) Create(req *fuse.CreateRequest, res *fuse.CreateResponse, intr fuse.Intr) (fuse.Node, fuse.Handle, fuse.Error) {
	dprintf("Create: %v (ctx.key = %#v)", req, ctx.key)
	if ctx.key == "" {
		ctx.key = req.Name
	} else {
		ctx.key += "/" + req.Name
	}
	if req.Mode&os.ModeDir > 0 {
		ctx.kind = tPrefix
	} else {
		ctx.kind = tObject
	}
	return ctx, ctx, nil
}

func (ctx s3fs) Remove(req *fuse.RemoveRequest, intr fuse.Intr) fuse.Error {
	dprintf("Remove: %v (ctx.key = %#v)", req, ctx.key)
	key := ctx.key + "/" + req.Name
	err := s3RemoveObj(key)
	if err != nil {
		dprintf("Remove: error removing %#v: %v", key, err)
		return fuse.EIO
	}
	dprintf("Remove: successfully removed %#v", key)
	return nil
}

func (ctx s3fs) WriteAll(buf []byte, intr fuse.Intr) fuse.Error {
	dprintf("WriteAll: %v", ctx)
	err := s3PutObj(ctx.key, buf)
	if err != nil {
		dprintf("WriteAll: error writing %#v: %v", ctx.key, err)
		return fuse.EIO
	}
	return nil
}
<commit_msg>don't prepend slash when prefix is empty<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"sigint.ca/fuse"
)

const (
	tUnknown = iota
	tPrefix
	tObject
)

type s3fs struct {
	bucket string
	key    string
	kind   int
	size   uint64
	body   io.ReadCloser
}

func (ctx s3fs) String() string {
	var kind string
	switch ctx.kind {
	case tUnknown:
		kind = "unknown"
	case tPrefix:
		kind = "prefix"
	case tObject:
		kind = "object"
	}
	return fmt.Sprintf("{key: %#v, kind: %s}", ctx.key, kind)
}

func (ctx s3fs) Root() (fuse.Node, fuse.Error) {
	dprintf("Root: %v", ctx)
	ctx.key = ""
	ctx.kind = tPrefix
	return ctx, nil
}

func (ctx s3fs) Attr() fuse.Attr {
	dprintf("Attr: %v", ctx)
	switch ctx.kind {
	case tPrefix:
		dprintf("Attr: type: directory")
		return fuse.Attr{
			Mode: os.ModeDir | 0755,
			Uid:  serverUid,
			Gid:  serverGid,
		}
	case tObject:
		dprintf("Attr: type: regular file")
		return fuse.Attr{
			Mode: 0644,
			Size: ctx.size,
			Uid:  serverUid,
			Gid:  serverGid,
		}
	default:
		dprintf("Attr: type: unsupported")
		return fuse.Attr{}
	}
}

func (ctx s3fs) Lookup(name string, intr fuse.Intr) (fuse.Node, fuse.Error) {
	dprintf("Lookup: %#v (ctx.key: %#v)", name, ctx.key)
	var key string
	if ctx.key == "" {
		key = name
	} else {
		key = ctx.key + "/" + name
	}
	kind, obj, ok := s3Lookup(key)
	if !ok {
		dprintf("Lookup: returned ENOENT for %#v / %#v", ctx.key, name)
		return nil, fuse.ENOENT
	}
	ctx.key = key
	ctx.kind = kind
	if obj != nil {
		ctx.size = uint64(*obj.ContentLength)
		ctx.body = obj.Body
	} else {
		ctx.size = 0
		ctx.body = nil
	}
	dprintf("Lookup: got %v", ctx)
	return ctx, nil
}

func (ctx s3fs) ReadDir(intr fuse.Intr) ([]fuse.Dirent, fuse.Error) {
	dprintf("ReadDir: %v", ctx)
	if ctx.kind != tPrefix {
		return nil, fuse.EIO
	}

	objs, tPrefixes, err := s3ListDir(ctx.key)
	if err != nil {
		dprintf("ReadDir: %v", err)
	}

	out := make([]fuse.Dirent, 0, len(objs)+len(tPrefixes))
	for _, obj := range objs {
		out = append(out, fuse.Dirent{Name: filepath.Base(obj)})
	}
	for _, pre := range tPrefixes {
		out = append(out, fuse.Dirent{Name: filepath.Base(pre)})
	}

	dprintf("ReadDir: got %d dirents: %v", len(out), out)
	return out, nil
}

func (ctx s3fs) ReadAll(intr fuse.Intr) ([]byte, fuse.Error) {
	dprintf("ReadAll: %v", ctx)
	defer ctx.body.Close()
	buf, err := ioutil.ReadAll(ctx.body)
	if err != nil {
		dprintf("ReadAll: error reading %#v: %v", ctx.key, err)
		return nil, fuse.EIO
	}
	return buf, nil
}

func (ctx s3fs) Create(req *fuse.CreateRequest, res *fuse.CreateResponse, intr fuse.Intr) (fuse.Node, fuse.Handle, fuse.Error) {
	dprintf("Create: %v (ctx.key = %#v)", req, ctx.key)
	if ctx.key == "" {
		ctx.key = req.Name
	} else {
		ctx.key += "/" + req.Name
	}
	if req.Mode&os.ModeDir > 0 {
		ctx.kind = tPrefix
	} else {
		ctx.kind = tObject
	}
	return ctx, ctx, nil
}

func (ctx s3fs) Remove(req *fuse.RemoveRequest, intr fuse.Intr) fuse.Error {
	dprintf("Remove: %v (ctx.key = %#v)", req, ctx.key)
	var key string
	if ctx.key == "" {
		key = req.Name
	} else {
		key = ctx.key + "/" + req.Name
	}
	err := s3RemoveObj(key)
	if err != nil {
		dprintf("Remove: error removing %#v: %v", key, err)
		return fuse.EIO
	}
	dprintf("Remove: successfully removed %#v", key)
	return nil
}

func (ctx s3fs) WriteAll(buf []byte, intr fuse.Intr) fuse.Error {
	dprintf("WriteAll: %v", ctx)
	err := s3PutObj(ctx.key, buf)
	if err != nil {
		dprintf("WriteAll: error writing %#v: %v", ctx.key, err)
		return fuse.EIO
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/ansible-semaphore/semaphore/db"
	"github.com/ansible-semaphore/semaphore/db/sql"
	"github.com/ansible-semaphore/semaphore/util"
	"github.com/snikch/goodman/transaction"
	"math/rand"
	"os"
	"time"
)

var tablesShouldBeTruncated = [...]string{
	"access_key",
	"event",
	"user__token",
	"project",
	"task__output",
	"task",
	"session",
	"project__environment",
	"project__inventory",
	"project__repository",
	"project__template",
	"project__schedule",
	"project__user",
	"user",
	"project__view",
}

// Test Runner User
func addTestRunnerUser() {
	uid := getUUID()
	testRunnerUser = &db.User{
		Username: "ITU-" + uid,
		Name:     "ITU-" + uid,
		Email:    uid + "@semaphore.test",
		Created:  db.GetParsedTime(time.Now()),
		Admin:    true,
	}

	dbConnect()
	defer store.Sql().Db.Close()

	truncateAll()

	newUser, err := store.CreateUserWithoutPassword(*testRunnerUser)

	if err != nil {
		panic(err)
	}

	testRunnerUser.ID = newUser.ID

	addToken(adminToken, testRunnerUser.ID)
}

func truncateAll() {
	tx, err := store.Sql().Begin()
	if err != nil {
		panic(err)
	}

	_, err = tx.Exec("SET FOREIGN_KEY_CHECKS = 0")
	if err == nil {
		for _, tableName := range tablesShouldBeTruncated {
			tx.Exec("TRUNCATE TABLE " + tableName)
		}
		tx.Exec("SET FOREIGN_KEY_CHECKS = 1")
	}

	if err := tx.Commit(); err != nil {
		panic(err)
	}
}

func removeTestRunnerUser(transactions []*transaction.Transaction) {
	dbConnect()
	defer store.Sql().Db.Close()
	deleteToken(adminToken, testRunnerUser.ID)
	deleteObject(testRunnerUser)
}

// Parameter Substitution
func setupObjectsAndPaths(t *transaction.Transaction) {
	alterRequestPath(t)
	alterRequestBody(t)
}

// Object Lifecycle
func addUserProjectRelation(pid int, user int) {
	_, err := store.CreateProjectUser(db.ProjectUser{
		ProjectID: pid,
		UserID:    user,
		Admin:     true,
	})
	if err != nil {
		panic(err)
	}
}

func deleteUserProjectRelation(pid int, user int) {
	err := store.DeleteProjectUser(pid, user)
	if err != nil {
		panic(err)
	}
}

func addAccessKey(pid *int) *db.AccessKey {
	uid := getUUID()
	secret := "5up3r53cr3t\n"

	key, err := store.CreateAccessKey(db.AccessKey{
		Name:      "ITK-" + uid,
		Type:      "ssh",
		Secret:    &secret,
		ProjectID: pid,
	})

	if err != nil {
		panic(err)
	}
	return &key
}

func addProject() *db.Project {
	uid := getUUID()
	project := db.Project{
		Name:    "ITP-" + uid,
		Created: time.Now(),
	}
	project, err := store.CreateProject(project)
	if err != nil {
		panic(err)
	}
	return &project
}

func addUser() *db.User {
	uid := getUUID()
	user := db.User{
		Created:  time.Now(),
		Username: "ITU-" + uid,
		Email:    "test@semaphore." + uid,
	}

	user, err := store.CreateUserWithoutPassword(user)

	if err != nil {
		panic(err)
	}
	return &user
}

func addView() *db.View {
	view, err := store.CreateView(db.View{
		ProjectID: userProject.ID,
		Title:     "Test",
		Position:  1,
	})

	if err != nil {
		panic(err)
	}

	return &view
}

func addSchedule() *db.Schedule {
	schedule, err := store.CreateSchedule(db.Schedule{
		TemplateID: int(templateID),
		CronFormat: "* * * 1 *",
		ProjectID:  userProject.ID,
	})

	if err != nil {
		panic(err)
	}

	return &schedule
}

func addTask() *db.Task {
	t := db.Task{
		ProjectID:  userProject.ID,
		TemplateID: int(templateID),
		Status:     "testing",
		UserID:     &userPathTestUser.ID,
		Created:    db.GetParsedTime(time.Now()),
	}
	if err := store.Sql().Insert(&t); err != nil {
		fmt.Println("error during insertion of task:")
		if j, err := json.Marshal(t); err == nil {
			fmt.Println(string(j))
		} else {
			fmt.Println("can not stringify task object")
		}
		panic(err)
	}
	return &t
}

func deleteObject(i interface{}) {
	_, err := store.Sql().Delete(i)
	if err != nil {
		panic(err)
	}
}

// Token Handling
func addToken(tok string, user int) {
	_, err := store.CreateAPIToken(db.APIToken{
		ID:      tok,
		Created: time.Now(),
		UserID:  user,
		Expired: false,
	})
	if err != nil {
		panic(err)
	}
}

func deleteToken(tok string, user int) {
	token := db.APIToken{
		ID:     tok,
		UserID: user,
	}
	deleteObject(&token)
}

// HELPERS
var r *rand.Rand
var randSetup = false

func getUUID() string {
	if !randSetup {
		r = rand.New(rand.NewSource(time.Now().UnixNano()))
		randSetup = true
	}
	return randomString(8)
}
func randomString(strlen int) string {
	const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
	result := ""
	for i := 0; i < strlen; i++ {
		index := r.Intn(len(chars))
		result += chars[index : index+1]
	}
	return result
}

func loadConfig() {
	cwd, _ := os.Getwd()
	file, _ := os.Open(cwd + "/.dredd/config.json")
	if err := json.NewDecoder(file).Decode(&util.Config); err != nil {
		fmt.Println("Could not decode configuration!")
		panic(err)
	}
}

var store sql.SqlDb

func dbConnect() {
	store = sql.SqlDb{}

	if err := store.Connect(); err != nil {
		panic(err)
	}
}

func stringInSlice(a string, list []string) (int, bool) {
	for k, b := range list {
		if b == a {
			return k, true
		}
	}
	return 0, false
}

func printError(err error) {
	if err != nil {
		//fmt.Println(err)
		panic(err)
	}
}
<commit_msg>test: use store to create task<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/ansible-semaphore/semaphore/db"
	"github.com/ansible-semaphore/semaphore/db/sql"
	"github.com/ansible-semaphore/semaphore/util"
	"github.com/snikch/goodman/transaction"
	"math/rand"
	"os"
	"time"
)

var tablesShouldBeTruncated = [...]string{
	"access_key",
	"event",
	"user__token",
	"project",
	"task__output",
	"task",
	"session",
	"project__environment",
	"project__inventory",
	"project__repository",
	"project__template",
	"project__schedule",
	"project__user",
	"user",
	"project__view",
}

// Test Runner User
func addTestRunnerUser() {
	uid := getUUID()
	testRunnerUser = &db.User{
		Username: "ITU-" + uid,
		Name:     "ITU-" + uid,
		Email:    uid + "@semaphore.test",
		Created:  db.GetParsedTime(time.Now()),
		Admin:    true,
	}

	dbConnect()
	defer store.Sql().Db.Close()

	truncateAll()

	newUser, err := store.CreateUserWithoutPassword(*testRunnerUser)

	if err != nil {
		panic(err)
	}

	testRunnerUser.ID = newUser.ID

	addToken(adminToken, testRunnerUser.ID)
}

func truncateAll() {
	tx, err := store.Sql().Begin()
	if err != nil {
		panic(err)
	}

	_, err = tx.Exec("SET FOREIGN_KEY_CHECKS = 0")
	if err == nil {
		for _, tableName := range tablesShouldBeTruncated {
			tx.Exec("TRUNCATE TABLE " + tableName)
		}
		tx.Exec("SET FOREIGN_KEY_CHECKS = 1")
	}

	if err := tx.Commit(); err != nil {
		panic(err)
	}
}

func removeTestRunnerUser(transactions []*transaction.Transaction) {
	dbConnect()
	defer store.Sql().Db.Close()
	deleteToken(adminToken, testRunnerUser.ID)
	deleteObject(testRunnerUser)
}

// Parameter Substitution
func setupObjectsAndPaths(t *transaction.Transaction) {
	alterRequestPath(t)
	alterRequestBody(t)
}

// Object Lifecycle
func addUserProjectRelation(pid int, user int) {
	_, err := store.CreateProjectUser(db.ProjectUser{
		ProjectID: pid,
		UserID:    user,
		Admin:     true,
	})
	if err != nil {
		panic(err)
	}
}

func deleteUserProjectRelation(pid int, user int) {
	err := store.DeleteProjectUser(pid, user)
	if err != nil {
		panic(err)
	}
}

func addAccessKey(pid *int) *db.AccessKey {
	uid := getUUID()
	secret := "5up3r53cr3t\n"

	key, err := store.CreateAccessKey(db.AccessKey{
		Name:      "ITK-" + uid,
		Type:      "ssh",
		Secret:    &secret,
		ProjectID: pid,
	})

	if err != nil {
		panic(err)
	}
	return &key
}

func addProject() *db.Project {
	uid := getUUID()
	project := db.Project{
		Name:    "ITP-" + uid,
		Created: time.Now(),
	}
	project, err := store.CreateProject(project)
	if err != nil {
		panic(err)
	}
	return &project
}

func addUser() *db.User {
	uid := getUUID()
	user := db.User{
		Created:  time.Now(),
		Username: "ITU-" + uid,
		Email:    "test@semaphore." + uid,
	}

	user, err := store.CreateUserWithoutPassword(user)

	if err != nil {
		panic(err)
	}
	return &user
}

func addView() *db.View {
	view, err := store.CreateView(db.View{
		ProjectID: userProject.ID,
		Title:     "Test",
		Position:  1,
	})

	if err != nil {
		panic(err)
	}

	return &view
}

func addSchedule() *db.Schedule {
	schedule, err := store.CreateSchedule(db.Schedule{
		TemplateID: int(templateID),
		CronFormat: "* * * 1 *",
		ProjectID:  userProject.ID,
	})

	if err != nil {
		panic(err)
	}

	return &schedule
}

func addTask() *db.Task {
	t := db.Task{
		ProjectID:  userProject.ID,
		TemplateID: int(templateID),
		Status:     "testing",
		UserID:     &userPathTestUser.ID,
		Created:    db.GetParsedTime(time.Now()),
	}
	_, err := store.CreateTask(t)
	if err != nil {
		fmt.Println("error during insertion of task:")
		if j, err := json.Marshal(t); err == nil {
			fmt.Println(string(j))
		} else {
			fmt.Println("can not stringify task object")
		}
		panic(err)
	}
	return &t
}

func deleteObject(i interface{}) {
	_, err := store.Sql().Delete(i)
	if err != nil {
		panic(err)
	}
}

// Token Handling
func addToken(tok string, user int) {
	_, err := store.CreateAPIToken(db.APIToken{
		ID:      tok,
		Created: time.Now(),
		UserID:  user,
		Expired: false,
	})
	if err != nil {
		panic(err)
	}
}

func deleteToken(tok string, user int) {
	token := db.APIToken{
		ID:     tok,
		UserID: user,
	}
	deleteObject(&token)
}

// HELPERS
var r *rand.Rand
var randSetup = false

func getUUID() string {
	if !randSetup {
		r = rand.New(rand.NewSource(time.Now().UnixNano()))
		randSetup = true
	}
	return randomString(8)
}
func randomString(strlen int) string {
	const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
	result := ""
	for i := 0; i < strlen; i++ {
		index := r.Intn(len(chars))
		result += chars[index : index+1]
	}
	return result
}

func loadConfig() {
	cwd, _ := os.Getwd()
	file, _ := os.Open(cwd + "/.dredd/config.json")
	if err := json.NewDecoder(file).Decode(&util.Config); err != nil {
		fmt.Println("Could not decode configuration!")
		panic(err)
	}
}

var store sql.SqlDb

func dbConnect() {
	store = sql.SqlDb{}

	if err := store.Connect(); err != nil {
		panic(err)
	}
}

func stringInSlice(a string, list []string) (int, bool) {
	for k, b := range list {
		if b == a {
			return k, true
		}
	}
	return 0, false
}

func printError(err error) {
	if err != nil {
		//fmt.Println(err)
		panic(err)
	}
}
<|endoftext|>


<commit_before>package weed_server

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"mime"
	"mime/multipart"
	"net/http"
	"net/url"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/images"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")

type JsonEncode struct {
	Data   interface{} `json:"data"`
	Msg    string      `json:"msg"`
	Status int         `json:"status"`
}

func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Methods", "GET")
	w.Header().Set("Access-Control-Max-Age", "1000")

	n := new(storage.Needle)
	vid, fid, filename, ext, _ := parseURLPath(r.URL.Path)
	volumeId, err := storage.NewVolumeId(vid)
	if err != nil {
		glog.V(2).Infoln("parsing error:", err, r.URL.Path)
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	err = n.ParsePath(fid)
	if err != nil {
		glog.V(2).Infoln("parsing fid error:", err, r.URL.Path)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	glog.V(4).Infoln("volume", volumeId, "reading", n)
	if !vs.store.HasVolume(volumeId) {
		if !vs.ReadRedirect {
			glog.V(2).Infoln("volume is not local:", err, r.URL.Path)
			w.WriteHeader(http.StatusNotFound)
			return
		}
		lookupResult, err := operation.Lookup(vs.GetMasterNode(), volumeId.String())
		glog.V(2).Infoln("volume", volumeId, "found on", lookupResult, "error", err)
		if err == nil && len(lookupResult.Locations) > 0 {
			u, _ := url.Parse(util.NormalizeUrl(lookupResult.Locations[0].PublicUrl))
			u.Path = r.URL.Path
			arg := url.Values{}
			if c := r.FormValue("collection"); c != "" {
				arg.Set("collection", c)
			}
			u.RawQuery = arg.Encode()
			http.Redirect(w, r, u.String(), http.StatusMovedPermanently)

		} else {
			glog.V(2).Infoln("lookup error:", err, r.URL.Path)
			w.WriteHeader(http.StatusNotFound)
		}
		return
	}
	cookie := n.Cookie
	count, e := vs.store.ReadVolumeNeedle(volumeId, n)
	glog.V(4).Infoln("read bytes", count, "error", e)
	if e != nil || count < 0 {
		glog.V(0).Infoln("read error:", e, r.URL.Path)
		w.WriteHeader(http.StatusNotFound)
		return
	}
	defer n.ReleaseMemory()
	if n.Cookie != cookie {
		glog.V(0).Infoln("request", r.URL.Path, "with unmaching cookie seen:", cookie, "expected:", n.Cookie, "from", r.RemoteAddr, "agent", r.UserAgent())
		w.WriteHeader(http.StatusNotFound)
		return
	}
	if n.LastModified != 0 {
		w.Header().Set("Last-Modified", time.Unix(int64(n.LastModified), 0).UTC().Format(http.TimeFormat))
		if r.Header.Get("If-Modified-Since") != "" {
			if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil {
				if t.Unix() >= int64(n.LastModified) {
					w.WriteHeader(http.StatusNotModified)
					return
				}
			}
		}
	}
	etag := n.Etag()
	if inm := r.Header.Get("If-None-Match"); inm == etag {
		w.WriteHeader(http.StatusNotModified)
		return
	}
	w.Header().Set("Etag", etag)

	if n.HasPairs() {
		pairMap := make(map[string]string)
		err = json.Unmarshal(n.Pairs, &pairMap)
		if err != nil {
			glog.V(0).Infoln("Unmarshal pairs error:", err)
		}
		for k, v := range pairMap {
			w.Header().Set(k, v)
		}
	}

	if vs.tryHandleChunkedFile(n, filename, w, r) {
		return
	}

	if n.NameSize > 0 && filename == "" {
		filename = string(n.Name)
		if ext == "" {
			ext = path.Ext(filename)
		}
	}
	mtype := ""
	if n.MimeSize > 0 {
		mt := string(n.Mime)
		if !strings.HasPrefix(mt, "application/octet-stream") {
			mtype = mt
		}
	}
	ext = strings.ToLower(ext) // 后缀先转小写，防止匹配不上大写的后缀
	if ext != ".gz" {
		if n.IsGzipped() {
			if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
				w.Header().Set("Content-Encoding", "gzip")
			} else {
				if n.Data, err = operation.UnGzipData(n.Data); err != nil {
					glog.V(0).Infoln("ungzip error:", err, r.URL.Path)
				}
			}
		}
	}

	if ext == ".png" || ext == ".jpg" || ext == ".jpeg" || ext == ".gif" {
		width, height, rotate, fill := 0, 0, 0, 0
		if r.FormValue("w") != "" {
			width, _ = strconv.Atoi(r.FormValue("w"))
		}
		if r.FormValue("h") != "" {
			height, _ = strconv.Atoi(r.FormValue("h"))
		}
		if r.FormValue("r") != "" {
			rotate, _ = strconv.Atoi(r.FormValue("r"))
		}
		if r.FormValue("f") != "" {
			fill, _ = strconv.Atoi(r.FormValue("f"))
		}

		//缓存
		if r.Header.Get("exist") != "1" && (r.FormValue("w") != "" || r.FormValue("h") != "" || r.FormValue("r") != "" || r.FormValue("f") != "") {
			//glog.V(0).Infoln("生成")
			n.Data, _, _ = images.Resized(ext, n.Data, width, height, fill)
			n.Data = images.Rotate(ext, n.Data, rotate)

			reqUrl := r.Header.Get("path") + "?w=" + r.FormValue("w") + "&h=" + r.FormValue("h") + "&r=" + r.FormValue("r") + "&f=" + r.FormValue("f")
			jwt := security.GetJwt(r)
			_, err = operation.Upload("http://"+r.Host+reqUrl, filename, bytes.NewReader(n.Data), false, "image/jpeg", nil, jwt)
			if err != nil {
				glog.V(0).Infoln(err)
			}
		}
	}

	if r.FormValue("md5") != "" {
		md5Ctx := md5.New()
		md5Ctx.Write(n.Data)
		cipherStr := md5Ctx.Sum(nil)
		glog.V(0).Infoln("md5", hex.EncodeToString(cipherStr))

		fileInfo := map[string]interface{}{
			"url": r.Header.Get("path"),
			"md5": hex.EncodeToString(cipherStr),
		}
		fmt.Fprintf(w, "%v", (&JsonEncode{fileInfo, "success", 200}).ReturnJson())
		return
	}

	//重命名
	if r.FormValue("rename") != "" {
		filename = r.FormValue("rename") + ext
	}

	if e := writeResponseContent(filename, mtype, bytes.NewReader(n.Data), w, r); e != nil {
		glog.V(2).Infoln("response write error:", e)
	}
}

func (vs *VolumeServer) FaviconHandler(w http.ResponseWriter, r *http.Request) {
	data, err := images.Asset("favicon/favicon.ico")
	if err != nil {
		glog.V(2).Infoln("favicon read error:", err)
		return
	}

	if e := writeResponseContent("favicon.ico", "image/x-icon", bytes.NewReader(data), w, r); e != nil {
		glog.V(2).Infoln("response write error:", e)
	}
}

func (vs *VolumeServer) tryHandleChunkedFile(n *storage.Needle, fileName string, w http.ResponseWriter, r *http.Request) (processed bool) {
	if !n.IsChunkedManifest() {
		return false
	}

	chunkManifest, e := operation.LoadChunkManifest(n.Data, n.IsGzipped())
	if e != nil {
		glog.V(0).Infof("load chunked manifest (%s) error: %v", r.URL.Path, e)
		return false
	}
	if fileName == "" && chunkManifest.Name != "" {
		fileName = chunkManifest.Name
	}
	mType := ""
	if chunkManifest.Mime != "" {
		mt := chunkManifest.Mime
		if !strings.HasPrefix(mt, "application/octet-stream") {
			mType = mt
		}
	}

	w.Header().Set("X-File-Store", "chunked")

	chunkedFileReader := &operation.ChunkedFileReader{
		Manifest: chunkManifest,
		Master:   vs.GetMasterNode(),
	}
	defer chunkedFileReader.Close()
	if e := writeResponseContent(fileName, mType, chunkedFileReader, w, r); e != nil {
		glog.V(2).Infoln("response write error:", e)
	}
	return true
}

func writeResponseContent(filename, mimeType string, rs io.ReadSeeker, w http.ResponseWriter, r *http.Request) error {
	totalSize, e := rs.Seek(0, 2)
	if mimeType == "" {
		if ext := path.Ext(filename); ext != "" {
			mimeType = mime.TypeByExtension(ext)
		}
	}
	if mimeType != "" {
		w.Header().Set("Content-Type", mimeType)
	}
	if filename != "" {
		contentDisposition := "inline"
		if r.FormValue("dl") != "" {
			if dl, _ := strconv.ParseBool(r.FormValue("dl")); dl {
				contentDisposition = "attachment"
			}
		}
		w.Header().Set("Content-Disposition", contentDisposition+`; filename="`+fileNameEscaper.Replace(filename)+`"`)
	}
	w.Header().Set("Accept-Ranges", "bytes")
	if r.Method == "HEAD" {
		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
		return nil
	}
	rangeReq := r.Header.Get("Range")
	if rangeReq == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
		if _, e = rs.Seek(0, 0); e != nil {
			return e
		}
		_, e = io.Copy(w, rs)
		return e
	}

	//the rest is dealing with partial content request
	//mostly copy from src/pkg/net/http/fs.go
	ranges, err := parseRange(rangeReq, totalSize)
	if err != nil {
		http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
		return nil
	}
	if sumRangesSize(ranges) > totalSize {
		// The total number of bytes in all the ranges
		// is larger than the size of the file by
		// itself, so this is probably an attack, or a
		// dumb client.  Ignore the range request.
		return nil
	}
	if len(ranges) == 0 {
		return nil
	}
	if len(ranges) == 1 {
		// RFC 2616, Section 14.16:
		// "When an HTTP message includes the content of a single
		// range (for example, a response to a request for a
		// single range, or to a request for a set of ranges
		// that overlap without any holes), this content is
		// transmitted with a Content-Range header, and a
		// Content-Length header showing the number of bytes
		// actually transferred.
		// ...
		// A response to a request for a single range MUST NOT
		// be sent using the multipart/byteranges media type."
		ra := ranges[0]
		w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
		w.Header().Set("Content-Range", ra.contentRange(totalSize))
		w.WriteHeader(http.StatusPartialContent)
		if _, e = rs.Seek(ra.start, 0); e != nil {
			return e
		}

		_, e = io.CopyN(w, rs, ra.length)
		return e
	}
	// process multiple ranges
	for _, ra := range ranges {
		if ra.start > totalSize {
			http.Error(w, "Out of Range", http.StatusRequestedRangeNotSatisfiable)
			return nil
		}
	}
	sendSize := rangesMIMESize(ranges, mimeType, totalSize)
	pr, pw := io.Pipe()
	mw := multipart.NewWriter(pw)
	w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary())
	sendContent := pr
	defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish.
	go func() {
		for _, ra := range ranges {
			part, e := mw.CreatePart(ra.mimeHeader(mimeType, totalSize))
			if e != nil {
				pw.CloseWithError(e)
				return
			}
			if _, e = rs.Seek(ra.start, 0); e != nil {
				pw.CloseWithError(e)
				return
			}
			if _, e = io.CopyN(part, rs, ra.length); e != nil {
				pw.CloseWithError(e)
				return
			}
		}
		mw.Close()
		pw.Close()
	}()
	if w.Header().Get("Content-Encoding") == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10))
	}
	w.WriteHeader(http.StatusPartialContent)
	_, e = io.CopyN(w, sendContent, sendSize)
	return e
}

//返回json
func (j *JsonEncode) ReturnJson() string {
	b, _ := json.MarshalIndent(j, "", "	")
	return string(b)
}
<commit_msg>增加尺寸显示<commit_after>package weed_server

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"mime"
	"mime/multipart"
	"net/http"
	"net/url"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/images"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")

type JsonEncode struct {
	Data   interface{} `json:"data"`
	Msg    string      `json:"msg"`
	Status int         `json:"status"`
}

func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Methods", "GET")
	w.Header().Set("Access-Control-Max-Age", "1000")

	n := new(storage.Needle)
	vid, fid, filename, ext, _ := parseURLPath(r.URL.Path)
	volumeId, err := storage.NewVolumeId(vid)
	if err != nil {
		glog.V(2).Infoln("parsing error:", err, r.URL.Path)
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	err = n.ParsePath(fid)
	if err != nil {
		glog.V(2).Infoln("parsing fid error:", err, r.URL.Path)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	glog.V(4).Infoln("volume", volumeId, "reading", n)
	if !vs.store.HasVolume(volumeId) {
		if !vs.ReadRedirect {
			glog.V(2).Infoln("volume is not local:", err, r.URL.Path)
			w.WriteHeader(http.StatusNotFound)
			return
		}
		lookupResult, err := operation.Lookup(vs.GetMasterNode(), volumeId.String())
		glog.V(2).Infoln("volume", volumeId, "found on", lookupResult, "error", err)
		if err == nil && len(lookupResult.Locations) > 0 {
			u, _ := url.Parse(util.NormalizeUrl(lookupResult.Locations[0].PublicUrl))
			u.Path = r.URL.Path
			arg := url.Values{}
			if c := r.FormValue("collection"); c != "" {
				arg.Set("collection", c)
			}
			u.RawQuery = arg.Encode()
			http.Redirect(w, r, u.String(), http.StatusMovedPermanently)

		} else {
			glog.V(2).Infoln("lookup error:", err, r.URL.Path)
			w.WriteHeader(http.StatusNotFound)
		}
		return
	}
	cookie := n.Cookie
	count, e := vs.store.ReadVolumeNeedle(volumeId, n)
	glog.V(4).Infoln("read bytes", count, "error", e)
	if e != nil || count < 0 {
		glog.V(0).Infoln("read error:", e, r.URL.Path)
		w.WriteHeader(http.StatusNotFound)
		return
	}
	defer n.ReleaseMemory()
	if n.Cookie != cookie {
		glog.V(0).Infoln("request", r.URL.Path, "with unmaching cookie seen:", cookie, "expected:", n.Cookie, "from", r.RemoteAddr, "agent", r.UserAgent())
		w.WriteHeader(http.StatusNotFound)
		return
	}
	if n.LastModified != 0 {
		w.Header().Set("Last-Modified", time.Unix(int64(n.LastModified), 0).UTC().Format(http.TimeFormat))
		if r.Header.Get("If-Modified-Since") != "" {
			if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil {
				if t.Unix() >= int64(n.LastModified) {
					w.WriteHeader(http.StatusNotModified)
					return
				}
			}
		}
	}
	etag := n.Etag()
	if inm := r.Header.Get("If-None-Match"); inm == etag {
		w.WriteHeader(http.StatusNotModified)
		return
	}
	w.Header().Set("Etag", etag)

	if n.HasPairs() {
		pairMap := make(map[string]string)
		err = json.Unmarshal(n.Pairs, &pairMap)
		if err != nil {
			glog.V(0).Infoln("Unmarshal pairs error:", err)
		}
		for k, v := range pairMap {
			w.Header().Set(k, v)
		}
	}

	if vs.tryHandleChunkedFile(n, filename, w, r) {
		return
	}

	if n.NameSize > 0 && filename == "" {
		filename = string(n.Name)
		if ext == "" {
			ext = path.Ext(filename)
		}
	}
	mtype := ""
	if n.MimeSize > 0 {
		mt := string(n.Mime)
		if !strings.HasPrefix(mt, "application/octet-stream") {
			mtype = mt
		}
	}
	ext = strings.ToLower(ext) // 后缀先转小写，防止匹配不上大写的后缀
	if ext != ".gz" {
		if n.IsGzipped() {
			if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
				w.Header().Set("Content-Encoding", "gzip")
			} else {
				if n.Data, err = operation.UnGzipData(n.Data); err != nil {
					glog.V(0).Infoln("ungzip error:", err, r.URL.Path)
				}
			}
		}
	}

	if ext == ".png" || ext == ".jpg" || ext == ".jpeg" || ext == ".gif" {
		width, height, rotate, fill := 0, 0, 0, 0
		if r.FormValue("w") != "" {
			width, _ = strconv.Atoi(r.FormValue("w"))
		}
		if r.FormValue("h") != "" {
			height, _ = strconv.Atoi(r.FormValue("h"))
		}
		if r.FormValue("r") != "" {
			rotate, _ = strconv.Atoi(r.FormValue("r"))
		}
		if r.FormValue("f") != "" {
			fill, _ = strconv.Atoi(r.FormValue("f"))
		}

		//缓存
		if r.Header.Get("exist") != "1" && (r.FormValue("w") != "" || r.FormValue("h") != "" || r.FormValue("r") != "" || r.FormValue("f") != "") {
			//glog.V(0).Infoln("生成")
			n.Data, _, _ = images.Resized(ext, n.Data, width, height, fill)
			n.Data = images.Rotate(ext, n.Data, rotate)

			reqUrl := r.Header.Get("path") + "?w=" + r.FormValue("w") + "&h=" + r.FormValue("h") + "&r=" + r.FormValue("r") + "&f=" + r.FormValue("f")
			jwt := security.GetJwt(r)
			_, err = operation.Upload("http://"+r.Host+reqUrl, filename, bytes.NewReader(n.Data), false, "image/jpeg", nil, jwt)
			if err != nil {
				glog.V(0).Infoln(err)
			}
		}
	}

	if r.FormValue("md5") != "" {
		md5Ctx := md5.New()
		md5Ctx.Write(n.Data)
		cipherStr := md5Ctx.Sum(nil)
		// glog.V(0).Infoln("md5", hex.EncodeToString(cipherStr))

		fileInfo := map[string]interface{}{
			"url": r.Header.Get("path"),
			"md5": hex.EncodeToString(cipherStr),
		}
		fmt.Fprintf(w, "%v", (&JsonEncode{fileInfo, "success", 200}).ReturnJson())
		return
	}

	if r.FormValue("size") != "" {
		// glog.V(0).Infoln("size", len(n.Data))
		fileInfo := map[string]interface{}{
			"size": len(n.Data),
		}
		fmt.Fprintf(w, "%v", (&JsonEncode{fileInfo, "success", 200}).ReturnJson())
		return
	}

	//重命名
	if r.FormValue("rename") != "" {
		filename = r.FormValue("rename") + ext
	}

	if e := writeResponseContent(filename, mtype, bytes.NewReader(n.Data), w, r); e != nil {
		glog.V(2).Infoln("response write error:", e)
	}
}

func (vs *VolumeServer) FaviconHandler(w http.ResponseWriter, r *http.Request) {
	data, err := images.Asset("favicon/favicon.ico")
	if err != nil {
		glog.V(2).Infoln("favicon read error:", err)
		return
	}

	if e := writeResponseContent("favicon.ico", "image/x-icon", bytes.NewReader(data), w, r); e != nil {
		glog.V(2).Infoln("response write error:", e)
	}
}

func (vs *VolumeServer) tryHandleChunkedFile(n *storage.Needle, fileName string, w http.ResponseWriter, r *http.Request) (processed bool) {
	if !n.IsChunkedManifest() {
		return false
	}

	chunkManifest, e := operation.LoadChunkManifest(n.Data, n.IsGzipped())
	if e != nil {
		glog.V(0).Infof("load chunked manifest (%s) error: %v", r.URL.Path, e)
		return false
	}
	if fileName == "" && chunkManifest.Name != "" {
		fileName = chunkManifest.Name
	}
	mType := ""
	if chunkManifest.Mime != "" {
		mt := chunkManifest.Mime
		if !strings.HasPrefix(mt, "application/octet-stream") {
			mType = mt
		}
	}

	w.Header().Set("X-File-Store", "chunked")

	chunkedFileReader := &operation.ChunkedFileReader{
		Manifest: chunkManifest,
		Master:   vs.GetMasterNode(),
	}
	defer chunkedFileReader.Close()
	if e := writeResponseContent(fileName, mType, chunkedFileReader, w, r); e != nil {
		glog.V(2).Infoln("response write error:", e)
	}
	return true
}

func writeResponseContent(filename, mimeType string, rs io.ReadSeeker, w http.ResponseWriter, r *http.Request) error {
	totalSize, e := rs.Seek(0, 2)
	if mimeType == "" {
		if ext := path.Ext(filename); ext != "" {
			mimeType = mime.TypeByExtension(ext)
		}
	}
	if mimeType != "" {
		w.Header().Set("Content-Type", mimeType)
	}
	if filename != "" {
		contentDisposition := "inline"
		if r.FormValue("dl") != "" {
			if dl, _ := strconv.ParseBool(r.FormValue("dl")); dl {
				contentDisposition = "attachment"
			}
		}
		w.Header().Set("Content-Disposition", contentDisposition+`; filename="`+fileNameEscaper.Replace(filename)+`"`)
	}
	w.Header().Set("Accept-Ranges", "bytes")
	if r.Method == "HEAD" {
		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
		return nil
	}
	rangeReq := r.Header.Get("Range")
	if rangeReq == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
		if _, e = rs.Seek(0, 0); e != nil {
			return e
		}
		_, e = io.Copy(w, rs)
		return e
	}

	//the rest is dealing with partial content request
	//mostly copy from src/pkg/net/http/fs.go
	ranges, err := parseRange(rangeReq, totalSize)
	if err != nil {
		http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
		return nil
	}
	if sumRangesSize(ranges) > totalSize {
		// The total number of bytes in all the ranges
		// is larger than the size of the file by
		// itself, so this is probably an attack, or a
		// dumb client.  Ignore the range request.
		return nil
	}
	if len(ranges) == 0 {
		return nil
	}
	if len(ranges) == 1 {
		// RFC 2616, Section 14.16:
		// "When an HTTP message includes the content of a single
		// range (for example, a response to a request for a
		// single range, or to a request for a set of ranges
		// that overlap without any holes), this content is
		// transmitted with a Content-Range header, and a
		// Content-Length header showing the number of bytes
		// actually transferred.
		// ...
		// A response to a request for a single range MUST NOT
		// be sent using the multipart/byteranges media type."
		ra := ranges[0]
		w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
		w.Header().Set("Content-Range", ra.contentRange(totalSize))
		w.WriteHeader(http.StatusPartialContent)
		if _, e = rs.Seek(ra.start, 0); e != nil {
			return e
		}

		_, e = io.CopyN(w, rs, ra.length)
		return e
	}
	// process multiple ranges
	for _, ra := range ranges {
		if ra.start > totalSize {
			http.Error(w, "Out of Range", http.StatusRequestedRangeNotSatisfiable)
			return nil
		}
	}
	sendSize := rangesMIMESize(ranges, mimeType, totalSize)
	pr, pw := io.Pipe()
	mw := multipart.NewWriter(pw)
	w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary())
	sendContent := pr
	defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish.
	go func() {
		for _, ra := range ranges {
			part, e := mw.CreatePart(ra.mimeHeader(mimeType, totalSize))
			if e != nil {
				pw.CloseWithError(e)
				return
			}
			if _, e = rs.Seek(ra.start, 0); e != nil {
				pw.CloseWithError(e)
				return
			}
			if _, e = io.CopyN(part, rs, ra.length); e != nil {
				pw.CloseWithError(e)
				return
			}
		}
		mw.Close()
		pw.Close()
	}()
	if w.Header().Get("Content-Encoding") == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10))
	}
	w.WriteHeader(http.StatusPartialContent)
	_, e = io.CopyN(w, sendContent, sendSize)
	return e
}

//返回json
func (j *JsonEncode) ReturnJson() string {
	b, _ := json.MarshalIndent(j, "", "	")
	return string(b)
}
<|endoftext|>


<commit_before>package rpc

import (
	"context"
	"encoding/json"
	"time"

	msgpack "gopkg.in/vmihailenco/msgpack.v2"

	"github.com/TykTechnologies/tyk/storage"
)

type AnalyticsRecord struct {
	Method        string
	Path          string
	RawPath       string
	ContentLength int64
	UserAgent     string
	Day           int
	Month         time.Month
	Year          int
	Hour          int
	ResponseCode  int
	APIKey        string
	TimeStamp     time.Time
	APIVersion    string
	APIName       string
	APIID         string
	OrgID         string
	OauthID       string
	RequestTime   int64
	RawRequest    string
	RawResponse   string
	IPAddress     string
	Geo           GeoData
	Tags          []string
	Alias         string
	TrackPath     bool
	ExpireAt      time.Time `bson:"expireAt" json:"expireAt"`
}
type GeoData struct {
	Country struct {
		ISOCode string `maxminddb:"iso_code"`
	} `maxminddb:"country"`

	City struct {
		GeoNameID uint              `maxminddb:"geoname_id"`
		Names     map[string]string `maxminddb:"names"`
	} `maxminddb:"city"`

	Location struct {
		Latitude  float64 `maxminddb:"latitude"`
		Longitude float64 `maxminddb:"longitude"`
		TimeZone  string  `maxminddb:"time_zone"`
	} `maxminddb:"location"`
}

const analyticsKeyName = "tyk-system-analytics"

// RPCPurger will purge analytics data into a Mongo database, requires that the Mongo DB string is specified
// in the Config object
type Purger struct {
	Store storage.Handler
}

// Connect Connects to RPC
func (r *Purger) Connect() {
	if !values.ClientIsConnected() {
		Log.Error("RPC client is not connected, use Connect method 1st")
	}

	// setup RPC func if needed
	if !addedFuncs["Ping"] {
		dispatcher.AddFunc("Ping", func() bool {
			return false
		})
		addedFuncs["Ping"] = true
	}
	if !addedFuncs["PurgeAnalyticsData"] {
		dispatcher.AddFunc("PurgeAnalyticsData", func(data string) error {
			return nil
		})
		addedFuncs["PurgeAnalyticsData"] = true
	}

	Log.Info("RPC Analytics client using singleton")
}

// PurgeLoop starts the loop that will pull data out of the in-memory
// store and into RPC.
func (r Purger) PurgeLoop(ctx context.Context) {
	tick := time.NewTicker(10 * time.Second)
	for {
		select {
		case <-ctx.Done():
			return
		case <-tick.C:
			r.PurgeCache()
		}
	}
}

// PurgeCache will pull the data from the in-memory store and drop it into the specified MongoDB collection
func (r *Purger) PurgeCache() {
	if !values.ClientIsConnected() {
		Log.Error("RPC client is not connected, use Connect method 1st")
	}

	if _, err := FuncClientSingleton("Ping", nil); err != nil {
		Log.WithError(err).Error("Can't purge cache, failed to ping RPC")
		return
	}

	analyticsValues := r.Store.GetAndDeleteSet(analyticsKeyName)
	if len(analyticsValues) == 0 {
		return
	}
	keys := make([]interface{}, len(analyticsValues))

	for i, v := range analyticsValues {
		decoded := AnalyticsRecord{}
		if err := msgpack.Unmarshal([]byte(v.(string)), &decoded); err != nil {
			Log.WithError(err).Error("Couldn't unmarshal analytics data")
		} else {
			Log.WithField("decoded", decoded).Debug("Decoded Record")
			keys[i] = decoded
		}
	}

	data, err := json.Marshal(keys)
	if err != nil {
		Log.WithError(err).Error("Failed to marshal analytics data")
		return
	}

	// Send keys to RPC
	if _, err := FuncClientSingleton("PurgeAnalyticsData", string(data)); err != nil {
		EmitErrorEvent(FuncClientSingletonCall, "PurgeAnalyticsData", err)
		Log.Warn("Failed to call purge, retrying: ", err)
	}
}
<commit_msg>adding multiple keys logic in analytics cache purger for slave gateways (#3487)<commit_after>package rpc

import (
	"context"
	"encoding/json"
	"fmt"
	"time"

	msgpack "gopkg.in/vmihailenco/msgpack.v2"

	"github.com/TykTechnologies/tyk/storage"
)

type AnalyticsRecord struct {
	Method        string
	Path          string
	RawPath       string
	ContentLength int64
	UserAgent     string
	Day           int
	Month         time.Month
	Year          int
	Hour          int
	ResponseCode  int
	APIKey        string
	TimeStamp     time.Time
	APIVersion    string
	APIName       string
	APIID         string
	OrgID         string
	OauthID       string
	RequestTime   int64
	RawRequest    string
	RawResponse   string
	IPAddress     string
	Geo           GeoData
	Tags          []string
	Alias         string
	TrackPath     bool
	ExpireAt      time.Time `bson:"expireAt" json:"expireAt"`
}
type GeoData struct {
	Country struct {
		ISOCode string `maxminddb:"iso_code"`
	} `maxminddb:"country"`

	City struct {
		GeoNameID uint              `maxminddb:"geoname_id"`
		Names     map[string]string `maxminddb:"names"`
	} `maxminddb:"city"`

	Location struct {
		Latitude  float64 `maxminddb:"latitude"`
		Longitude float64 `maxminddb:"longitude"`
		TimeZone  string  `maxminddb:"time_zone"`
	} `maxminddb:"location"`
}

const ANALYTICS_KEYNAME = "tyk-system-analytics"

// RPCPurger will purge analytics data into a Mongo database, requires that the Mongo DB string is specified
// in the Config object
type Purger struct {
	Store storage.Handler
}

// Connect Connects to RPC
func (r *Purger) Connect() {
	if !values.ClientIsConnected() {
		Log.Error("RPC client is not connected, use Connect method 1st")
	}

	// setup RPC func if needed
	if !addedFuncs["Ping"] {
		dispatcher.AddFunc("Ping", func() bool {
			return false
		})
		addedFuncs["Ping"] = true
	}
	if !addedFuncs["PurgeAnalyticsData"] {
		dispatcher.AddFunc("PurgeAnalyticsData", func(data string) error {
			return nil
		})
		addedFuncs["PurgeAnalyticsData"] = true
	}

	Log.Info("RPC Analytics client using singleton")
}

// PurgeLoop starts the loop that will pull data out of the in-memory
// store and into RPC.
func (r Purger) PurgeLoop(ctx context.Context) {
	tick := time.NewTicker(10 * time.Second)
	for {
		select {
		case <-ctx.Done():
			return
		case <-tick.C:
			r.PurgeCache()
		}
	}
}

// PurgeCache will pull the data from the in-memory store and drop it into the specified MongoDB collection
func (r *Purger) PurgeCache() {
	if !values.ClientIsConnected() {
		Log.Error("RPC client is not connected, use Connect method 1st")
	}

	if _, err := FuncClientSingleton("Ping", nil); err != nil {
		Log.WithError(err).Error("Can't purge cache, failed to ping RPC")
		return
	}

	for i := -1; i < 10; i++ {
		var analyticsKeyName string
		if i == -1 {
			//if it's the first iteration, we look for tyk-system-analytics to maintain backwards compatibility or if analytics_config.enable_multiple_analytics_keys is disabled in the gateway
			analyticsKeyName = ANALYTICS_KEYNAME
		} else {
			analyticsKeyName = fmt.Sprintf("%v_%v", ANALYTICS_KEYNAME, i)
		}

		analyticsValues := r.Store.GetAndDeleteSet(analyticsKeyName)
		if len(analyticsValues) == 0 {
			continue
		}
		keys := make([]interface{}, len(analyticsValues))

		for i, v := range analyticsValues {
			decoded := AnalyticsRecord{}
			if err := msgpack.Unmarshal([]byte(v.(string)), &decoded); err != nil {
				Log.WithError(err).Error("Couldn't unmarshal analytics data")
			} else {
				Log.WithField("decoded", decoded).Debug("Decoded Record")
				keys[i] = decoded
			}
		}

		data, err := json.Marshal(keys)
		if err != nil {
			Log.WithError(err).Error("Failed to marshal analytics data")
			return
		}

		// Send keys to RPC
		if _, err := FuncClientSingleton("PurgeAnalyticsData", string(data)); err != nil {
			EmitErrorEvent(FuncClientSingletonCall, "PurgeAnalyticsData", err)
			Log.Warn("Failed to call purge, retrying: ", err)
		}
	}
}
<|endoftext|>