<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"strconv"

	"github.com/julienschmidt/httprouter"
)

func toUint64(value string, dflt uint64) (uint64, error) {
	if value == "" {
		return dflt, nil
	}
	if i, err := strconv.ParseUint(value, 10, 64); err != nil {
		return 0, err
	} else {
		return i, nil
	}
}

func VersionHandler(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	version := Version{
		Version: buildVersion,
		Commit:  buildCommit,
		Date:    buildDate,
		SDK:     buildGoSDK,
	}
	w.Header().Set("Content-type", "application/json")

	data, err := json.Marshal(&version)
	if err != nil {
		version = Version{Meta: Meta{Error: err.Error()}}
		data, _ := json.Marshal(&version)
		w.WriteHeader(500)
		w.Write(data)
		return
	}
	w.Write(data)
}

func TeamsHandler(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	a := make([]Team, 0)
	p := getProjects()
	for _, v := range p {
		a = append(a, Team{Name: v.Team})
	}
	w.Header().Set("Content-type", "application/json")

	teams := Teams{Teams: a}
	data, err := json.Marshal(&teams)
	if err != nil {
		teams := Teams{Meta: Meta{Error: err.Error()}}
		data, _ := json.Marshal(&teams)
		w.WriteHeader(500)
		w.Write(data)
		return
	}
	w.Write(data)
}

func ProjectsHandler(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	team := r.URL.Query().Get("team")
	arr := make([]Project, 0)
	if team != "" {
		for _, v := range getProjects() {
			if team == v.Team {
				arr = append(arr, v)
			}
		}
	} else {
		for _, v := range getProjects() {
			arr = append(arr, v)
		}
	}

	w.Header().Set("Content-type", "application/json")

	p := Projects{Projects: arr}
	data, err := json.Marshal(&p)
	if err != nil {
		p := Projects{Meta: Meta{Error: err.Error()}}
		data, _ := json.Marshal(&p)
		w.WriteHeader(500)
		w.Write(data)
		return
	}
	w.Write(data)
}

func ExecuteBuildHandler(decap Decap) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		team := params.ByName("team")
		library := params.ByName("library")

		if _, present := projectByTeamLibrary(team, library); !present {
			w.WriteHeader(404)
			return
		}

		branches := r.URL.Query()["branch"]
		if len(branches) == 0 {
			// todo add a message
			w.WriteHeader(400)
			return
		}

		event := UserBuildEvent{TeamFld: team, LibraryFld: library, BranchesFld: branches}
		go decap.LaunchBuild(event)
	}
}

func HooksHandler(buildScriptsRepo, buildScriptsBranch string, decap Decap) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		repoManager := params.ByName("repomanager")

		if r.Body == nil {
			Log.Println("Expecting an HTTP entity")
			w.WriteHeader(400)
			return
		}

		data, err := ioutil.ReadAll(r.Body)
		if err != nil {
			Log.Println(err)
			w.WriteHeader(500)
			return
		}
		defer func() {
			r.Body.Close()
		}()

		Log.Printf("%s hook received: %s\n", repoManager, data)

		switch repoManager {
		case "buildscripts":
			p, err := assembleProjects(buildScriptsRepo, buildScriptsBranch)
			if err != nil {
				Log.Println(err)
				w.WriteHeader(500)
			} else {
				setProjects(p)
			}
		case "github":
			eventType := r.Header.Get("X-Github-Event")
			switch eventType {
			case "create":
				event := GithubEvent{}
				if err := json.Unmarshal(data, &event); err != nil {
					Log.Println(err)
					w.WriteHeader(500)
					return
				}
				go decap.LaunchBuild(event)
			case "push":
				event := GithubEvent{}
				if err := json.Unmarshal(data, &event); err != nil {
					Log.Println(err)
					w.WriteHeader(500)
					return
				}
				go decap.LaunchBuild(event)
			default:
				w.WriteHeader(400)
				return
			}
		default:
			Log.Printf("repomanager %s not supported\n", repoManager)
			w.WriteHeader(400)
			return
		}
		w.WriteHeader(200)
	}
}

// todo Cleanup with PreStop might help explain to the world the state of the container immediately before termination.
// See https://godoc.org/github.com/kubernetes/kubernetes/pkg/api/v1#Lifecycle
func StopBuildHandler(decap Decap) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		buildID := params.ByName("id")
		if err := decap.DeletePod(buildID); err != nil {
			Log.Println(err)
		}
	}
}

func ProjectBranchesHandler(repoClients map[string]SCMClient) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		team := params.ByName("team")
		library := params.ByName("library")

		project, present := projectByTeamLibrary(team, library)
		if !present {
			w.WriteHeader(404)
			return
		}

		switch project.Descriptor.RepoManager {
		case "github":
			w.Header().Set("Content-type", "application/json")
			repoClient := repoClients["github"]
			nativeBranches, err := repoClient.GetBranches(project.Team, project.Library)
			if err != nil {
				Log.Print(err)
				data, _ := json.Marshal(&Branches{Meta: Meta{Error: err.Error()}})
				w.WriteHeader(500)
				w.Write(data)
				return
			}

			branches := Branches{Branches: nativeBranches}
			data, err := json.Marshal(&branches)
			if err != nil {
				Log.Print(err)
				data, _ := json.Marshal(&Branches{Meta: Meta{Error: err.Error()}})
				w.WriteHeader(500)
				w.Write(data)
				return
			}

			w.Write(data)
			return
		}
		w.WriteHeader(400)
	}
}

// Return gzipped console log, or console log in plain text if Accept: text/plain is set
func LogHandler(storageService StorageService) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		buildID := params.ByName("id")
		data, err := storageService.GetConsoleLog(buildID)
		if err != nil {
			Log.Println(err)
			w.WriteHeader(500)
			return
		}
		if r.Header.Get("Accept") == "text/plain" {
			inputBuffer := bytes.NewBuffer(data)
			source, err := gzip.NewReader(inputBuffer)
			if err != nil {
				Log.Println(err)
				w.WriteHeader(500)
				return
			}
			defer func() {
				source.Close()
			}()

			var outputBuffer = new(bytes.Buffer)
			io.Copy(outputBuffer, source)
			w.Header().Set("Content-type", "text/plain")
			w.Write(outputBuffer.Bytes())

		} else {
			w.Header().Set("Content-type", "application/x-gzip")
			w.Write(data)
		}
	}
}

// Return artifacts gzipped tarball, or file listing in tarball if Accept: text/plain is set
func ArtifactsHandler(storageService StorageService) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		buildID := params.ByName("id")
		data, err := storageService.GetArtifacts(buildID)
		if err != nil {
			Log.Println(err)
			w.WriteHeader(500)
			return
		}
		if r.Header.Get("Accept") == "text/plain" {
			// unzip
			inputBuffer := bytes.NewBuffer(data)
			source, err := gzip.NewReader(inputBuffer)
			if err != nil {
				Log.Println(err)
				w.WriteHeader(500)
				return
			}
			defer func() {
				source.Close()
			}()

			var outputBuffer = new(bytes.Buffer)
			io.Copy(outputBuffer, source)

			newData := outputBuffer.Bytes()

			// get tar manifest
			plainReader := bytes.NewReader(newData)
			tr := tar.NewReader(plainReader)
			records := ""
			for {
				hdr, err := tr.Next()
				if err == io.EOF {
					break
				}
				if err != nil {
					Log.Println(err)
					w.WriteHeader(500)
					return
				}
				records = fmt.Sprintf("%s\n%s", records, hdr.Name)
			}
			w.Header().Set("Content-type", "text/plain")
			w.Write([]byte(records))
		} else {
			w.Header().Set("Content-type", "application/x-gzip")
			w.Write(data)
		}
	}
}

func BuildsHandler(storageService StorageService) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		project := params.ByName("team")
		library := params.ByName("library")

		since, err := toUint64(r.URL.Query().Get("since"), 0)
		if err != nil {
			builds := Builds{Meta: Meta{Error: err.Error()}}
			data, _ := json.Marshal(&builds)
			w.WriteHeader(400)
			w.Write(data)
			return
		}

		limit, err := toUint64(r.URL.Query().Get("limit"), math.MaxUint64)
		if err != nil {
			builds := Builds{Meta: Meta{Error: err.Error()}}
			data, _ := json.Marshal(&builds)
			w.WriteHeader(400)
			w.Write(data)
			return
		}

		buildList, err := storageService.GetBuildsByProject(Project{Team: project, Library: library}, since, limit)

		if err != nil {
			builds := Builds{Meta: Meta{Error: err.Error()}}
			data, _ := json.Marshal(&builds)
			w.WriteHeader(502)
			w.Write(data)
			return
		}

		builds := Builds{Builds: buildList}
		data, err := json.Marshal(&builds)
		if err != nil {
			builds := Builds{Meta: Meta{Error: err.Error()}}
			builds.Meta.Error = err.Error()
			data, _ := json.Marshal(&builds)
			w.WriteHeader(500)
			w.Write(data)
			return
		}
		w.Write(data)
	}
}
<commit_msg>default limit to 100 when querying for team/library builds<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"

	"github.com/julienschmidt/httprouter"
)

func toUint64(value string, dflt uint64) (uint64, error) {
	if value == "" {
		return dflt, nil
	}
	if i, err := strconv.ParseUint(value, 10, 64); err != nil {
		return 0, err
	} else {
		return i, nil
	}
}

func VersionHandler(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	version := Version{
		Version: buildVersion,
		Commit:  buildCommit,
		Date:    buildDate,
		SDK:     buildGoSDK,
	}
	w.Header().Set("Content-type", "application/json")

	data, err := json.Marshal(&version)
	if err != nil {
		version = Version{Meta: Meta{Error: err.Error()}}
		data, _ := json.Marshal(&version)
		w.WriteHeader(500)
		w.Write(data)
		return
	}
	w.Write(data)
}

func TeamsHandler(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	a := make([]Team, 0)
	p := getProjects()
	for _, v := range p {
		a = append(a, Team{Name: v.Team})
	}
	w.Header().Set("Content-type", "application/json")

	teams := Teams{Teams: a}
	data, err := json.Marshal(&teams)
	if err != nil {
		teams := Teams{Meta: Meta{Error: err.Error()}}
		data, _ := json.Marshal(&teams)
		w.WriteHeader(500)
		w.Write(data)
		return
	}
	w.Write(data)
}

func ProjectsHandler(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	team := r.URL.Query().Get("team")
	arr := make([]Project, 0)
	if team != "" {
		for _, v := range getProjects() {
			if team == v.Team {
				arr = append(arr, v)
			}
		}
	} else {
		for _, v := range getProjects() {
			arr = append(arr, v)
		}
	}

	w.Header().Set("Content-type", "application/json")

	p := Projects{Projects: arr}
	data, err := json.Marshal(&p)
	if err != nil {
		p := Projects{Meta: Meta{Error: err.Error()}}
		data, _ := json.Marshal(&p)
		w.WriteHeader(500)
		w.Write(data)
		return
	}
	w.Write(data)
}

func ExecuteBuildHandler(decap Decap) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		team := params.ByName("team")
		library := params.ByName("library")

		if _, present := projectByTeamLibrary(team, library); !present {
			w.WriteHeader(404)
			return
		}

		branches := r.URL.Query()["branch"]
		if len(branches) == 0 {
			// todo add a message
			w.WriteHeader(400)
			return
		}

		event := UserBuildEvent{TeamFld: team, LibraryFld: library, BranchesFld: branches}
		go decap.LaunchBuild(event)
	}
}

func HooksHandler(buildScriptsRepo, buildScriptsBranch string, decap Decap) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		repoManager := params.ByName("repomanager")

		if r.Body == nil {
			Log.Println("Expecting an HTTP entity")
			w.WriteHeader(400)
			return
		}

		data, err := ioutil.ReadAll(r.Body)
		if err != nil {
			Log.Println(err)
			w.WriteHeader(500)
			return
		}
		defer func() {
			r.Body.Close()
		}()

		Log.Printf("%s hook received: %s\n", repoManager, data)

		switch repoManager {
		case "buildscripts":
			p, err := assembleProjects(buildScriptsRepo, buildScriptsBranch)
			if err != nil {
				Log.Println(err)
				w.WriteHeader(500)
			} else {
				setProjects(p)
			}
		case "github":
			eventType := r.Header.Get("X-Github-Event")
			switch eventType {
			case "create":
				event := GithubEvent{}
				if err := json.Unmarshal(data, &event); err != nil {
					Log.Println(err)
					w.WriteHeader(500)
					return
				}
				go decap.LaunchBuild(event)
			case "push":
				event := GithubEvent{}
				if err := json.Unmarshal(data, &event); err != nil {
					Log.Println(err)
					w.WriteHeader(500)
					return
				}
				go decap.LaunchBuild(event)
			default:
				w.WriteHeader(400)
				return
			}
		default:
			Log.Printf("repomanager %s not supported\n", repoManager)
			w.WriteHeader(400)
			return
		}
		w.WriteHeader(200)
	}
}

// todo Cleanup with PreStop might help explain to the world the state of the container immediately before termination.
// See https://godoc.org/github.com/kubernetes/kubernetes/pkg/api/v1#Lifecycle
func StopBuildHandler(decap Decap) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		buildID := params.ByName("id")
		if err := decap.DeletePod(buildID); err != nil {
			Log.Println(err)
		}
	}
}

func ProjectBranchesHandler(repoClients map[string]SCMClient) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		team := params.ByName("team")
		library := params.ByName("library")

		project, present := projectByTeamLibrary(team, library)
		if !present {
			w.WriteHeader(404)
			return
		}

		switch project.Descriptor.RepoManager {
		case "github":
			w.Header().Set("Content-type", "application/json")
			repoClient := repoClients["github"]
			nativeBranches, err := repoClient.GetBranches(project.Team, project.Library)
			if err != nil {
				Log.Print(err)
				data, _ := json.Marshal(&Branches{Meta: Meta{Error: err.Error()}})
				w.WriteHeader(500)
				w.Write(data)
				return
			}

			branches := Branches{Branches: nativeBranches}
			data, err := json.Marshal(&branches)
			if err != nil {
				Log.Print(err)
				data, _ := json.Marshal(&Branches{Meta: Meta{Error: err.Error()}})
				w.WriteHeader(500)
				w.Write(data)
				return
			}

			w.Write(data)
			return
		}
		w.WriteHeader(400)
	}
}

// Return gzipped console log, or console log in plain text if Accept: text/plain is set
func LogHandler(storageService StorageService) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		buildID := params.ByName("id")
		data, err := storageService.GetConsoleLog(buildID)
		if err != nil {
			Log.Println(err)
			w.WriteHeader(500)
			return
		}
		if r.Header.Get("Accept") == "text/plain" {
			inputBuffer := bytes.NewBuffer(data)
			source, err := gzip.NewReader(inputBuffer)
			if err != nil {
				Log.Println(err)
				w.WriteHeader(500)
				return
			}
			defer func() {
				source.Close()
			}()

			var outputBuffer = new(bytes.Buffer)
			io.Copy(outputBuffer, source)
			w.Header().Set("Content-type", "text/plain")
			w.Write(outputBuffer.Bytes())

		} else {
			w.Header().Set("Content-type", "application/x-gzip")
			w.Write(data)
		}
	}
}

// Return artifacts gzipped tarball, or file listing in tarball if Accept: text/plain is set
func ArtifactsHandler(storageService StorageService) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		buildID := params.ByName("id")
		data, err := storageService.GetArtifacts(buildID)
		if err != nil {
			Log.Println(err)
			w.WriteHeader(500)
			return
		}
		if r.Header.Get("Accept") == "text/plain" {
			// unzip
			inputBuffer := bytes.NewBuffer(data)
			source, err := gzip.NewReader(inputBuffer)
			if err != nil {
				Log.Println(err)
				w.WriteHeader(500)
				return
			}
			defer func() {
				source.Close()
			}()

			var outputBuffer = new(bytes.Buffer)
			io.Copy(outputBuffer, source)

			newData := outputBuffer.Bytes()

			// get tar manifest
			plainReader := bytes.NewReader(newData)
			tr := tar.NewReader(plainReader)
			records := ""
			for {
				hdr, err := tr.Next()
				if err == io.EOF {
					break
				}
				if err != nil {
					Log.Println(err)
					w.WriteHeader(500)
					return
				}
				records = fmt.Sprintf("%s\n%s", records, hdr.Name)
			}
			w.Header().Set("Content-type", "text/plain")
			w.Write([]byte(records))
		} else {
			w.Header().Set("Content-type", "application/x-gzip")
			w.Write(data)
		}
	}
}

func BuildsHandler(storageService StorageService) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		project := params.ByName("team")
		library := params.ByName("library")

		since, err := toUint64(r.URL.Query().Get("since"), 0)
		if err != nil {
			builds := Builds{Meta: Meta{Error: err.Error()}}
			data, _ := json.Marshal(&builds)
			w.WriteHeader(400)
			w.Write(data)
			return
		}

		limit, err := toUint64(r.URL.Query().Get("limit"), 100)
		if err != nil {
			builds := Builds{Meta: Meta{Error: err.Error()}}
			data, _ := json.Marshal(&builds)
			w.WriteHeader(400)
			w.Write(data)
			return
		}

		buildList, err := storageService.GetBuildsByProject(Project{Team: project, Library: library}, since, limit)

		if err != nil {
			builds := Builds{Meta: Meta{Error: err.Error()}}
			data, _ := json.Marshal(&builds)
			w.WriteHeader(502)
			w.Write(data)
			return
		}

		builds := Builds{Builds: buildList}
		data, err := json.Marshal(&builds)
		if err != nil {
			builds := Builds{Meta: Meta{Error: err.Error()}}
			builds.Meta.Error = err.Error()
			data, _ := json.Marshal(&builds)
			w.WriteHeader(500)
			w.Write(data)
			return
		}
		w.Write(data)
	}
}
<|endoftext|>


<commit_before>package main

import (
  "os"
  "fmt"
  "time"
  "strconv"
  "strings"
  "regexp"
  "net/http"
  "crypto/tls"
  "path/filepath"
  "github.com/jawher/mow.cli"
  "github.com/skratchdot/open-golang/open"
  "github.com/mh-cbon/gssc"
)

type LogResponseWriter struct {
  impl http.ResponseWriter
  bytesWritten int
  statusCode int
}

func NewLogResponseWriter(writer http.ResponseWriter) *LogResponseWriter {
  return &LogResponseWriter{
    impl: writer,
    bytesWritten: 0,
    statusCode: 200,
  }
}

func (writer *LogResponseWriter) Header() http.Header {
  return writer.impl.Header()
}

func (writer *LogResponseWriter) Write(bytes []byte) (int, error) {
  writer.bytesWritten += len(bytes)
  return writer.impl.Write(bytes)
}

func (writer *LogResponseWriter) WriteHeader(statusCode int) {
  writer.statusCode = statusCode
  writer.impl.WriteHeader(statusCode)
}

func formatSize(bytes int) string {
  if bytes < 1000 {
    return strconv.Itoa(bytes)
  } else if bytes < 1000000 {
    return fmt.Sprintf("%.2fK", float32(bytes) / 1000)
  } else {
    return fmt.Sprintf("%.2fM", float32(bytes) / 1000000)
  }
}

func LogHandler(h http.Handler, log string) http.Handler {
  formatter, _ := regexp.Compile("%.")

  return http.HandlerFunc(func (w http.ResponseWriter, r *http.Request) {
    logWriter := NewLogResponseWriter(w)
    h.ServeHTTP(logWriter, r)

    ip := r.RemoteAddr
    if i := strings.Index(ip, ":"); i >= 0 {
      ip = ip[:i]
    }

    line := string(formatter.ReplaceAllFunc([]byte(log), func(match []byte) []byte {
      switch string(match[1]) {
      case "i":
        return []byte(ip)
      case "t":
        return []byte(time.Now().Format("2/Jan/2006:15:04:05 -0700"))
      case "m":
        return []byte(r.Method)
      case "u":
        return []byte(r.URL.String())
      case "s":
        return []byte(strconv.Itoa(logWriter.statusCode))
      case "b":
        return []byte(formatSize(logWriter.bytesWritten))
      case "a":
        return []byte(r.Header.Get("User-Agent"))
      case "%":
        return []byte("%")
      default:
        return match
      }
    }))

    fmt.Println(line)
  })
}

type NoCacheResponseWriter struct {
  impl http.ResponseWriter
}

func (writer *NoCacheResponseWriter) Header() http.Header {
  header := writer.impl.Header()
  header.Set("Cache-Control", "private, max-age=0, no-cache")
  header.Del("Last-Modified")
  header.Del("ETag")
  return header
}

func (writer *NoCacheResponseWriter) Write(bytes []byte) (int, error) {
  return writer.impl.Write(bytes)
}

func (writer *NoCacheResponseWriter) WriteHeader(statusCode int) {
  writer.impl.WriteHeader(statusCode)
}

func NoCacheHandler(h http.Handler) http.Handler {
  return http.HandlerFunc(func (w http.ResponseWriter, r *http.Request) {
    r.Header.Set("Cache-Control", "no-cache")
    r.Header.Del("If-Modified-Since")
    r.Header.Del("If-None-Match")
    h.ServeHTTP(&NoCacheResponseWriter{w}, r)
  })
}

func main() {
  app := cli.App("sfs", "Static File Server - https://github.com/schmich/sfs")

  port := app.IntOpt("p port", 8080, "Listening port")
  iface := app.StringOpt("i iface interface", "127.0.0.1", "Listening interface")
  secure := app.BoolOpt("s secure", false, "Enable HTTPS with self-signed TLS certificate")
  allIface := app.BoolOpt("g global", false, "Listen on all interfaces (overrides -i)")
  dir := app.StringOpt("d dir directory", ".", "Directory to serve")
  browser := app.BoolOpt("b browser", false, "Launch web browser")
  log := app.StringOpt("l log", "%i - %m %u %s", "Log format: %i %t %m %u %s %b %a")
  quiet := app.BoolOpt("q quiet", false, "Disable logging")
  cache := app.BoolOpt("c cache", false, "Allow cached responses")

  app.Version("v version", "sfs " + Version)

  app.Action = func () {
    var err error

    *dir, err = filepath.Abs(*dir)
    if err != nil {
      panic(err)
    }

    if *allIface {
      *iface = "0.0.0.0"
    }

    portPart := ":" + strconv.Itoa(*port)
    listen := *iface + portPart

    handler := http.FileServer(http.Dir(*dir))
    if !*cache {
      handler = NoCacheHandler(handler)
    }

    if *quiet {
      *log = ""
    }

    if strings.TrimSpace(*log) != "" {
      handler = LogHandler(handler, *log)
    }

    protocol := "HTTP"
    if *secure {
      protocol = "HTTPS"
    }

    fmt.Printf(">> Serving %s\n", *dir)
    fmt.Printf(">> Listening on %s (%s)\n", listen, protocol)
    fmt.Printf(">> Ctrl+C to stop\n")

    if *browser {
      url := "http://127.0.0.1" + portPart
      open.Start(url)
    }

    server := &http.Server{
      Addr: listen,
      Handler: handler,
    }

    if *secure {
      server.TLSConfig = &tls.Config{
        InsecureSkipVerify: true,
        GetCertificate: gssc.GetCertificate(*iface),
      }
      panic(server.ListenAndServeTLS("", ""))
    } else {
      panic(server.ListenAndServe())
    }
  }

  app.Run(os.Args)
}
<commit_msg>Fix #8: Add digest authentication support.<commit_after>package main

import (
  "os"
  "fmt"
  "time"
  "strconv"
  "strings"
  "regexp"
  "net/http"
  "crypto/tls"
  "path/filepath"
  "github.com/jawher/mow.cli"
  "github.com/skratchdot/open-golang/open"
  "github.com/mh-cbon/gssc"
  "github.com/abbot/go-http-auth"
  "golang.org/x/crypto/ssh/terminal"
)

type LogResponseWriter struct {
  impl http.ResponseWriter
  bytesWritten int
  statusCode int
}

func NewLogResponseWriter(writer http.ResponseWriter) *LogResponseWriter {
  return &LogResponseWriter{
    impl: writer,
    bytesWritten: 0,
    statusCode: 200,
  }
}

func (writer *LogResponseWriter) Header() http.Header {
  return writer.impl.Header()
}

func (writer *LogResponseWriter) Write(bytes []byte) (int, error) {
  writer.bytesWritten += len(bytes)
  return writer.impl.Write(bytes)
}

func (writer *LogResponseWriter) WriteHeader(statusCode int) {
  writer.statusCode = statusCode
  writer.impl.WriteHeader(statusCode)
}

func formatSize(bytes int) string {
  if bytes < 1000 {
    return strconv.Itoa(bytes)
  } else if bytes < 1000000 {
    return fmt.Sprintf("%.2fK", float32(bytes) / 1000)
  } else {
    return fmt.Sprintf("%.2fM", float32(bytes) / 1000000)
  }
}

func LogHandler(h http.Handler, log string) http.Handler {
  formatter, _ := regexp.Compile("%.")

  return http.HandlerFunc(func (w http.ResponseWriter, r *http.Request) {
    logWriter := NewLogResponseWriter(w)
    h.ServeHTTP(logWriter, r)

    ip := r.RemoteAddr
    if i := strings.Index(ip, ":"); i >= 0 {
      ip = ip[:i]
    }

    line := string(formatter.ReplaceAllFunc([]byte(log), func(match []byte) []byte {
      switch string(match[1]) {
      case "i":
        return []byte(ip)
      case "t":
        return []byte(time.Now().Format("2/Jan/2006:15:04:05 -0700"))
      case "m":
        return []byte(r.Method)
      case "u":
        return []byte(r.URL.String())
      case "s":
        return []byte(strconv.Itoa(logWriter.statusCode))
      case "b":
        return []byte(formatSize(logWriter.bytesWritten))
      case "a":
        return []byte(r.Header.Get("User-Agent"))
      case "%":
        return []byte("%")
      default:
        return match
      }
    }))

    fmt.Println(line)
  })
}

type NoCacheResponseWriter struct {
  impl http.ResponseWriter
}

func (writer *NoCacheResponseWriter) Header() http.Header {
  header := writer.impl.Header()
  header.Set("Cache-Control", "private, max-age=0, no-cache")
  header.Del("Last-Modified")
  header.Del("ETag")
  return header
}

func (writer *NoCacheResponseWriter) Write(bytes []byte) (int, error) {
  return writer.impl.Write(bytes)
}

func (writer *NoCacheResponseWriter) WriteHeader(statusCode int) {
  writer.impl.WriteHeader(statusCode)
}

func NoCacheHandler(h http.Handler) http.Handler {
  return http.HandlerFunc(func (w http.ResponseWriter, r *http.Request) {
    r.Header.Set("Cache-Control", "no-cache")
    r.Header.Del("If-Modified-Since")
    r.Header.Del("If-None-Match")
    h.ServeHTTP(&NoCacheResponseWriter{w}, r)
  })
}

func AuthHandler(h http.Handler, realm, username, password string) http.Handler {
  handler := http.HandlerFunc(func (w http.ResponseWriter, r *http.Request) {
    h.ServeHTTP(w, r)
  })

  authenticator := auth.NewDigestAuthenticator(realm, func (user, realm string) string {
    return password
  })

  authenticator.PlainTextSecrets = true

  return authenticator.Wrap(func (w http.ResponseWriter, r *auth.AuthenticatedRequest) {
    if (username != "") && (r.Username != username) {
      authenticator.RequireAuth(w, &r.Request)
    } else {
      handler(w, &r.Request)
    }
  })
}

func readPassword(prompt string) string {
  fmt.Print(prompt)
  password, err := terminal.ReadPassword(int(os.Stdin.Fd()))
  fmt.Println()
  if err != nil {
    panic(err)
  }
  return string(password)
}

func main() {
  app := cli.App("sfs", "Static File Server - https://github.com/schmich/sfs")
  app.Spec = "[-p=<port>] [-i=<interface>] [-s] [-a [USER] PASS] [-g] [-d=<dir>] [-b] [-l=<format>] [-q] [-c]"

  port := app.IntOpt("p port", 8080, "Listening port")
  iface := app.StringOpt("i iface interface", "127.0.0.1", "Listening interface")
  secure := app.BoolOpt("s secure", false, "Enable HTTPS with self-signed TLS certificate")
  auth := app.BoolOpt("a auth", false, "Enable HTTP digest authentication")
  authUser := app.StringArg("USER", "", "Username for digest authentication")
  authPass := app.StringArg("PASS", "", "Password for digest authentication")
  allIface := app.BoolOpt("g global", false, "Listen on all interfaces (overrides -i)")
  dir := app.StringOpt("d dir directory", ".", "Directory to serve")
  browser := app.BoolOpt("b browser", false, "Launch web browser")
  log := app.StringOpt("l log", "%i - %m %u %s", "Log format: %i %t %m %u %s %b %a")
  quiet := app.BoolOpt("q quiet", false, "Disable logging")
  cache := app.BoolOpt("c cache", false, "Allow cached responses")

  app.Version("v version", "sfs " + Version)

  app.Action = func () {
    var err error

    *dir, err = filepath.Abs(*dir)
    if err != nil {
      panic(err)
    }

    if *allIface {
      *iface = "0.0.0.0"
    }

    portPart := ":" + strconv.Itoa(*port)
    listen := *iface + portPart

    handler := http.FileServer(http.Dir(*dir))
    if !*cache {
      handler = NoCacheHandler(handler)
    }

    if *quiet {
      *log = ""
    }

    withAuth := ""
    if *auth {
      withAuth = " with digest authentication"
      if *authPass == "-" {
        *authPass = readPassword("HTTP digest authentication password? ")
      }
      handler = AuthHandler(handler, *iface, *authUser, *authPass)
    }

    if strings.TrimSpace(*log) != "" {
      handler = LogHandler(handler, *log)
    }

    protocol := "http"
    if *secure {
      protocol = "https"
    }

    fmt.Printf(">> Serving %s\n", *dir)
    fmt.Printf(">> Listening on %s://%s%s\n", protocol, listen, withAuth)
    fmt.Printf(">> Ctrl+C to stop\n")

    if *browser {
      url := "http://127.0.0.1" + portPart
      open.Start(url)
    }

    server := &http.Server{
      Addr: listen,
      Handler: handler,
    }

    if *secure {
      server.TLSConfig = &tls.Config{
        InsecureSkipVerify: true,
        GetCertificate: gssc.GetCertificate(*iface),
      }
      panic(server.ListenAndServeTLS("", ""))
    } else {
      panic(server.ListenAndServe())
    }
  }

  app.Run(os.Args)
}
<|endoftext|>


<commit_before><commit_msg>lots of commits for goinstall<commit_after>package sql
<|endoftext|>


<commit_before>// 23 august 2012
package main

import (
	"fmt"
	"strings"
	"os"
	"io/ioutil"
	"github.com/ziutek/mymysql/mysql"
	_ "github.com/ziutek/mymysql/thrsafe"
	"log"
)

const sqlport = "3306"

var db mysql.Conn
var getconsoles, getgames, getwikitext, getredirect, getcatlist mysql.Stmt

func init() {
	passwd_file, err := os.Open("/home/andlabs/src/segaret_scans/.passwd")
	if err != nil {
		log.Fatalf("could not get password")
	}
	passwd, err := ioutil.ReadAll(passwd_file)
	if err != nil {
		log.Fatalf("could not get password")
	}
	passwd_file.Close()

	db = mysql.New("tcp", "", "127.0.0.1:" + sqlport,
		"andlabs", string(passwd), "sonicret_sega")
	err = db.Connect()
	if err != nil {
		log.Fatalf("could not connect to database: %v", err)
	}

	getconsoles, err = db.Prepare(
		`SELECT cat_title, cat_pages
			FROM wiki_category
			ORDER BY cat_title ASC;`)
	if err != nil {
		log.Fatalf("could not prepare console list query: %v", err)
	}

	getgames, err = db.Prepare(
		`SELECT wiki_page.page_title
			FROM wiki_page, wiki_categorylinks
			WHERE wiki_categorylinks.cl_to = ?
				AND wiki_page.page_id = wiki_categorylinks.cl_from
				AND wiki_page.page_namespace = 0
			ORDER BY wiki_page.page_title ASC;`)
	if err != nil {
		log.Fatalf("could not prepare game list query: %v", err)
	}

	getwikitext, err = db.Prepare(
		`SELECT wiki_text.old_text, wiki_page.page_id
			FROM wiki_page, wiki_revision, wiki_text
			WHERE wiki_page.page_namespace = 0
				AND wiki_page.page_title = ?
				AND wiki_page.page_latest = wiki_revision.rev_id
				AND wiki_revision.rev_text_id = wiki_text.old_id;`)
	if err != nil {
		log.Fatalf("could not prepare wikitext query (for scan list): %v", err)
	}

	getredirect, err = db.Prepare(
		`SELECT rd_title
			FROM wiki_redirect
			WHERE rd_from = ?;`)
	if err != nil {
		log.Fatalf("could not prepare redirect query (for scan list): %v", err)
	}

	getcatlist, err = db.Prepare(
		`SELECT wiki_categorylinks.cl_to
			FROM wiki_page, wiki_categorylinks
			WHERE wiki_page.page_namespace = 6
				AND wiki_page.page_title = ?
				AND wiki_categorylinks.cl_from = wiki_page.page_id;`)
	if err != nil {
		log.Fatalf("could not prepare category list query (for checking a scan): %v", err)
	}
}

// TODO see if mediawiki has a better definition
func canonicalize(pageName string) string {
	return strings.Replace(pageName, " ", "_", -1)
}

func sql_getconsoles() ([]string, []int32, error) {
	var consoles []string
	var nMembers []int32

	res, err := getconsoles.Run()
	if err != nil {
		return nil, nil, fmt.Errorf("could not run console list query: %v", err)
	}
	gl, err := res.GetRows()
	if err != nil {
		return nil, nil, fmt.Errorf("could not get console list result rows: %v", err)
	}
	nameField := res.Map("cat_title")
	if nameField < 0 {
		return nil, nil, fmt.Errorf("could not locate console names: %v", err)
	}
	countField := res.Map("cat_pages")
	if countField < 0 {
		return nil, nil, fmt.Errorf("could not locate console game count: %v", err)
	}
	for _, v := range gl {
		c := string(v[nameField].([]byte))
		if strings.HasSuffix(c, "_games") {
			// make human readable and drop _games
			c = strings.Replace(c, "_", " ", -1)
			consoles = append(consoles, c[:len(c) - len(" games")])
			nMembers = append(nMembers, v[countField].(int32))
		}
	}
	return consoles, nMembers, nil
}

func sql_getgames(console string) ([]string, error) {
	var games []string

	category := canonicalize(console) + "_games"
	res, err := getgames.Run(category)
	if err != nil {
		return nil, fmt.Errorf("could not run game list query: %v", err)
	}
	gl, err := res.GetRows()
	if err != nil {
		return nil, fmt.Errorf("could not get game list result rows: %v", err)
	}
	for _, v := range gl {
		games = append(games, string(v[0].([]byte)))
	}
	return games, nil
}

// get wikitext, following all redirects
func sql_getwikitext(page string) (string, error) {
	var wikitext string

	curTitle := canonicalize(page)
	for {
		res, err := getwikitext.Run(curTitle)
		if err != nil {
			return "", fmt.Errorf("could not run wikitext query (for scan list): %v", err)
		}
		wt, err := res.GetRows()
		if err != nil {
			return "", fmt.Errorf("could not get wikitext result rows (for scan list): %v", err)
		}
		textField := res.Map("old_text")
		if textField < 0 {
			return "", fmt.Errorf("could not locate page text (for scan list): %v", err)
		}
		wikitext = string(wt[0][textField].([]byte))
		idField := res.Map("page_id")
		if idField < 0 {
			return "", fmt.Errorf("could not locate page id (for scan list): %v", err)
		}
		id := wt[0][idField].(uint32)
		redir_res, err := getredirect.Run(id)
		if err != nil {
			return "", fmt.Errorf("could not get redirect result rows (for scan list): %v", err)
		}
		rd, err := redir_res.GetRows()
		if err != nil {
			return "", fmt.Errorf("could not get redirect result rows (for scan list): %v", err)
		}
		if len(rd) == 0 {					// no redirect, so finished
			break
		}
		curTitle = string(rd[0][0].([]byte))	// not finished; follow redirect
	}
	return wikitext, nil
}

func sql_getcatlist(file string) ([]string, error) {
	var categories []string

	res, err := getcatlist.Run(canonicalize(file))
	if err != nil {
		return nil, fmt.Errorf("could not run category list query (For checking a scan): %v", err)
	}
	cl, err := res.GetRows()
	if err != nil {
		return nil, fmt.Errorf("could not get category list result rows (for checking a scan): %v", err)
	}
	for _, v := range cl {
		categories = append(categories, string(v[0].([]byte)))
	}
	return categories, nil
}

/*
func main() {
	games, err := sql_getgames("Mega Drive")
	if err != nil {
		fmt.Printf("error grabbing game list: %v\n", err)
	} else {
		fmt.Println(strings.Join(games, "\n"))
	}

	wt, err := sql_getwikitext("Thunder_Force_IV")
	if err != nil {
		fmt.Printf("error grabbing wikitext: %v\n", err)
	} else {
		fmt.Println(wt)
	}

	categories, err := sql_getcatlist("ThunderForce4 MD JP Box.jpg")
	if err != nil {
		fmt.Printf("error grabbing category list: %v\n", err)
	} else {
		fmt.Println(strings.Join(categories, "\n"))
	}
}
*/
<commit_msg>Fixed a bug where off-site redirects resulted in the program hanging.<commit_after>// 23 august 2012
package main

import (
	"fmt"
	"strings"
	"os"
	"io/ioutil"
	"github.com/ziutek/mymysql/mysql"
	_ "github.com/ziutek/mymysql/thrsafe"
	"log"
)

const sqlport = "3306"

var db mysql.Conn
var getconsoles, getgames, getwikitext, getredirect, getcatlist mysql.Stmt

func init() {
	passwd_file, err := os.Open("/home/andlabs/src/segaret_scans/.passwd")
	if err != nil {
		log.Fatalf("could not get password")
	}
	passwd, err := ioutil.ReadAll(passwd_file)
	if err != nil {
		log.Fatalf("could not get password")
	}
	passwd_file.Close()

	db = mysql.New("tcp", "", "127.0.0.1:" + sqlport,
		"andlabs", string(passwd), "sonicret_sega")
	err = db.Connect()
	if err != nil {
		log.Fatalf("could not connect to database: %v", err)
	}

	getconsoles, err = db.Prepare(
		`SELECT cat_title, cat_pages
			FROM wiki_category
			ORDER BY cat_title ASC;`)
	if err != nil {
		log.Fatalf("could not prepare console list query: %v", err)
	}

	getgames, err = db.Prepare(
		`SELECT wiki_page.page_title
			FROM wiki_page, wiki_categorylinks
			WHERE wiki_categorylinks.cl_to = ?
				AND wiki_page.page_id = wiki_categorylinks.cl_from
				AND wiki_page.page_namespace = 0
			ORDER BY wiki_page.page_title ASC;`)
	if err != nil {
		log.Fatalf("could not prepare game list query: %v", err)
	}

	getwikitext, err = db.Prepare(
		`SELECT wiki_text.old_text, wiki_page.page_id
			FROM wiki_page, wiki_revision, wiki_text
			WHERE wiki_page.page_namespace = 0
				AND wiki_page.page_title = ?
				AND wiki_page.page_latest = wiki_revision.rev_id
				AND wiki_revision.rev_text_id = wiki_text.old_id;`)
	if err != nil {
		log.Fatalf("could not prepare wikitext query (for scan list): %v", err)
	}

	getredirect, err = db.Prepare(
		`SELECT rd_title
			FROM wiki_redirect
			WHERE rd_from = ?
				AND rd_interwiki IS NULL;`)	// don't cross sites
	if err != nil {
		log.Fatalf("could not prepare redirect query (for scan list): %v", err)
	}

	getcatlist, err = db.Prepare(
		`SELECT wiki_categorylinks.cl_to
			FROM wiki_page, wiki_categorylinks
			WHERE wiki_page.page_namespace = 6
				AND wiki_page.page_title = ?
				AND wiki_categorylinks.cl_from = wiki_page.page_id;`)
	if err != nil {
		log.Fatalf("could not prepare category list query (for checking a scan): %v", err)
	}
}

// TODO see if mediawiki has a better definition
func canonicalize(pageName string) string {
	return strings.Replace(pageName, " ", "_", -1)
}

func sql_getconsoles() ([]string, []int32, error) {
	var consoles []string
	var nMembers []int32

	res, err := getconsoles.Run()
	if err != nil {
		return nil, nil, fmt.Errorf("could not run console list query: %v", err)
	}
	gl, err := res.GetRows()
	if err != nil {
		return nil, nil, fmt.Errorf("could not get console list result rows: %v", err)
	}
	nameField := res.Map("cat_title")
	if nameField < 0 {
		return nil, nil, fmt.Errorf("could not locate console names: %v", err)
	}
	countField := res.Map("cat_pages")
	if countField < 0 {
		return nil, nil, fmt.Errorf("could not locate console game count: %v", err)
	}
	for _, v := range gl {
		c := string(v[nameField].([]byte))
		if strings.HasSuffix(c, "_games") {
			// make human readable and drop _games
			c = strings.Replace(c, "_", " ", -1)
			consoles = append(consoles, c[:len(c) - len(" games")])
			nMembers = append(nMembers, v[countField].(int32))
		}
	}
	return consoles, nMembers, nil
}

func sql_getgames(console string) ([]string, error) {
	var games []string

	category := canonicalize(console) + "_games"
	res, err := getgames.Run(category)
	if err != nil {
		return nil, fmt.Errorf("could not run game list query: %v", err)
	}
	gl, err := res.GetRows()
	if err != nil {
		return nil, fmt.Errorf("could not get game list result rows: %v", err)
	}
	for _, v := range gl {
		games = append(games, string(v[0].([]byte)))
	}
	return games, nil
}

// get wikitext, following all redirects
func sql_getwikitext(page string) (string, error) {
	var wikitext string

	curTitle := canonicalize(page)
	for {
		res, err := getwikitext.Run(curTitle)
		if err != nil {
			return "", fmt.Errorf("could not run wikitext query (for scan list): %v", err)
		}
		wt, err := res.GetRows()
		if err != nil {
			return "", fmt.Errorf("could not get wikitext result rows (for scan list): %v", err)
		}
		textField := res.Map("old_text")
		if textField < 0 {
			return "", fmt.Errorf("could not locate page text (for scan list): %v", err)
		}
		wikitext = string(wt[0][textField].([]byte))
		idField := res.Map("page_id")
		if idField < 0 {
			return "", fmt.Errorf("could not locate page id (for scan list): %v", err)
		}
		id := wt[0][idField].(uint32)
		redir_res, err := getredirect.Run(id)
		if err != nil {
			return "", fmt.Errorf("could not get redirect result rows (for scan list): %v", err)
		}
		rd, err := redir_res.GetRows()
		if err != nil {
			return "", fmt.Errorf("could not get redirect result rows (for scan list): %v", err)
		}
		if len(rd) == 0 {					// no redirect, so finished
			break
		}
		curTitle = string(rd[0][0].([]byte))	// not finished; follow redirect
	}
	return wikitext, nil
}

func sql_getcatlist(file string) ([]string, error) {
	var categories []string

	res, err := getcatlist.Run(canonicalize(file))
	if err != nil {
		return nil, fmt.Errorf("could not run category list query (For checking a scan): %v", err)
	}
	cl, err := res.GetRows()
	if err != nil {
		return nil, fmt.Errorf("could not get category list result rows (for checking a scan): %v", err)
	}
	for _, v := range cl {
		categories = append(categories, string(v[0].([]byte)))
	}
	return categories, nil
}

/*
func main() {
	games, err := sql_getgames("Mega Drive")
	if err != nil {
		fmt.Printf("error grabbing game list: %v\n", err)
	} else {
		fmt.Println(strings.Join(games, "\n"))
	}

	wt, err := sql_getwikitext("Thunder_Force_IV")
	if err != nil {
		fmt.Printf("error grabbing wikitext: %v\n", err)
	} else {
		fmt.Println(wt)
	}

	categories, err := sql_getcatlist("ThunderForce4 MD JP Box.jpg")
	if err != nil {
		fmt.Printf("error grabbing category list: %v\n", err)
	} else {
		fmt.Println(strings.Join(categories, "\n"))
	}
}
*/
<|endoftext|>


<commit_before>package webservice

import (
        "fmt"
        "encoding/json"
        "log"
        "net/http"
        )

func handleSend(w http.ResponseWriter, r *http.Request) {
    fmt.Println("Handling Send Request")
}

func handleList(w http.ResponseWriter, r *http.Request) {
    w.Header().Set("Content-Type", "application/json")

    messages := [3]string{"Test Message 1", "Test Message 2", "Test Message 3"}

    outgoingJSON, error := json.Marshal(messages)

    if error != nil {
        log.Println(error.Error())
        http.Error(w, error.Error(), http.StatusInternalServerError)
        return
    }

    fmt.Fprint(w, string(outgoingJSON))
}

func Serve() {
    http.HandleFunc("/send", handleSend)
    http.HandleFunc("/list", handleList)
    err := http.ListenAndServe(":9090", nil) // set listen port
    if err != nil {
        log.Fatal("ListenAndServe: ", err)
    }
}

<commit_msg>alphabetize serve.go webservice imports<commit_after>package webservice

import (
        "encoding/json"
        "fmt"
	"log"
        "net/http"
        )

func handleSend(w http.ResponseWriter, r *http.Request) {
    fmt.Println("Handling Send Request")
}

func handleList(w http.ResponseWriter, r *http.Request) {
    w.Header().Set("Content-Type", "application/json")

    messages := [3]string{"Test Message 1", "Test Message 2", "Test Message 3"}

    outgoingJSON, error := json.Marshal(messages)

    if error != nil {
        log.Println(error.Error())
        http.Error(w, error.Error(), http.StatusInternalServerError)
        return
    }

    fmt.Fprint(w, string(outgoingJSON))
}

func Serve() {
    http.HandleFunc("/send", handleSend)
    http.HandleFunc("/list", handleList)
    err := http.ListenAndServe(":9090", nil) // set listen port
    if err != nil {
        log.Fatal("ListenAndServe: ", err)
    }
}

<|endoftext|>


<commit_before>package filesys

import (
	"context"
	"os"
	"path"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"
)

type Dir struct {
	Path       string
	wfs        *WFS
	attributes *filer_pb.FuseAttributes
}

var _ = fs.Node(&Dir{})
var _ = fs.NodeCreater(&Dir{})
var _ = fs.NodeMkdirer(&Dir{})
var _ = fs.NodeRequestLookuper(&Dir{})
var _ = fs.HandleReadDirAller(&Dir{})
var _ = fs.NodeRemover(&Dir{})
var _ = fs.NodeRenamer(&Dir{})
var _ = fs.NodeSetattrer(&Dir{})

func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error {

	glog.V(3).Infof("dir Attr %s", dir.Path)

	// https://github.com/bazil/fuse/issues/196
	attr.Valid = time.Second

	if dir.Path == dir.wfs.option.FilerMountRootPath {
		dir.setRootDirAttributes(attr)
		return nil
	}

	item := dir.wfs.listDirectoryEntriesCache.Get(dir.Path)
	var entry *filer_pb.Entry
	if item != nil && !item.Expired() {
		entry = item.Value().(*filer_pb.Entry)
	}

	if entry != nil {
		entry := item.Value().(*filer_pb.Entry)

		attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
		attr.Ctime = time.Unix(entry.Attributes.Crtime, 0)
		attr.Mode = os.FileMode(entry.Attributes.FileMode)
		attr.Gid = entry.Attributes.Gid
		attr.Uid = entry.Attributes.Uid

		return nil
	}

	glog.V(3).Infof("dir Attr cache miss %s", dir.Path)

	entry, err := filer2.GetEntry(ctx, dir.wfs, dir.Path)
	if err != nil {
		glog.V(2).Infof("read dir %s attr: %v, error: %v", dir.Path, dir.attributes, err)
		return err
	}
	if entry == nil {
		return fuse.ENOENT
	}
	dir.attributes = entry.Attributes

	glog.V(2).Infof("dir %s: %v perm: %v", dir.Path, dir.attributes, os.FileMode(dir.attributes.FileMode))

	attr.Mode = os.FileMode(dir.attributes.FileMode) | os.ModeDir

	attr.Mtime = time.Unix(dir.attributes.Mtime, 0)
	attr.Ctime = time.Unix(dir.attributes.Crtime, 0)
	attr.Gid = dir.attributes.Gid
	attr.Uid = dir.attributes.Uid

	return nil
}

func (dir *Dir) setRootDirAttributes(attr *fuse.Attr) {
	attr.Uid = dir.wfs.option.MountUid
	attr.Gid = dir.wfs.option.MountGid
	attr.Mode = dir.wfs.option.MountMode
	attr.Crtime = dir.wfs.option.MountCtime
	attr.Ctime = dir.wfs.option.MountCtime
	attr.Mtime = dir.wfs.option.MountMtime
	attr.Atime = dir.wfs.option.MountMtime
}

func (dir *Dir) newFile(name string, entry *filer_pb.Entry) *File {
	return &File{
		Name:           name,
		dir:            dir,
		wfs:            dir.wfs,
		entry:          entry,
		entryViewCache: nil,
	}
}

func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
	resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {

	request := &filer_pb.CreateEntryRequest{
		Directory: dir.Path,
		Entry: &filer_pb.Entry{
			Name:        req.Name,
			IsDirectory: req.Mode&os.ModeDir > 0,
			Attributes: &filer_pb.FuseAttributes{
				Mtime:       time.Now().Unix(),
				Crtime:      time.Now().Unix(),
				FileMode:    uint32(req.Mode &^ dir.wfs.option.Umask),
				Uid:         req.Uid,
				Gid:         req.Gid,
				Collection:  dir.wfs.option.Collection,
				Replication: dir.wfs.option.Replication,
				TtlSec:      dir.wfs.option.TtlSec,
			},
		},
	}
	glog.V(1).Infof("create: %v", request)

	if request.Entry.IsDirectory {
		if err := dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
			if _, err := client.CreateEntry(ctx, request); err != nil {
				glog.V(0).Infof("create %s/%s: %v", dir.Path, req.Name, err)
				return fuse.EIO
			}
			return nil
		}); err != nil {
			return nil, nil, err
		}
	}

	file := dir.newFile(req.Name, request.Entry)
	if !request.Entry.IsDirectory {
		file.isOpen = true
	}
	fh := dir.wfs.AcquireHandle(file, req.Uid, req.Gid)
	fh.dirtyMetadata = true
	return file, fh, nil

}

func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {

	err := dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.CreateEntryRequest{
			Directory: dir.Path,
			Entry: &filer_pb.Entry{
				Name:        req.Name,
				IsDirectory: true,
				Attributes: &filer_pb.FuseAttributes{
					Mtime:    time.Now().Unix(),
					Crtime:   time.Now().Unix(),
					FileMode: uint32(req.Mode &^ dir.wfs.option.Umask),
					Uid:      req.Uid,
					Gid:      req.Gid,
				},
			},
		}

		glog.V(1).Infof("mkdir: %v", request)
		if _, err := client.CreateEntry(ctx, request); err != nil {
			glog.V(0).Infof("mkdir %s/%s: %v", dir.Path, req.Name, err)
			return fuse.EIO
		}

		return nil
	})

	if err == nil {
		node := &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs}
		return node, nil
	}

	return nil, err
}

func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (node fs.Node, err error) {

	glog.V(4).Infof("dir Lookup %s: %s", dir.Path, req.Name)

	var entry *filer_pb.Entry
	fullFilePath := path.Join(dir.Path, req.Name)

	item := dir.wfs.listDirectoryEntriesCache.Get(fullFilePath)
	if item != nil && !item.Expired() {
		entry = item.Value().(*filer_pb.Entry)
	}

	if entry == nil {
		glog.V(3).Infof("dir Lookup cache miss %s", fullFilePath)
		entry, err = filer2.GetEntry(ctx, dir.wfs, fullFilePath)
		if err != nil {
			return nil, err
		}
		dir.wfs.listDirectoryEntriesCache.Set(fullFilePath, entry, 5*time.Second)
	} else {
		glog.V(4).Infof("dir Lookup cache hit %s", fullFilePath)
	}

	if entry != nil {
		if entry.IsDirectory {
			node = &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs, attributes: entry.Attributes}
		} else {
			node = dir.newFile(req.Name, entry)
		}

		resp.EntryValid = time.Duration(0)
		resp.Attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
		resp.Attr.Ctime = time.Unix(entry.Attributes.Crtime, 0)
		resp.Attr.Mode = os.FileMode(entry.Attributes.FileMode)
		resp.Attr.Gid = entry.Attributes.Gid
		resp.Attr.Uid = entry.Attributes.Uid

		return node, nil
	}

	return nil, fuse.ENOENT
}

func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {

	glog.V(3).Infof("dir ReadDirAll %s", dir.Path)

	cacheTtl := 5 * time.Minute

	readErr := filer2.ReadDirAllEntries(ctx, dir.wfs, dir.Path, "", func(entry *filer_pb.Entry, isLast bool) {
		if entry.IsDirectory {
			dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_Dir}
			ret = append(ret, dirent)
		} else {
			dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File}
			ret = append(ret, dirent)
		}
		dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, cacheTtl)
	})
	if readErr != nil {
		glog.V(0).Infof("list %s: %v", dir.Path, err)
		return ret, fuse.EIO
	}

	return ret, err
}

func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {

	if !req.Dir {
		return dir.removeOneFile(ctx, req)
	}

	return dir.removeFolder(ctx, req)

}

func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) error {

	entry, err := filer2.GetEntry(ctx, dir.wfs, path.Join(dir.Path, req.Name))
	if err != nil {
		return err
	}

	dir.wfs.deleteFileChunks(ctx, entry.Chunks)

	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.DeleteEntryRequest{
			Directory:    dir.Path,
			Name:         req.Name,
			IsDeleteData: false,
		}

		glog.V(3).Infof("remove file: %v", request)
		_, err := client.DeleteEntry(ctx, request)
		if err != nil {
			glog.V(3).Infof("remove file %s/%s: %v", dir.Path, req.Name, err)
			return fuse.ENOENT
		}

		dir.wfs.listDirectoryEntriesCache.Delete(path.Join(dir.Path, req.Name))

		return nil
	})

}

func (dir *Dir) removeFolder(ctx context.Context, req *fuse.RemoveRequest) error {

	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.DeleteEntryRequest{
			Directory:    dir.Path,
			Name:         req.Name,
			IsDeleteData: true,
		}

		glog.V(3).Infof("remove directory entry: %v", request)
		_, err := client.DeleteEntry(ctx, request)
		if err != nil {
			glog.V(3).Infof("remove %s/%s: %v", dir.Path, req.Name, err)
			return fuse.ENOENT
		}

		dir.wfs.listDirectoryEntriesCache.Delete(path.Join(dir.Path, req.Name))

		return nil
	})

}

func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {

	if dir.attributes == nil {
		return nil
	}

	glog.V(3).Infof("%v dir setattr %+v, fh=%d", dir.Path, req, req.Handle)
	if req.Valid.Mode() {
		dir.attributes.FileMode = uint32(req.Mode)
	}

	if req.Valid.Uid() {
		dir.attributes.Uid = req.Uid
	}

	if req.Valid.Gid() {
		dir.attributes.Gid = req.Gid
	}

	if req.Valid.Mtime() {
		dir.attributes.Mtime = req.Mtime.Unix()
	}

	parentDir, name := filer2.FullPath(dir.Path).DirAndName()
	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.UpdateEntryRequest{
			Directory: parentDir,
			Entry: &filer_pb.Entry{
				Name:       name,
				Attributes: dir.attributes,
			},
		}

		glog.V(1).Infof("set attr directory entry: %v", request)
		_, err := client.UpdateEntry(ctx, request)
		if err != nil {
			glog.V(0).Infof("UpdateEntry %s: %v", dir.Path, err)
			return fuse.EIO
		}

		dir.wfs.listDirectoryEntriesCache.Delete(dir.Path)

		return nil
	})

}
<commit_msg>rm will clean the cache first<commit_after>package filesys

import (
	"context"
	"os"
	"path"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"
)

type Dir struct {
	Path       string
	wfs        *WFS
	attributes *filer_pb.FuseAttributes
}

var _ = fs.Node(&Dir{})
var _ = fs.NodeCreater(&Dir{})
var _ = fs.NodeMkdirer(&Dir{})
var _ = fs.NodeRequestLookuper(&Dir{})
var _ = fs.HandleReadDirAller(&Dir{})
var _ = fs.NodeRemover(&Dir{})
var _ = fs.NodeRenamer(&Dir{})
var _ = fs.NodeSetattrer(&Dir{})

func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error {

	glog.V(3).Infof("dir Attr %s", dir.Path)

	// https://github.com/bazil/fuse/issues/196
	attr.Valid = time.Second

	if dir.Path == dir.wfs.option.FilerMountRootPath {
		dir.setRootDirAttributes(attr)
		return nil
	}

	item := dir.wfs.listDirectoryEntriesCache.Get(dir.Path)
	var entry *filer_pb.Entry
	if item != nil && !item.Expired() {
		entry = item.Value().(*filer_pb.Entry)
	}

	if entry != nil {
		entry := item.Value().(*filer_pb.Entry)

		attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
		attr.Ctime = time.Unix(entry.Attributes.Crtime, 0)
		attr.Mode = os.FileMode(entry.Attributes.FileMode)
		attr.Gid = entry.Attributes.Gid
		attr.Uid = entry.Attributes.Uid

		return nil
	}

	glog.V(3).Infof("dir Attr cache miss %s", dir.Path)

	entry, err := filer2.GetEntry(ctx, dir.wfs, dir.Path)
	if err != nil {
		glog.V(2).Infof("read dir %s attr: %v, error: %v", dir.Path, dir.attributes, err)
		return err
	}
	if entry == nil {
		return fuse.ENOENT
	}
	dir.attributes = entry.Attributes

	glog.V(2).Infof("dir %s: %v perm: %v", dir.Path, dir.attributes, os.FileMode(dir.attributes.FileMode))

	attr.Mode = os.FileMode(dir.attributes.FileMode) | os.ModeDir

	attr.Mtime = time.Unix(dir.attributes.Mtime, 0)
	attr.Ctime = time.Unix(dir.attributes.Crtime, 0)
	attr.Gid = dir.attributes.Gid
	attr.Uid = dir.attributes.Uid

	return nil
}

func (dir *Dir) setRootDirAttributes(attr *fuse.Attr) {
	attr.Uid = dir.wfs.option.MountUid
	attr.Gid = dir.wfs.option.MountGid
	attr.Mode = dir.wfs.option.MountMode
	attr.Crtime = dir.wfs.option.MountCtime
	attr.Ctime = dir.wfs.option.MountCtime
	attr.Mtime = dir.wfs.option.MountMtime
	attr.Atime = dir.wfs.option.MountMtime
}

func (dir *Dir) newFile(name string, entry *filer_pb.Entry) *File {
	return &File{
		Name:           name,
		dir:            dir,
		wfs:            dir.wfs,
		entry:          entry,
		entryViewCache: nil,
	}
}

func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
	resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {

	request := &filer_pb.CreateEntryRequest{
		Directory: dir.Path,
		Entry: &filer_pb.Entry{
			Name:        req.Name,
			IsDirectory: req.Mode&os.ModeDir > 0,
			Attributes: &filer_pb.FuseAttributes{
				Mtime:       time.Now().Unix(),
				Crtime:      time.Now().Unix(),
				FileMode:    uint32(req.Mode &^ dir.wfs.option.Umask),
				Uid:         req.Uid,
				Gid:         req.Gid,
				Collection:  dir.wfs.option.Collection,
				Replication: dir.wfs.option.Replication,
				TtlSec:      dir.wfs.option.TtlSec,
			},
		},
	}
	glog.V(1).Infof("create: %v", request)

	if request.Entry.IsDirectory {
		if err := dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
			if _, err := client.CreateEntry(ctx, request); err != nil {
				glog.V(0).Infof("create %s/%s: %v", dir.Path, req.Name, err)
				return fuse.EIO
			}
			return nil
		}); err != nil {
			return nil, nil, err
		}
	}

	file := dir.newFile(req.Name, request.Entry)
	if !request.Entry.IsDirectory {
		file.isOpen = true
	}
	fh := dir.wfs.AcquireHandle(file, req.Uid, req.Gid)
	fh.dirtyMetadata = true
	return file, fh, nil

}

func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {

	err := dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.CreateEntryRequest{
			Directory: dir.Path,
			Entry: &filer_pb.Entry{
				Name:        req.Name,
				IsDirectory: true,
				Attributes: &filer_pb.FuseAttributes{
					Mtime:    time.Now().Unix(),
					Crtime:   time.Now().Unix(),
					FileMode: uint32(req.Mode &^ dir.wfs.option.Umask),
					Uid:      req.Uid,
					Gid:      req.Gid,
				},
			},
		}

		glog.V(1).Infof("mkdir: %v", request)
		if _, err := client.CreateEntry(ctx, request); err != nil {
			glog.V(0).Infof("mkdir %s/%s: %v", dir.Path, req.Name, err)
			return fuse.EIO
		}

		return nil
	})

	if err == nil {
		node := &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs}
		return node, nil
	}

	return nil, err
}

func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (node fs.Node, err error) {

	glog.V(4).Infof("dir Lookup %s: %s", dir.Path, req.Name)

	var entry *filer_pb.Entry
	fullFilePath := path.Join(dir.Path, req.Name)

	item := dir.wfs.listDirectoryEntriesCache.Get(fullFilePath)
	if item != nil && !item.Expired() {
		entry = item.Value().(*filer_pb.Entry)
	}

	if entry == nil {
		glog.V(3).Infof("dir Lookup cache miss %s", fullFilePath)
		entry, err = filer2.GetEntry(ctx, dir.wfs, fullFilePath)
		if err != nil {
			return nil, err
		}
		dir.wfs.listDirectoryEntriesCache.Set(fullFilePath, entry, 5*time.Minute)
	} else {
		glog.V(4).Infof("dir Lookup cache hit %s", fullFilePath)
	}

	if entry != nil {
		if entry.IsDirectory {
			node = &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs, attributes: entry.Attributes}
		} else {
			node = dir.newFile(req.Name, entry)
		}

		resp.EntryValid = time.Duration(0)
		resp.Attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
		resp.Attr.Ctime = time.Unix(entry.Attributes.Crtime, 0)
		resp.Attr.Mode = os.FileMode(entry.Attributes.FileMode)
		resp.Attr.Gid = entry.Attributes.Gid
		resp.Attr.Uid = entry.Attributes.Uid

		return node, nil
	}

	return nil, fuse.ENOENT
}

func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {

	glog.V(3).Infof("dir ReadDirAll %s", dir.Path)

	cacheTtl := 5 * time.Minute

	readErr := filer2.ReadDirAllEntries(ctx, dir.wfs, dir.Path, "", func(entry *filer_pb.Entry, isLast bool) {
		if entry.IsDirectory {
			dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_Dir}
			ret = append(ret, dirent)
		} else {
			dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File}
			ret = append(ret, dirent)
		}
		dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, cacheTtl)
	})
	if readErr != nil {
		glog.V(0).Infof("list %s: %v", dir.Path, err)
		return ret, fuse.EIO
	}

	return ret, err
}

func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {

	if !req.Dir {
		return dir.removeOneFile(ctx, req)
	}

	return dir.removeFolder(ctx, req)

}

func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) error {

	entry, err := filer2.GetEntry(ctx, dir.wfs, path.Join(dir.Path, req.Name))
	if err != nil {
		return err
	}

	dir.wfs.deleteFileChunks(ctx, entry.Chunks)

	dir.wfs.listDirectoryEntriesCache.Delete(path.Join(dir.Path, req.Name))

	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.DeleteEntryRequest{
			Directory:    dir.Path,
			Name:         req.Name,
			IsDeleteData: false,
		}

		glog.V(3).Infof("remove file: %v", request)
		_, err := client.DeleteEntry(ctx, request)
		if err != nil {
			glog.V(3).Infof("remove file %s/%s: %v", dir.Path, req.Name, err)
			return fuse.ENOENT
		}

		return nil
	})

}

func (dir *Dir) removeFolder(ctx context.Context, req *fuse.RemoveRequest) error {

	dir.wfs.listDirectoryEntriesCache.Delete(path.Join(dir.Path, req.Name))

	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.DeleteEntryRequest{
			Directory:    dir.Path,
			Name:         req.Name,
			IsDeleteData: true,
		}

		glog.V(3).Infof("remove directory entry: %v", request)
		_, err := client.DeleteEntry(ctx, request)
		if err != nil {
			glog.V(3).Infof("remove %s/%s: %v", dir.Path, req.Name, err)
			return fuse.ENOENT
		}

		return nil
	})

}

func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {

	if dir.attributes == nil {
		return nil
	}

	glog.V(3).Infof("%v dir setattr %+v, fh=%d", dir.Path, req, req.Handle)
	if req.Valid.Mode() {
		dir.attributes.FileMode = uint32(req.Mode)
	}

	if req.Valid.Uid() {
		dir.attributes.Uid = req.Uid
	}

	if req.Valid.Gid() {
		dir.attributes.Gid = req.Gid
	}

	if req.Valid.Mtime() {
		dir.attributes.Mtime = req.Mtime.Unix()
	}

	parentDir, name := filer2.FullPath(dir.Path).DirAndName()
	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.UpdateEntryRequest{
			Directory: parentDir,
			Entry: &filer_pb.Entry{
				Name:       name,
				Attributes: dir.attributes,
			},
		}

		glog.V(1).Infof("set attr directory entry: %v", request)
		_, err := client.UpdateEntry(ctx, request)
		if err != nil {
			glog.V(0).Infof("UpdateEntry %s: %v", dir.Path, err)
			return fuse.EIO
		}

		dir.wfs.listDirectoryEntriesCache.Delete(dir.Path)

		return nil
	})

}
<|endoftext|>


<commit_before>package logger

import (
	"io"
	"os"

	"context"

	"fmt"

	"sync"

	"cloud.google.com/go/errorreporting"
	"github.com/mgutz/logxi/v1"
	"github.com/pkg/errors"
)

// UnaLogger wraps a logxi logger
// and delegate to some of it's logging methods
type UnaLogger interface {
	Debug(msg string, args ...interface{})
	Info(msg string, args ...interface{})
	Error(msg string, err error, args ...interface{})
	Fatal(msg string, err error, args ...interface{})
	Underlying() log.Logger
}

type unaLogger struct {
	Logger log.Logger
	name   string
}

// Config contains Name and FileName for the logger
type Config struct {
	Name     string
	FileName string
}

// Keep a list of loggers that we can use in the SetLevel func
var loggers []UnaLogger

var errorClient *errorreporting.Client

// logger for internal use
var lgr UnaLogger

// InitErrorReporting will enable the errors of all calls to Error and Fatal to be sent to Google Error Reporting
// It also enables the
func InitErrorReporting(ctx context.Context, projectID, serviceName, serviceVersion string) error {
	lgr = New("unalogger")
	client, err := errorreporting.NewClient(ctx, projectID,
		errorreporting.Config{
			ServiceName:    serviceName,
			ServiceVersion: serviceVersion})
	if err != nil {
		return err
	}

	errorClient = client

	return nil
}

// ReportPanics should be defered in every new scope where you want to catch pancis and have them pass on to Stackdriver
// Error Reporting
func ReportPanics(ctx context.Context) {
	if errorClient == nil {
		panic("The errorClient was nil, initialize it with InitErrorReporting before deferring this functiond")
	}
	x := recover()
	if x == nil {
		return
	}
	switch e := x.(type) {
	case string:
		err := errorClient.ReportSync(ctx, errorreporting.Entry{Error: errors.New(e)})
		if err != nil {
			lgr.Error("Couldn't do a ReportSync to Stackdriver Error Reporting", err)
		}
	}
	// repanics so the app execution stops
	panic(fmt.Sprintf("Repanicked from logger: %s", x))
}

// CloseClient should be deferred right after calling InitErrorReporting to enure that the client is
// closed down gracefully
func CloseClient() {
	if errorClient == nil {
		panic("The errorClient was nil, initialize it with InitErrorReporting before deferring this functiond")
	}
	errorClient.Close()
}

// Deprecated: The functionality is split into InitErrorReporting, ReportPanics and CloseClient instead
// SetUpErrorReporting creates an ErrorReporting client and returns that client together with a reportPanics function.
// That function should be defered in every new scope where you want to catch pancis and have them pass on to Stackdriver
// Error Reporting
func SetUpErrorReporting(ctx context.Context, projectID, serviceName, serviceVersion string) (client *errorreporting.Client, reportPanics func()) {
	lgr := New("errorreporting")
	errClient := InitErrorReporting(ctx, projectID, serviceName, serviceVersion)
	if errClient != nil {
		lgr.Fatal("Couldn't create an errorreporting client", errClient)
	}
	return errorClient, func() {
		x := recover()
		if x == nil {
			return
		}
		switch e := x.(type) {
		case string:
			err := errorClient.ReportSync(ctx, errorreporting.Entry{Error: errors.New(e)})
			if err != nil {
				lgr.Error("Couldn't do a ReportSync to Stackdriver Error Reporting", err)
			}
		}
		// repanics so the app execution stops
		panic(fmt.Sprintf("Repanicked from logger: %v", x))
	}
}

func reportError(err error) {
	if errorClient != nil {
		errorClient.Report(errorreporting.Entry{
			Error: err,
		})
	}
}

var defaultsSet bool
var mutex = sync.Mutex{}

func setDefaults() {
	// These configurations are made to make the
	// log payload compatible with the LogEntry format used in Google Cloud Logging
	// https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry
	log.KeyMap.Level = "severity"
	log.KeyMap.Message = "message"
	log.KeyMap.Time = "timestamp"
	log.LevelMap[log.LevelFatal] = "CRITICAL"
	log.LevelMap[log.LevelError] = "ERROR"
	log.LevelMap[log.LevelInfo] = "INFO"
	log.LevelMap[log.LevelDebug] = "DEBUG"
	defaultsSet = true
}

// New creates a new logger with the given (string) name
func New(name string) UnaLogger {
	return NewLogger(Config{Name: name})
}

// NewLogger creates a new logger with the given (Config) name
func NewLogger(conf Config) UnaLogger {

	logxiLogger := log.New(conf.Name)
	if conf.FileName != "" {
		if file, err := os.Create(conf.FileName); err == nil {
			logxiLogger = log.NewLogger(file, conf.Name)
		}
	}
	unaLogger := &unaLogger{
		Logger: logxiLogger,
		name:   conf.Name,
	}

	// Add the logger to the list of loggers and set some defaults
	// Needs to use a mutex here so loggers can be created in different goroutines
	mutex.Lock()
	loggers = append(loggers, unaLogger)
	if !defaultsSet {
		setDefaults()
	}
	mutex.Unlock()

	return unaLogger
}

// SetWriter overrides the io.Writer of the underlying logxi logger
func (ul *unaLogger) SetWriter(writer io.Writer) {
	ul.Logger = log.NewLogger(writer, ul.name)
}

// Underlying returns the underlying logxi logger
func (ul unaLogger) Underlying() log.Logger {
	return ul.Logger
}

// Info logs to Stdout with an "INFO" prefix
func (ul unaLogger) Info(msg string, args ...interface{}) {
	ul.Logger.Info(msg, args...)
}

// Debug logs to Stdout with an "DEBUG" prefix if Debug level is enabled
func (ul unaLogger) Debug(msg string, args ...interface{}) {
	if ul.Logger.IsDebug() {
		ul.Logger.Debug(msg, args...)
	}
}

// Error logs to Stdout with an "Error" prefix
// It also adds an "error" key to the provided err(error) argument
func (ul unaLogger) Error(msg string, err error, args ...interface{}) {
	reportError(err)
	_ = ul.Logger.Error(msg, append(args, "error", err)...)
}

// Fatal logs to Stdout with an "Fatal" prefix
// It also adds an "error" key to the provided err(error) argument
func (ul unaLogger) Fatal(msg string, err error, args ...interface{}) {
	reportError(err)
	ul.Logger.Fatal(msg, append(args, "error", err)...)
}
<commit_msg>Ignoring the error when closing the client<commit_after>package logger

import (
	"io"
	"os"

	"context"

	"fmt"

	"sync"

	"cloud.google.com/go/errorreporting"
	"github.com/mgutz/logxi/v1"
	"github.com/pkg/errors"
)

// UnaLogger wraps a logxi logger
// and delegate to some of it's logging methods
type UnaLogger interface {
	Debug(msg string, args ...interface{})
	Info(msg string, args ...interface{})
	Error(msg string, err error, args ...interface{})
	Fatal(msg string, err error, args ...interface{})
	Underlying() log.Logger
}

type unaLogger struct {
	Logger log.Logger
	name   string
}

// Config contains Name and FileName for the logger
type Config struct {
	Name     string
	FileName string
}

// Keep a list of loggers that we can use in the SetLevel func
var loggers []UnaLogger

var errorClient *errorreporting.Client

// logger for internal use
var lgr UnaLogger

// InitErrorReporting will enable the errors of all calls to Error and Fatal to be sent to Google Error Reporting
// It also enables the
func InitErrorReporting(ctx context.Context, projectID, serviceName, serviceVersion string) error {
	lgr = New("unalogger")
	client, err := errorreporting.NewClient(ctx, projectID,
		errorreporting.Config{
			ServiceName:    serviceName,
			ServiceVersion: serviceVersion})
	if err != nil {
		return err
	}

	errorClient = client

	return nil
}

// ReportPanics should be defered in every new scope where you want to catch pancis and have them pass on to Stackdriver
// Error Reporting
func ReportPanics(ctx context.Context) {
	if errorClient == nil {
		panic("The errorClient was nil, initialize it with InitErrorReporting before deferring this functiond")
	}
	x := recover()
	if x == nil {
		return
	}
	switch e := x.(type) {
	case string:
		err := errorClient.ReportSync(ctx, errorreporting.Entry{Error: errors.New(e)})
		if err != nil {
			lgr.Error("Couldn't do a ReportSync to Stackdriver Error Reporting", err)
		}
	}
	// repanics so the app execution stops
	panic(fmt.Sprintf("Repanicked from logger: %s", x))
}

// CloseClient should be deferred right after calling InitErrorReporting to enure that the client is
// closed down gracefully
func CloseClient() {
	if errorClient == nil {
		panic("The errorClient was nil, initialize it with InitErrorReporting before deferring this functiond")
	}
	var _ = errorClient.Close() // Ignoring this error
}

// Deprecated: The functionality is split into InitErrorReporting, ReportPanics and CloseClient instead
// SetUpErrorReporting creates an ErrorReporting client and returns that client together with a reportPanics function.
// That function should be defered in every new scope where you want to catch pancis and have them pass on to Stackdriver
// Error Reporting
func SetUpErrorReporting(ctx context.Context, projectID, serviceName, serviceVersion string) (client *errorreporting.Client, reportPanics func()) {
	lgr := New("errorreporting")
	errClient := InitErrorReporting(ctx, projectID, serviceName, serviceVersion)
	if errClient != nil {
		lgr.Fatal("Couldn't create an errorreporting client", errClient)
	}
	return errorClient, func() {
		x := recover()
		if x == nil {
			return
		}
		switch e := x.(type) {
		case string:
			err := errorClient.ReportSync(ctx, errorreporting.Entry{Error: errors.New(e)})
			if err != nil {
				lgr.Error("Couldn't do a ReportSync to Stackdriver Error Reporting", err)
			}
		}
		// repanics so the app execution stops
		panic(fmt.Sprintf("Repanicked from logger: %v", x))
	}
}

func reportError(err error) {
	if errorClient != nil {
		errorClient.Report(errorreporting.Entry{
			Error: err,
		})
	}
}

var defaultsSet bool
var mutex = sync.Mutex{}

func setDefaults() {
	// These configurations are made to make the
	// log payload compatible with the LogEntry format used in Google Cloud Logging
	// https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry
	log.KeyMap.Level = "severity"
	log.KeyMap.Message = "message"
	log.KeyMap.Time = "timestamp"
	log.LevelMap[log.LevelFatal] = "CRITICAL"
	log.LevelMap[log.LevelError] = "ERROR"
	log.LevelMap[log.LevelInfo] = "INFO"
	log.LevelMap[log.LevelDebug] = "DEBUG"
	defaultsSet = true
}

// New creates a new logger with the given (string) name
func New(name string) UnaLogger {
	return NewLogger(Config{Name: name})
}

// NewLogger creates a new logger with the given (Config) name
func NewLogger(conf Config) UnaLogger {

	logxiLogger := log.New(conf.Name)
	if conf.FileName != "" {
		if file, err := os.Create(conf.FileName); err == nil {
			logxiLogger = log.NewLogger(file, conf.Name)
		}
	}
	unaLogger := &unaLogger{
		Logger: logxiLogger,
		name:   conf.Name,
	}

	// Add the logger to the list of loggers and set some defaults
	// Needs to use a mutex here so loggers can be created in different goroutines
	mutex.Lock()
	loggers = append(loggers, unaLogger)
	if !defaultsSet {
		setDefaults()
	}
	mutex.Unlock()

	return unaLogger
}

// SetWriter overrides the io.Writer of the underlying logxi logger
func (ul *unaLogger) SetWriter(writer io.Writer) {
	ul.Logger = log.NewLogger(writer, ul.name)
}

// Underlying returns the underlying logxi logger
func (ul unaLogger) Underlying() log.Logger {
	return ul.Logger
}

// Info logs to Stdout with an "INFO" prefix
func (ul unaLogger) Info(msg string, args ...interface{}) {
	ul.Logger.Info(msg, args...)
}

// Debug logs to Stdout with an "DEBUG" prefix if Debug level is enabled
func (ul unaLogger) Debug(msg string, args ...interface{}) {
	if ul.Logger.IsDebug() {
		ul.Logger.Debug(msg, args...)
	}
}

// Error logs to Stdout with an "Error" prefix
// It also adds an "error" key to the provided err(error) argument
func (ul unaLogger) Error(msg string, err error, args ...interface{}) {
	reportError(err)
	_ = ul.Logger.Error(msg, append(args, "error", err)...)
}

// Fatal logs to Stdout with an "Fatal" prefix
// It also adds an "error" key to the provided err(error) argument
func (ul unaLogger) Fatal(msg string, err error, args ...interface{}) {
	reportError(err)
	ul.Logger.Fatal(msg, append(args, "error", err)...)
}
<|endoftext|>


<commit_before><commit_msg>Use faster zlib implementation.<commit_after><|endoftext|>


<commit_before>// Copyright 2012 The Oneslang Authors.  All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package log

import (
	"fmt"
	"io"
	"log"
)

type Logger struct {
	priority int
	prefix   string
	logger   *log.Logger
}

// New creates a new Logger.
func New(out io.Writer, prefix string, flag int, priority int) *Logger {
	return &Logger{
		priority: priority,
		prefix: prefix,
		logger: log.New(out, prefix, flag),
	}
}

// Sets the output prefix for the logger.
func (me *Logger) SetPrefix(prefix string) {
	me.prefix = prefix
	me.logger.SetPrefix(prefix)
}

func (me *Logger) setFullPrefix(priority int) {
	if me.logger.Flags()&Lpriority != 0 {
		me.logger.SetPrefix(fmt.Sprintf("%s ", priorityName[priority]) + me.prefix)
	}
}

// Calls Output to print to the logger.
func (me *Logger) print(priority int, v ...interface{}) {
	if priority <= me.priority {
		me.setFullPrefix(priority)
		me.logger.Print(v...)
	}
}

// Calls Output to printf to the logger.
func (me *Logger) printf(priority int, format string, v ...interface{}) {
	if priority <= me.priority {
		me.setFullPrefix(priority)
		me.logger.Printf(format, v...)
	}
}

// Calls Output to println to the logger.
func (me *Logger) println(priority int, v ...interface{}) {
	if priority <= me.priority {
		me.setFullPrefix(priority)
		me.logger.Println(v...)
	}
}

// Priority returns the output priority for the logger.
func (me *Logger) Priority() int {
	return me.priority
}

// SetPriority sets the output priority for the logger.
func (me *Logger) SetPriority(priority int) {
	me.priority = priority
}

// Flags returns the output layouts for the logger.
func (me *Logger) Flags() int {
	return me.logger.Flags()
}

// SetFlags sets the output layouts for the logger.
func (me *Logger) SetFlags(layouts int) {
	me.logger.SetFlags(layouts)
}

// Calls Output to print to the logger with the Fatal level.
func (me *Logger) Fatal(v ...interface{}) {
	me.setFullPrefix(Pfatal)
	me.logger.Fatal(v...)
}

// Calls Output to printf to the logger with the Fatal level.
func (me *Logger) Fatalf(format string, v ...interface{}) {
	me.setFullPrefix(Pfatal)
	me.logger.Fatalf(format, v...)
}

// Calls Output to println to the logger with the Fatal level.
func (me *Logger) Fatalln(v ...interface{}) {
	me.setFullPrefix(Pfatal)
	me.logger.Fatalln(v...)
}

// Calls Output to print to the logger with the Error level.
func (me *Logger) Error(v ...interface{}) {
	me.print(Perror, v...)
}

// Calls Output to printf to the logger with the Error level.
func (me *Logger) Errorf(format string, v ...interface{}) {
	me.printf(Perror, format, v...)
}

// Calls Output to println to the logger with the Error level.
func (me *Logger) Errorln(v ...interface{}) {
	me.println(Perror, v...)
}

// Calls Output to print to the logger with the Warn level.
func (me *Logger) Warn(v ...interface{}) {
	me.print(Pwarn, v...)
}

// Calls Output to printf to the logger with the Warn level.
func (me *Logger) Warnf(format string, v ...interface{}) {
	me.printf(Pwarn, format, v...)
}

// Calls Output to println to the logger with the Warn level.
func (me *Logger) Warnln(v ...interface{}) {
	me.println(Pwarn, v...)
}

// Calls Output to print to the logger with the Info level.
func (me *Logger) Info(v ...interface{}) {
	me.print(Pinfo, v...)
}

// Calls Output to printf to the logger with the Info level.
func (me *Logger) Infof(format string, v ...interface{}) {
	me.printf(Pinfo, format, v...)
}

// Calls Output to println to the logger with the Info level.
func (me *Logger) Infoln(v ...interface{}) {
	me.println(Pinfo, v...)
}

// Calls Output to print to the logger with the Debug level.
func (me *Logger) Debug(v ...interface{}) {
	me.print(Pdebug, v...)
}

// Calls Output to printf to the logger with the Debug level.
func (me *Logger) Debugf(format string, v ...interface{}) {
	me.printf(Pdebug, format, v...)
}

// Calls Output to println to the logger with the Debug level.
func (me *Logger) Debugln(v ...interface{}) {
	me.println(Pdebug, v...)
}

// Calls Output to print to the logger with the Trace level.
func (me *Logger) Trace(v ...interface{}) {
	me.print(Ptrace, v...)
}

// Calls Output to printf to the logger with the Trace level.
func (me *Logger) Tracef(format string, v ...interface{}) {
	me.printf(Ptrace, format, v...)
}

// Calls Output to println to the logger with the Trace level.
func (me *Logger) Traceln(v ...interface{}) {
	me.println(Ptrace, v...)
}
<commit_msg>add Prefix() function<commit_after>// Copyright 2012 The Oneslang Authors.  All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package log

import (
	"fmt"
	"io"
	"log"
)

type Logger struct {
	priority int
	prefix   string
	logger   *log.Logger
}

// New creates a new Logger.
func New(out io.Writer, prefix string, flag int, priority int) *Logger {
	return &Logger{
		priority: priority,
		prefix: prefix,
		logger: log.New(out, prefix, flag),
	}
}

// Sets the output prefix for the logger.
func (me *Logger) SetPrefix(prefix string) {
	me.prefix = prefix
	me.logger.SetPrefix(prefix)
}

func (me *Logger) Prefix() string {
	return me.prefix
}

func (me *Logger) setFullPrefix(priority int) {
	if me.logger.Flags()&Lpriority != 0 {
		me.logger.SetPrefix(fmt.Sprintf("%s ", priorityName[priority]) + me.prefix)
	}
}

// Calls Output to print to the logger.
func (me *Logger) print(priority int, v ...interface{}) {
	if priority <= me.priority {
		me.setFullPrefix(priority)
		me.logger.Print(v...)
	}
}

// Calls Output to printf to the logger.
func (me *Logger) printf(priority int, format string, v ...interface{}) {
	if priority <= me.priority {
		me.setFullPrefix(priority)
		me.logger.Printf(format, v...)
	}
}

// Calls Output to println to the logger.
func (me *Logger) println(priority int, v ...interface{}) {
	if priority <= me.priority {
		me.setFullPrefix(priority)
		me.logger.Println(v...)
	}
}

// Priority returns the output priority for the logger.
func (me *Logger) Priority() int {
	return me.priority
}

// SetPriority sets the output priority for the logger.
func (me *Logger) SetPriority(priority int) {
	me.priority = priority
}

// Flags returns the output layouts for the logger.
func (me *Logger) Flags() int {
	return me.logger.Flags()
}

// SetFlags sets the output layouts for the logger.
func (me *Logger) SetFlags(layouts int) {
	me.logger.SetFlags(layouts)
}

// Calls Output to print to the logger with the Fatal level.
func (me *Logger) Fatal(v ...interface{}) {
	me.setFullPrefix(Pfatal)
	me.logger.Fatal(v...)
}

// Calls Output to printf to the logger with the Fatal level.
func (me *Logger) Fatalf(format string, v ...interface{}) {
	me.setFullPrefix(Pfatal)
	me.logger.Fatalf(format, v...)
}

// Calls Output to println to the logger with the Fatal level.
func (me *Logger) Fatalln(v ...interface{}) {
	me.setFullPrefix(Pfatal)
	me.logger.Fatalln(v...)
}

// Calls Output to print to the logger with the Error level.
func (me *Logger) Error(v ...interface{}) {
	me.print(Perror, v...)
}

// Calls Output to printf to the logger with the Error level.
func (me *Logger) Errorf(format string, v ...interface{}) {
	me.printf(Perror, format, v...)
}

// Calls Output to println to the logger with the Error level.
func (me *Logger) Errorln(v ...interface{}) {
	me.println(Perror, v...)
}

// Calls Output to print to the logger with the Warn level.
func (me *Logger) Warn(v ...interface{}) {
	me.print(Pwarn, v...)
}

// Calls Output to printf to the logger with the Warn level.
func (me *Logger) Warnf(format string, v ...interface{}) {
	me.printf(Pwarn, format, v...)
}

// Calls Output to println to the logger with the Warn level.
func (me *Logger) Warnln(v ...interface{}) {
	me.println(Pwarn, v...)
}

// Calls Output to print to the logger with the Info level.
func (me *Logger) Info(v ...interface{}) {
	me.print(Pinfo, v...)
}

// Calls Output to printf to the logger with the Info level.
func (me *Logger) Infof(format string, v ...interface{}) {
	me.printf(Pinfo, format, v...)
}

// Calls Output to println to the logger with the Info level.
func (me *Logger) Infoln(v ...interface{}) {
	me.println(Pinfo, v...)
}

// Calls Output to print to the logger with the Debug level.
func (me *Logger) Debug(v ...interface{}) {
	me.print(Pdebug, v...)
}

// Calls Output to printf to the logger with the Debug level.
func (me *Logger) Debugf(format string, v ...interface{}) {
	me.printf(Pdebug, format, v...)
}

// Calls Output to println to the logger with the Debug level.
func (me *Logger) Debugln(v ...interface{}) {
	me.println(Pdebug, v...)
}

// Calls Output to print to the logger with the Trace level.
func (me *Logger) Trace(v ...interface{}) {
	me.print(Ptrace, v...)
}

// Calls Output to printf to the logger with the Trace level.
func (me *Logger) Tracef(format string, v ...interface{}) {
	me.printf(Ptrace, format, v...)
}

// Calls Output to println to the logger with the Trace level.
func (me *Logger) Traceln(v ...interface{}) {
	me.println(Ptrace, v...)
}
<|endoftext|>


<commit_before>package logrus

import (
	"io"
	"os"
	"sync"
)

type Logger struct {
	// The logs are `io.Copy`'d to this in a mutex. It's common to set this to a
	// file, or leave it default which is `os.Stderr`. You can also set this to
	// something more adventorous, such as logging to Kafka.
	Out io.Writer
	// Hooks for the logger instance. These allow firing events based on logging
	// levels and log entries. For example, to send errors to an error tracking
	// service, log to StatsD or dump the core on fatal errors.
	Hooks LevelHooks
	// All log entries pass through the formatter before logged to Out. The
	// included formatters are `TextFormatter` and `JSONFormatter` for which
	// TextFormatter is the default. In development (when a TTY is attached) it
	// logs with colors, but to a file it wouldn't. You can easily implement your
	// own that implements the `Formatter` interface, see the `README` or included
	// formatters for examples.
	Formatter Formatter
	// The logging level the logger should log at. This is typically (and defaults
	// to) `logrus.Info`, which allows Info(), Warn(), Error() and Fatal() to be
	// logged. `logrus.Debug` is useful in
	Level Level
	// Used to sync writing to the log.
	mu sync.Mutex
}

// Creates a new logger. Configuration should be set by changing `Formatter`,
// `Out` and `Hooks` directly on the default logger instance. You can also just
// instantiate your own:
//
//    var log = &Logger{
//      Out: os.Stderr,
//      Formatter: new(JSONFormatter),
//      Hooks: make(LevelHooks),
//      Level: logrus.DebugLevel,
//    }
//
// It's recommended to make this a global instance called `log`.
func New() *Logger {
	return &Logger{
		Out:       os.Stderr,
		Formatter: new(TextFormatter),
		Hooks:     make(LevelHooks),
		Level:     InfoLevel,
	}
}

// Adds a field to the log entry, note that you it doesn't log until you call
// Debug, Print, Info, Warn, Fatal or Panic. It only creates a log entry.
// Ff you want multiple fields, use `WithFields`.
func (logger *Logger) WithField(key string, value interface{}) *Entry {
	return NewEntry(logger).WithField(key, value)
}

// Adds a struct of fields to the log entry. All it does is call `WithField` for
// each `Field`.
func (logger *Logger) WithFields(fields Fields) *Entry {
	return NewEntry(logger).WithFields(fields)
}

func (logger *Logger) Debugf(format string, args ...interface{}) {
	if logger.Level >= DebugLevel {
		NewEntry(logger).Debugf(format, args...)
	}
}

func (logger *Logger) Infof(format string, args ...interface{}) {
	if logger.Level >= InfoLevel {
		NewEntry(logger).Infof(format, args...)
	}
}

func (logger *Logger) Printf(format string, args ...interface{}) {
	NewEntry(logger).Printf(format, args...)
}

func (logger *Logger) Warnf(format string, args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warnf(format, args...)
	}
}

func (logger *Logger) Warningf(format string, args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warnf(format, args...)
	}
}

func (logger *Logger) Errorf(format string, args ...interface{}) {
	if logger.Level >= ErrorLevel {
		NewEntry(logger).Errorf(format, args...)
	}
}

func (logger *Logger) Fatalf(format string, args ...interface{}) {
	if logger.Level >= FatalLevel {
		NewEntry(logger).Fatalf(format, args...)
	}
	os.Exit(1)
}

func (logger *Logger) Panicf(format string, args ...interface{}) {
	if logger.Level >= PanicLevel {
		NewEntry(logger).Panicf(format, args...)
	}
}

func (logger *Logger) Debug(args ...interface{}) {
	if logger.Level >= DebugLevel {
		NewEntry(logger).Debug(args...)
	}
}

func (logger *Logger) Info(args ...interface{}) {
	if logger.Level >= InfoLevel {
		NewEntry(logger).Info(args...)
	}
}

func (logger *Logger) Print(args ...interface{}) {
	NewEntry(logger).Info(args...)
}

func (logger *Logger) Warn(args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warn(args...)
	}
}

func (logger *Logger) Warning(args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warn(args...)
	}
}

func (logger *Logger) Error(args ...interface{}) {
	if logger.Level >= ErrorLevel {
		NewEntry(logger).Error(args...)
	}
}

func (logger *Logger) Fatal(args ...interface{}) {
	if logger.Level >= FatalLevel {
		NewEntry(logger).Fatal(args...)
	}
	os.Exit(1)
}

func (logger *Logger) Panic(args ...interface{}) {
	if logger.Level >= PanicLevel {
		NewEntry(logger).Panic(args...)
	}
}

func (logger *Logger) Debugln(args ...interface{}) {
	if logger.Level >= DebugLevel {
		NewEntry(logger).Debugln(args...)
	}
}

func (logger *Logger) Infoln(args ...interface{}) {
	if logger.Level >= InfoLevel {
		NewEntry(logger).Infoln(args...)
	}
}

func (logger *Logger) Println(args ...interface{}) {
	NewEntry(logger).Println(args...)
}

func (logger *Logger) Warnln(args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warnln(args...)
	}
}

func (logger *Logger) Warningln(args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warnln(args...)
	}
}

func (logger *Logger) Errorln(args ...interface{}) {
	if logger.Level >= ErrorLevel {
		NewEntry(logger).Errorln(args...)
	}
}

func (logger *Logger) Fatalln(args ...interface{}) {
	if logger.Level >= FatalLevel {
		NewEntry(logger).Fatalln(args...)
	}
	os.Exit(1)
}

func (logger *Logger) Panicln(args ...interface{}) {
	if logger.Level >= PanicLevel {
		NewEntry(logger).Panicln(args...)
	}
}
<commit_msg>Fixes typo in WithField documentation<commit_after>package logrus

import (
	"io"
	"os"
	"sync"
)

type Logger struct {
	// The logs are `io.Copy`'d to this in a mutex. It's common to set this to a
	// file, or leave it default which is `os.Stderr`. You can also set this to
	// something more adventorous, such as logging to Kafka.
	Out io.Writer
	// Hooks for the logger instance. These allow firing events based on logging
	// levels and log entries. For example, to send errors to an error tracking
	// service, log to StatsD or dump the core on fatal errors.
	Hooks LevelHooks
	// All log entries pass through the formatter before logged to Out. The
	// included formatters are `TextFormatter` and `JSONFormatter` for which
	// TextFormatter is the default. In development (when a TTY is attached) it
	// logs with colors, but to a file it wouldn't. You can easily implement your
	// own that implements the `Formatter` interface, see the `README` or included
	// formatters for examples.
	Formatter Formatter
	// The logging level the logger should log at. This is typically (and defaults
	// to) `logrus.Info`, which allows Info(), Warn(), Error() and Fatal() to be
	// logged. `logrus.Debug` is useful in
	Level Level
	// Used to sync writing to the log.
	mu sync.Mutex
}

// Creates a new logger. Configuration should be set by changing `Formatter`,
// `Out` and `Hooks` directly on the default logger instance. You can also just
// instantiate your own:
//
//    var log = &Logger{
//      Out: os.Stderr,
//      Formatter: new(JSONFormatter),
//      Hooks: make(LevelHooks),
//      Level: logrus.DebugLevel,
//    }
//
// It's recommended to make this a global instance called `log`.
func New() *Logger {
	return &Logger{
		Out:       os.Stderr,
		Formatter: new(TextFormatter),
		Hooks:     make(LevelHooks),
		Level:     InfoLevel,
	}
}

// Adds a field to the log entry, note that you it doesn't log until you call
// Debug, Print, Info, Warn, Fatal or Panic. It only creates a log entry.
// If you want multiple fields, use `WithFields`.
func (logger *Logger) WithField(key string, value interface{}) *Entry {
	return NewEntry(logger).WithField(key, value)
}

// Adds a struct of fields to the log entry. All it does is call `WithField` for
// each `Field`.
func (logger *Logger) WithFields(fields Fields) *Entry {
	return NewEntry(logger).WithFields(fields)
}

func (logger *Logger) Debugf(format string, args ...interface{}) {
	if logger.Level >= DebugLevel {
		NewEntry(logger).Debugf(format, args...)
	}
}

func (logger *Logger) Infof(format string, args ...interface{}) {
	if logger.Level >= InfoLevel {
		NewEntry(logger).Infof(format, args...)
	}
}

func (logger *Logger) Printf(format string, args ...interface{}) {
	NewEntry(logger).Printf(format, args...)
}

func (logger *Logger) Warnf(format string, args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warnf(format, args...)
	}
}

func (logger *Logger) Warningf(format string, args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warnf(format, args...)
	}
}

func (logger *Logger) Errorf(format string, args ...interface{}) {
	if logger.Level >= ErrorLevel {
		NewEntry(logger).Errorf(format, args...)
	}
}

func (logger *Logger) Fatalf(format string, args ...interface{}) {
	if logger.Level >= FatalLevel {
		NewEntry(logger).Fatalf(format, args...)
	}
	os.Exit(1)
}

func (logger *Logger) Panicf(format string, args ...interface{}) {
	if logger.Level >= PanicLevel {
		NewEntry(logger).Panicf(format, args...)
	}
}

func (logger *Logger) Debug(args ...interface{}) {
	if logger.Level >= DebugLevel {
		NewEntry(logger).Debug(args...)
	}
}

func (logger *Logger) Info(args ...interface{}) {
	if logger.Level >= InfoLevel {
		NewEntry(logger).Info(args...)
	}
}

func (logger *Logger) Print(args ...interface{}) {
	NewEntry(logger).Info(args...)
}

func (logger *Logger) Warn(args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warn(args...)
	}
}

func (logger *Logger) Warning(args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warn(args...)
	}
}

func (logger *Logger) Error(args ...interface{}) {
	if logger.Level >= ErrorLevel {
		NewEntry(logger).Error(args...)
	}
}

func (logger *Logger) Fatal(args ...interface{}) {
	if logger.Level >= FatalLevel {
		NewEntry(logger).Fatal(args...)
	}
	os.Exit(1)
}

func (logger *Logger) Panic(args ...interface{}) {
	if logger.Level >= PanicLevel {
		NewEntry(logger).Panic(args...)
	}
}

func (logger *Logger) Debugln(args ...interface{}) {
	if logger.Level >= DebugLevel {
		NewEntry(logger).Debugln(args...)
	}
}

func (logger *Logger) Infoln(args ...interface{}) {
	if logger.Level >= InfoLevel {
		NewEntry(logger).Infoln(args...)
	}
}

func (logger *Logger) Println(args ...interface{}) {
	NewEntry(logger).Println(args...)
}

func (logger *Logger) Warnln(args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warnln(args...)
	}
}

func (logger *Logger) Warningln(args ...interface{}) {
	if logger.Level >= WarnLevel {
		NewEntry(logger).Warnln(args...)
	}
}

func (logger *Logger) Errorln(args ...interface{}) {
	if logger.Level >= ErrorLevel {
		NewEntry(logger).Errorln(args...)
	}
}

func (logger *Logger) Fatalln(args ...interface{}) {
	if logger.Level >= FatalLevel {
		NewEntry(logger).Fatalln(args...)
	}
	os.Exit(1)
}

func (logger *Logger) Panicln(args ...interface{}) {
	if logger.Level >= PanicLevel {
		NewEntry(logger).Panicln(args...)
	}
}
<|endoftext|>


<commit_before>package corpus

import (
	"github.com/marcuswestin/go-tp/ngram"
	"github.com/marcuswestin/go-tp/tokens"
)

/*
1000000 * 10        * 1000                   * 16                                / 1024 / 1024 / 1024
words   * n counts  * words seen at that pos * count for up to 65000 occurances  / kb   / mb   / gb
149 GB

1/10th of the english language
1000000/10 * 10        * 1000                   * 16                                / 1024 / 1024 / 1024
14 GB
*/

type CorpusBuilder struct {
	seenStrings map[string]bool
	tokenTable  *tokens.TokenTable
	ngramTables map[int]*ngram.NGramTable
	// ngramIndex  ngram.NGramIndex
}

func StartBuilding(gramNs ...int) *CorpusBuilder {
	c := &CorpusBuilder{make(map[string]bool), tokens.NewTokenTable(), map[int]*ngram.NGramTable{}}
	for _, gramN := range gramNs {
		c.ngramTables[gramN] = ngram.NewNGramTable(gramN, c.tokenTable)
	}
	return c
}

func (c *CorpusBuilder) ProcessDocument(document string) {
	for _, ngramTable := range c.ngramTables {
		ngramTable.ProcessDocument(document)
	}
}

func (c *CorpusBuilder) PrintCounts() {
	for _, ngramTable := range c.ngramTables {
		ngramTable.PrintCounts()
	}
}

func (c *CorpusBuilder) Freeze() {
	c.tokenTable.Freeze()
	for _, ngramTable := range c.ngramTables {
		ngramTable.Freeze()
	}
}

// func (c *CorpusBuilder) Finish() {
// 	if c.seenStrings == nil {
// 		panic("not extracting")
// 	}
// 	c.numTokens = len(c.seenStrings)
// 	for str := range c.seenStrings {

// 	}
// }

// func StartCorpusExtraction() {

// }

// // func ExtractTokens(corpus string)
<commit_msg>remove unused corpus builder code<commit_after>package corpus

import (
	"github.com/marcuswestin/go-tp/ngram"
	"github.com/marcuswestin/go-tp/tokens"
)

/*
1000000 * 10        * 1000                   * 16                                / 1024 / 1024 / 1024
words   * n counts  * words seen at that pos * count for up to 65000 occurances  / kb   / mb   / gb
149 GB

1/10th of the english language
1000000/10 * 10        * 1000                   * 16                                / 1024 / 1024 / 1024
14 GB
*/

type CorpusBuilder struct {
	tokenTable  *tokens.TokenTable
	ngramTables map[int]*ngram.NGramTable
}

func StartBuilding(gramNs ...int) *CorpusBuilder {
	c := &CorpusBuilder{make(map[string]bool), tokens.NewTokenTable(), map[int]*ngram.NGramTable{}}
	for _, gramN := range gramNs {
		c.ngramTables[gramN] = ngram.NewNGramTable(gramN, c.tokenTable)
	}
	return c
}

func (c *CorpusBuilder) ProcessDocument(document string) {
	for _, ngramTable := range c.ngramTables {
		ngramTable.ProcessDocument(document)
	}
}

func (c *CorpusBuilder) PrintCounts() {
	for _, ngramTable := range c.ngramTables {
		ngramTable.PrintCounts()
	}
}

func (c *CorpusBuilder) Freeze() {
	c.tokenTable.Freeze()
	for _, ngramTable := range c.ngramTables {
		ngramTable.Freeze()
	}
}
<|endoftext|>