<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A simple tool for mounting sample file systems, used by the tests in
// samples/.
package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"

	"github.com/jacobsa/bazilfuse"
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/samples/flushfs"
	"golang.org/x/net/context"
)

var fType = flag.String("type", "", "The name of the samples/ sub-dir.")
var fMountPoint = flag.String("mount_point", "", "Path to mount point.")
var fReadyFile = flag.Uint64("ready_file", 0, "FD to signal when ready.")

var fFlushesFile = flag.Uint64("flushfs.flushes_file", 0, "")
var fFsyncsFile = flag.Uint64("flushfs.fsyncs_file", 0, "")
var fFlushError = flag.Int("flushfs.flush_error", 0, "")
var fFsyncError = flag.Int("flushfs.fsync_error", 0, "")

func makeFlushFS() (fs fuse.FileSystem, err error) {
	// Check the flags.
	if *fFlushesFile == 0 || *fFsyncsFile == 0 {
		err = fmt.Errorf("You must set the flushfs flags.")
		return
	}

	// Set up the files.
	flushes := os.NewFile(uintptr(*fFlushesFile), "(flushes file)")
	fsyncs := os.NewFile(uintptr(*fFsyncsFile), "(fsyncs file)")

	// Set up errors.
	var flushErr error
	var fsyncErr error

	if *fFlushError != 0 {
		flushErr = bazilfuse.Errno(*fFlushError)
	}

	if *fFsyncError != 0 {
		fsyncErr = bazilfuse.Errno(*fFsyncError)
	}

	// Report flushes and fsyncs by writing the contents followed by a newline.
	report := func(f *os.File, outErr error) func(string) error {
		return func(s string) (err error) {
			buf := []byte(s)
			buf = append(buf, '\n')

			_, err = f.Write(buf)
			if err != nil {
				err = fmt.Errorf("Write: %v", err)
				return
			}

			err = outErr
			return
		}
	}

	reportFlush := report(flushes, flushErr)
	reportFsync := report(fsyncs, fsyncErr)

	// Create the file system.
	fs, err = flushfs.NewFileSystem(reportFlush, reportFsync)

	return
}

func makeFS() (fs fuse.FileSystem, err error) {
	switch *fType {
	default:
		err = fmt.Errorf("Unknown FS type: %v", *fType)

	case "flushfs":
		fs, err = makeFlushFS()
	}

	return
}

func getReadyFile() (f *os.File, err error) {
	if *fReadyFile == 0 {
		err = errors.New("You must set --ready_file.")
		return
	}

	f = os.NewFile(uintptr(*fReadyFile), "(ready file)")
	return
}

func main() {
	flag.Parse()

	// Allow parallelism in the file system implementation, to help flush out
	// bugs like https://github.com/jacobsa/fuse/issues/4.
	runtime.GOMAXPROCS(2)

	// Grab the file to signal when ready.
	readyFile, err := getReadyFile()
	if err != nil {
		log.Fatalf("getReadyFile:", err)
	}

	// Create an appropriate file system.
	fs, err := makeFS()
	if err != nil {
		log.Fatalf("makeFS: %v", err)
	}

	// Mount the file system.
	if *fMountPoint == "" {
		log.Fatalf("You must set --mount_point.")
	}

	mfs, err := fuse.Mount(*fMountPoint, fs, &fuse.MountConfig{})
	if err != nil {
		log.Fatalf("Mount: %v", err)
	}

	// Wait for it to be ready.
	if err = mfs.WaitForReady(context.Background()); err != nil {
		log.Fatalf("WaitForReady: %v", err)
	}

	// Signal that it is ready.
	_, err = readyFile.Write([]byte("x"))
	if err != nil {
		log.Fatalf("readyFile.Write: %v", err)
	}

	// Wait for it to be unmounted.
	if err = mfs.Join(context.Background()); err != nil {
		log.Fatalf("Join: %v", err)
	}
}
<commit_msg>Updated mount_sample.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A simple tool for mounting sample file systems, used by the tests in
// samples/.
package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"

	"github.com/jacobsa/bazilfuse"
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/samples/flushfs"
	"golang.org/x/net/context"
)

var fType = flag.String("type", "", "The name of the samples/ sub-dir.")
var fMountPoint = flag.String("mount_point", "", "Path to mount point.")
var fReadyFile = flag.Uint64("ready_file", 0, "FD to signal when ready.")

var fFlushesFile = flag.Uint64("flushfs.flushes_file", 0, "")
var fFsyncsFile = flag.Uint64("flushfs.fsyncs_file", 0, "")
var fFlushError = flag.Int("flushfs.flush_error", 0, "")
var fFsyncError = flag.Int("flushfs.fsync_error", 0, "")

func makeFlushFS() (server fuse.Server, err error) {
	// Check the flags.
	if *fFlushesFile == 0 || *fFsyncsFile == 0 {
		err = fmt.Errorf("You must set the flushfs flags.")
		return
	}

	// Set up the files.
	flushes := os.NewFile(uintptr(*fFlushesFile), "(flushes file)")
	fsyncs := os.NewFile(uintptr(*fFsyncsFile), "(fsyncs file)")

	// Set up errors.
	var flushErr error
	var fsyncErr error

	if *fFlushError != 0 {
		flushErr = bazilfuse.Errno(*fFlushError)
	}

	if *fFsyncError != 0 {
		fsyncErr = bazilfuse.Errno(*fFsyncError)
	}

	// Report flushes and fsyncs by writing the contents followed by a newline.
	report := func(f *os.File, outErr error) func(string) error {
		return func(s string) (err error) {
			buf := []byte(s)
			buf = append(buf, '\n')

			_, err = f.Write(buf)
			if err != nil {
				err = fmt.Errorf("Write: %v", err)
				return
			}

			err = outErr
			return
		}
	}

	reportFlush := report(flushes, flushErr)
	reportFsync := report(fsyncs, fsyncErr)

	// Create the file system.
	server, err = flushfs.NewFileSystem(reportFlush, reportFsync)

	return
}

func makeFS() (server fuse.Server, err error) {
	switch *fType {
	default:
		err = fmt.Errorf("Unknown FS type: %v", *fType)

	case "flushfs":
		server, err = makeFlushFS()
	}

	return
}

func getReadyFile() (f *os.File, err error) {
	if *fReadyFile == 0 {
		err = errors.New("You must set --ready_file.")
		return
	}

	f = os.NewFile(uintptr(*fReadyFile), "(ready file)")
	return
}

func main() {
	flag.Parse()

	// Allow parallelism in the file system implementation, to help flush out
	// bugs like https://github.com/jacobsa/fuse/issues/4.
	runtime.GOMAXPROCS(2)

	// Grab the file to signal when ready.
	readyFile, err := getReadyFile()
	if err != nil {
		log.Fatalf("getReadyFile:", err)
	}

	// Create an appropriate file system.
	server, err := makeFS()
	if err != nil {
		log.Fatalf("makeFS: %v", err)
	}

	// Mount the file system.
	if *fMountPoint == "" {
		log.Fatalf("You must set --mount_point.")
	}

	mfs, err := fuse.Mount(*fMountPoint, server, &fuse.MountConfig{})
	if err != nil {
		log.Fatalf("Mount: %v", err)
	}

	// Signal that it is ready.
	_, err = readyFile.Write([]byte("x"))
	if err != nil {
		log.Fatalf("readyFile.Write: %v", err)
	}

	// Wait for it to be unmounted.
	if err = mfs.Join(context.Background()); err != nil {
		log.Fatalf("Join: %v", err)
	}
}
<|endoftext|>


<commit_before>package log

import (
	"fmt"
	"os"
	"strings"
	"time"
)

type logLevel int

const (
	debug = logLevel(0)
	info  = logLevel(1)
	warn  = logLevel(2)
	error = logLevel(3)
	fatal = logLevel(4)
)

func (l logLevel) Int() int {
	return int(l)
}

func (l logLevel) Name() string {

	switch l {
	case debug:
		return "DBG"
	case info:
		return "NFO"
	case warn:
		return "WRN"
	case error:
		return "ERR"
	case fatal:
		return "FTL"
	default:
		return "???"
	}

}

var level = info

// SetLevel sets global logger level
func SetLevel(l string) {
	l = strings.ToLower(l)
	if l == "debug" {
		level = debug
	} else {
		level = info
	}
}

var out = os.Stderr

func accept(lvl logLevel) bool {
	return lvl.Int() >= level.Int()
}

func log(lvl logLevel, f string, args ...interface{}) {
	if accept(lvl) {
		m := fmt.Sprintf(f, args...)
		t := time.Now()
		fmt.Printf("%s[%s] %s\t%s%s", lvl.Color(), lvl.Name(), t, m, colorReset)
		fmt.Println()
		if lvl == fatal {
			panic(m)
		}
	}
}

// Debug prints debug message
func Debug(msg string) {
	log(debug, msg)
}

// Debugf prints formatted debug message
func Debugf(f string, args ...interface{}) {
	log(debug, f, args...)
}

// Info prints info log message
func Info(msg string) {
	log(info, msg)
}

// Infof prints formatted info log message
func Infof(f string, args ...interface{}) {
	log(info, f, args...)
}

// Warn prints warn log message
func Warn(msg string) {
	log(warn, msg)
}

// Warnf prints formatted warn log message
func Warnf(f string, args ...interface{}) {
	log(warn, f, args...)
}

// Error prints error log message
func Error(msg string) {
	log(error, msg)
}

// Errorf prints formatted error log message
func Errorf(f string, args ...interface{}) {
	log(error, f, args...)
}

// Fatal print fatal error and panic
func Fatal(msg string) {
	log(fatal, msg)
}

// Fatalf print formatted fatal error and panic
func Fatalf(f string, args ...interface{}) {
	log(fatal, f, args...)
}
<commit_msg>add mutex for logging<commit_after>package log

import (
	"fmt"
	"os"
	"strings"
	"time"
	"sync"
)

var mtx sync.Mutex

type logLevel int

const (
	debug = logLevel(0)
	info  = logLevel(1)
	warn  = logLevel(2)
	error = logLevel(3)
	fatal = logLevel(4)
)

func (l logLevel) Int() int {
	return int(l)
}

func (l logLevel) Name() string {

	switch l {
	case debug:
		return "DBG"
	case info:
		return "NFO"
	case warn:
		return "WRN"
	case error:
		return "ERR"
	case fatal:
		return "FTL"
	default:
		return "???"
	}

}

var level = info

// SetLevel sets global logger level
func SetLevel(l string) {
	l = strings.ToLower(l)
	if l == "debug" {
		level = debug
	} else {
		level = info
	}
}

var out = os.Stderr

func accept(lvl logLevel) bool {
	return lvl.Int() >= level.Int()
}

func log(lvl logLevel, f string, args ...interface{}) {
	if accept(lvl) {
		m := fmt.Sprintf(f, args...)
		t := time.Now()
		mtx.Lock()
		fmt.Printf("%s[%s] %s\t%s%s", lvl.Color(), lvl.Name(), t, m, colorReset)
		fmt.Println()
		mtx.Unlock()
		if lvl == fatal {
			panic(m)
		}
	}
}

// Debug prints debug message
func Debug(msg string) {
	log(debug, msg)
}

// Debugf prints formatted debug message
func Debugf(f string, args ...interface{}) {
	log(debug, f, args...)
}

// Info prints info log message
func Info(msg string) {
	log(info, msg)
}

// Infof prints formatted info log message
func Infof(f string, args ...interface{}) {
	log(info, f, args...)
}

// Warn prints warn log message
func Warn(msg string) {
	log(warn, msg)
}

// Warnf prints formatted warn log message
func Warnf(f string, args ...interface{}) {
	log(warn, f, args...)
}

// Error prints error log message
func Error(msg string) {
	log(error, msg)
}

// Errorf prints formatted error log message
func Errorf(f string, args ...interface{}) {
	log(error, f, args...)
}

// Fatal print fatal error and panic
func Fatal(msg string) {
	log(fatal, msg)
}

// Fatalf print formatted fatal error and panic
func Fatalf(f string, args ...interface{}) {
	log(fatal, f, args...)
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"

	hh "dmitryfrank.com/geekmarks/server/httphelper"
	"dmitryfrank.com/geekmarks/server/interror"

	"goji.io/pat"

	"github.com/gorilla/websocket"
	"github.com/juju/errors"
)

type WebSocketRequest struct {
	Method string `json:"method"`
	// Path after user address: e.g. the replica of "/api/my/tags" is "/tags".
	Path   string                 `json:"path"`
	Values map[string]interface{} `json:"values"`
	Body   interface{}            `json:"body,omitempty"`
}

type WebSocketResponse struct {
	Method string `json:"method"`
	// Path after user address: e.g. the replica of "/api/my/tags" is "/tags".
	Path   string                 `json:"path"`
	Values map[string]interface{} `json:"values,omitempty"`
	Status int                    `json:"status"`
	Body   interface{}            `json:"body"`
}

type route struct {
	pattern *pat.Pattern
	handler GMHandler
}

type WebSocketMux struct {
	routes []route
}

func (m *WebSocketMux) Add(pattern *pat.Pattern, handler GMHandler) {
	m.routes = append(m.routes, route{
		pattern: pattern,
		handler: handler,
	})
}

func (m *WebSocketMux) Handle(gmr *GMRequest) (resp interface{}, err error) {
	for _, route := range m.routes {
		if r2 := route.pattern.Match(gmr.HttpReq); r2 != nil {
			gmr.HttpReq = r2
			resp, err = route.handler(gmr)
			if err != nil {
				return nil, errors.Trace(err)
			}
			break
		}
	}

	if resp == nil {
		// TODO: use specific error for which hh.GetHTTPErrorCode would return 404
		return nil, errors.Errorf("page not found")
	}

	return resp, nil
}

func parseWebSocketRequest(reader io.Reader) (*WebSocketRequest, error) {
	var wsr *WebSocketRequest
	decoder := json.NewDecoder(reader)
	// UseNumber is needed to prevent from interpeting numbers in body as floats,
	// and re-marshalling them in scientific notation (in case the number is
	// large)
	decoder.UseNumber()
	err := decoder.Decode(&wsr)
	if err != nil {
		return nil, interror.WrapInternalError(
			err,
			errors.Errorf("invalid data"),
		)
	}

	return wsr, nil
}

func mkHttpReqFromWebSocketReq(wsr *WebSocketRequest) (*http.Request, error) {
	r := http.Request{}
	switch wsr.Method {
	case "GET":
		r.Method = wsr.Method
	case "POST":
		r.Method = wsr.Method
	case "PUT":
		r.Method = wsr.Method
	case "DELETE":
		r.Method = wsr.Method
	default:
		return nil, errors.Errorf("wrong method %q", wsr.Method)
	}

	r.URL = &url.URL{
		Path: wsr.Path,
	}

	bodyData, err := json.Marshal(wsr.Body)
	if err != nil {
		return nil, errors.Trace(err)
	}

	r.Body = ioutil.NopCloser(bytes.NewReader(bodyData))

	// TODO: Probably set Form values as well

	return &r, nil
}

var upgrader = websocket.Upgrader{
	CheckOrigin: func(r *http.Request) bool { return true },
}

func (gm *GMServer) webSocketConnect(
	w http.ResponseWriter,
	r *http.Request,
	gsu getSubjUser,
	wsMux GMHandler,
) error {
	subjUser, err := gm.getUserAndAuthorizeByReq(r, gsu, &authzArgs{})
	if err != nil {
		return errors.Trace(err)
	}

	caller := getAuthnUserDataByReq(r)

	fmt.Println("subj user:", subjUser)

	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		return errors.Trace(err)
	}

	go func() (err error) {
		defer func() {
			fmt.Printf("error: %s\n", err)
		}()
		for {
			messageType, reader, err := conn.NextReader()
			if err != nil {
				return errors.Trace(err)
			}

			status := http.StatusOK

			// Here we define and call this intermediary function, because the error
			// which happens there is not considered fatal: instead, it is reported
			// back to the client.
			resp, wsr, err := func() (resp interface{}, wsr *WebSocketRequest, err error) {
				wsr, err = parseWebSocketRequest(reader)
				if err != nil {
					return nil, wsr, errors.Trace(err)
				}

				gmr, err := makeGMRequestFromWebSocketRequest(
					wsr, caller, subjUser,
				)
				if err != nil {
					return nil, wsr, errors.Trace(err)
				}

				resp, err = wsMux(gmr)
				if err != nil {
					return nil, wsr, errors.Trace(err)
				}

				return resp, wsr, nil
			}()
			if err != nil {
				errResp := hh.GetErrorStruct(err)
				status = errResp.Status
				resp = errResp
			}

			if wsr == nil {
				wsr = &WebSocketRequest{}
			}

			fullResp := WebSocketResponse{
				Method: wsr.Method,
				Path:   wsr.Path,
				Values: wsr.Values,
				Status: status,
				Body:   resp,
			}

			w, err := conn.NextWriter(messageType)
			if err != nil {
				return errors.Trace(err)
			}

			encoder := json.NewEncoder(w)
			err = encoder.Encode(fullResp)
			if err != nil {
				return errors.Trace(err)
			}
			if err := w.Close(); err != nil {
				return errors.Trace(err)
			}
		}
	}()

	return nil
}
<commit_msg>Add basic logging of websocket requests<commit_after>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	hh "dmitryfrank.com/geekmarks/server/httphelper"
	"dmitryfrank.com/geekmarks/server/interror"

	"goji.io/pat"

	"github.com/golang/glog"
	"github.com/gorilla/websocket"
	"github.com/juju/errors"
)

type WebSocketRequest struct {
	Method string `json:"method"`
	// Path after user address: e.g. the replica of "/api/my/tags" is "/tags".
	Path   string                 `json:"path"`
	Values map[string]interface{} `json:"values"`
	Body   interface{}            `json:"body,omitempty"`
}

type WebSocketResponse struct {
	Method string `json:"method"`
	// Path after user address: e.g. the replica of "/api/my/tags" is "/tags".
	Path   string                 `json:"path"`
	Values map[string]interface{} `json:"values,omitempty"`
	Status int                    `json:"status"`
	Body   interface{}            `json:"body"`
}

type route struct {
	pattern *pat.Pattern
	handler GMHandler
}

type WebSocketMux struct {
	routes []route
}

func (m *WebSocketMux) Add(pattern *pat.Pattern, handler GMHandler) {
	m.routes = append(m.routes, route{
		pattern: pattern,
		handler: handler,
	})
}

func (m *WebSocketMux) Handle(gmr *GMRequest) (resp interface{}, err error) {
	for _, route := range m.routes {
		if r2 := route.pattern.Match(gmr.HttpReq); r2 != nil {
			gmr.HttpReq = r2
			resp, err = route.handler(gmr)
			if err != nil {
				return nil, errors.Trace(err)
			}
			break
		}
	}

	if resp == nil {
		// TODO: use specific error for which hh.GetHTTPErrorCode would return 404
		return nil, errors.Errorf("page not found")
	}

	return resp, nil
}

func parseWebSocketRequest(reader io.Reader) (*WebSocketRequest, error) {
	var wsr *WebSocketRequest
	decoder := json.NewDecoder(reader)
	// UseNumber is needed to prevent from interpeting numbers in body as floats,
	// and re-marshalling them in scientific notation (in case the number is
	// large)
	decoder.UseNumber()
	err := decoder.Decode(&wsr)
	if err != nil {
		return nil, interror.WrapInternalError(
			err,
			errors.Errorf("invalid data"),
		)
	}

	return wsr, nil
}

func mkHttpReqFromWebSocketReq(wsr *WebSocketRequest) (*http.Request, error) {
	r := http.Request{}
	switch wsr.Method {
	case "GET":
		r.Method = wsr.Method
	case "POST":
		r.Method = wsr.Method
	case "PUT":
		r.Method = wsr.Method
	case "DELETE":
		r.Method = wsr.Method
	default:
		return nil, errors.Errorf("wrong method %q", wsr.Method)
	}

	r.URL = &url.URL{
		Path: wsr.Path,
	}

	bodyData, err := json.Marshal(wsr.Body)
	if err != nil {
		return nil, errors.Trace(err)
	}

	r.Body = ioutil.NopCloser(bytes.NewReader(bodyData))

	// TODO: Probably set Form values as well

	return &r, nil
}

var upgrader = websocket.Upgrader{
	CheckOrigin: func(r *http.Request) bool { return true },
}

func (gm *GMServer) webSocketConnect(
	w http.ResponseWriter,
	r *http.Request,
	gsu getSubjUser,
	wsMux GMHandler,
) error {
	subjUser, err := gm.getUserAndAuthorizeByReq(r, gsu, &authzArgs{})
	if err != nil {
		return errors.Trace(err)
	}

	caller := getAuthnUserDataByReq(r)

	fmt.Println("subj user:", subjUser)

	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		return errors.Trace(err)
	}

	go func() (err error) {
		defer func() {
			fmt.Printf("error: %s\n", err)
		}()
		for {
			messageType, reader, err := conn.NextReader()
			if err != nil {
				return errors.Trace(err)
			}

			// Start timer
			start := time.Now()

			status := http.StatusOK

			// Here we define and call this intermediary function, because the error
			// which happens there is not considered fatal: instead, it is reported
			// back to the client.
			resp, wsr, err := func() (resp interface{}, wsr *WebSocketRequest, err error) {
				wsr, err = parseWebSocketRequest(reader)
				if err != nil {
					return nil, wsr, errors.Trace(err)
				}

				gmr, err := makeGMRequestFromWebSocketRequest(
					wsr, caller, subjUser,
				)
				if err != nil {
					return nil, wsr, errors.Trace(err)
				}

				resp, err = wsMux(gmr)
				if err != nil {
					return nil, wsr, errors.Trace(err)
				}

				return resp, wsr, nil
			}()
			if err != nil {
				errResp := hh.GetErrorStruct(err)
				status = errResp.Status
				resp = errResp
			}

			if wsr == nil {
				wsr = &WebSocketRequest{}
			}

			fullResp := WebSocketResponse{
				Method: wsr.Method,
				Path:   wsr.Path,
				Values: wsr.Values,
				Status: status,
				Body:   resp,
			}

			// Stop timer
			end := time.Now()
			latency := end.Sub(start)

			w, err := conn.NextWriter(messageType)
			if err != nil {
				return errors.Trace(err)
			}

			encoder := json.NewEncoder(w)
			err = encoder.Encode(fullResp)
			if err != nil {
				return errors.Trace(err)
			}
			if err := w.Close(); err != nil {
				return errors.Trace(err)
			}

			glog.Infof("%q: %13v", wsr, latency)
		}
	}()

	return nil
}
<|endoftext|>


<commit_before>//
// Copyright 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package credentials

import (
	"fmt"
	"golang.org/x/net/context"
	"github.com/google/oauth2l/sgauth/internal"
)

// DefaultTokenURL is Google's OAuth 2.0 token URL to use with the service
// account flow.
const DefaultTokenURL = "https://oauth2.googleapis.com/token"

// DefaultAuthURL is Google's OAuth 2.0 Auth URL to use with the end-user
// authentication flow.
const DefaultAuthURL = "https://accounts.google.com/o/oauth2/auth"

// JSON key file types.
const (
	ServiceAccountKey  = "service_account"
	UserCredentialsKey = "authorized_user"
	OAuthClientKey = "oauth_client"
)

// Contains data for OAuthClient key.
type OAuthClient struct {
	ProjectID    string `json:"project_id"`
	ClientSecret string `json:"client_secret"`
	ClientID     string `json:"client_id"`
	TokenURL     string `json:"token_uri"`
	AuthURL      string `json:"auth_uri"`
	RedirectURL  []string `json:"redirect_uris"`
}

// File is the unmarshalled representation of a credentials file.
type File struct {
	// serviceAccountKey or userCredentialsKey
	// Otherwise empty.
	Type string `json:"type"`

	// Service Account fields
	ClientEmail  string `json:"client_email"`
	PrivateKeyID string `json:"private_key_id"`
	PrivateKey   string `json:"private_key"`
	TokenURL     string `json:"token_uri"`
	AuthURL      string `json:"auth_uri"`
	ProjectID    string `json:"project_id"`

	// User Credential fields
	// (These typically come from gcloud auth.)
	ClientSecret string `json:"client_secret"`
	ClientID     string `json:"client_id"`
	RefreshToken string `json:"refresh_token"`

	// Web application credential
	Web OAuthClient `json:"web"`

	// Other application credential
	Installed OAuthClient `json:"installed"`
}

// Returns the credential type of the file.
func (f *File) CredentialsType() string {
	if f.Type != "" {
		return f.Type
	} else if f.Web.ProjectID != "" || f.Installed.ProjectID != "" {
		return OAuthClientKey
	}
	return ""
}

// Construct the corresponding token source based on the type of the file.
func (f *File) TokenSource(ctx context.Context, scopes []string,
	handler func(string)(string, error), state string) (internal.TokenSource, error) {
	switch f.CredentialsType() {
	case ServiceAccountKey:
		cfg := JWTConfigFromFile(f, scopes)
		return cfg.TokenSource(ctx), nil
	case UserCredentialsKey:
		authURL := f.AuthURL
		tokenURL := f.TokenURL
		// Falling back to default URLs only if file URLs are empty
		if authURL == "" {
			authURL = DefaultAuthURL
		}
		if tokenURL == "" {
			tokenURL = DefaultTokenURL
		}
		cfg := &internal.Config{
			ClientID:     f.ClientID,
			ClientSecret: f.ClientSecret,
			Scopes:       scopes,
			Endpoint:     internal.Endpoint{
				AuthURL: authURL,
				TokenURL: tokenURL,
			},
		}
		tok := &internal.Token{RefreshToken: f.RefreshToken}
		return cfg.TokenSource(ctx, tok), nil
	case OAuthClientKey:
		var oauth OAuthClient
		if (f.Web.ProjectID != "") {
			oauth = f.Web
		} else {
			oauth = f.Installed
		}

		if handler == nil {
			handler = defaultAuthorizeFlowHandler
		}

		cfg := &internal.Config{
			ClientID:     oauth.ClientID,
			ClientSecret: oauth.ClientSecret,
			Scopes:       scopes,
			Endpoint:     internal.Endpoint{
				AuthURL: oauth.AuthURL,
				TokenURL: oauth.TokenURL,
			},
			FlowHandler: handler,
			State: state,
			RedirectURL: oauth.RedirectURL[0],
		}
		return cfg.TokenSource(ctx, nil), nil
	default:
		return nil, fmt.Errorf("unknown credential type: %q", f.Type)
	}
}

// Default 3LO authorization handler. Prints the authorization URL on stdout
// and reads the verification code from stdin.
func defaultAuthorizeFlowHandler(authorizeUrl string) (string, error) {
	// Print the url on console, let user authorize and paste the token back.
	fmt.Printf("Go to the following link in your browser:\n\n   %s\n\n", authorizeUrl)
	fmt.Println("Enter verification code: ")
	var code string
	fmt.Scanln(&code)
	return code, nil
}
<commit_msg>Add redirect url check for end-user client key file (#64)<commit_after>//
// Copyright 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package credentials

import (
	"errors"
	"fmt"
	"golang.org/x/net/context"
	"github.com/google/oauth2l/sgauth/internal"
)

// DefaultTokenURL is Google's OAuth 2.0 token URL to use with the service
// account flow.
const DefaultTokenURL = "https://oauth2.googleapis.com/token"

// DefaultAuthURL is Google's OAuth 2.0 Auth URL to use with the end-user
// authentication flow.
const DefaultAuthURL = "https://accounts.google.com/o/oauth2/auth"

// JSON key file types.
const (
	ServiceAccountKey  = "service_account"
	UserCredentialsKey = "authorized_user"
	OAuthClientKey = "oauth_client"
)

// Contains data for OAuthClient key.
type OAuthClient struct {
	ProjectID    string `json:"project_id"`
	ClientSecret string `json:"client_secret"`
	ClientID     string `json:"client_id"`
	TokenURL     string `json:"token_uri"`
	AuthURL      string `json:"auth_uri"`
	RedirectURL  []string `json:"redirect_uris"`
}

// File is the unmarshalled representation of a credentials file.
type File struct {
	// serviceAccountKey or userCredentialsKey
	// Otherwise empty.
	Type string `json:"type"`

	// Service Account fields
	ClientEmail  string `json:"client_email"`
	PrivateKeyID string `json:"private_key_id"`
	PrivateKey   string `json:"private_key"`
	TokenURL     string `json:"token_uri"`
	AuthURL      string `json:"auth_uri"`
	ProjectID    string `json:"project_id"`

	// User Credential fields
	// (These typically come from gcloud auth.)
	ClientSecret string `json:"client_secret"`
	ClientID     string `json:"client_id"`
	RefreshToken string `json:"refresh_token"`

	// Web application credential
	Web OAuthClient `json:"web"`

	// Other application credential
	Installed OAuthClient `json:"installed"`
}

// Returns the credential type of the file.
func (f *File) CredentialsType() string {
	if f.Type != "" {
		return f.Type
	} else if f.Web.ProjectID != "" || f.Installed.ProjectID != "" {
		return OAuthClientKey
	}
	return ""
}

// Construct the corresponding token source based on the type of the file.
func (f *File) TokenSource(ctx context.Context, scopes []string,
	handler func(string)(string, error), state string) (internal.TokenSource, error) {
	switch f.CredentialsType() {
	case ServiceAccountKey:
		cfg := JWTConfigFromFile(f, scopes)
		return cfg.TokenSource(ctx), nil
	case UserCredentialsKey:
		authURL := f.AuthURL
		tokenURL := f.TokenURL
		// Falling back to default URLs only if file URLs are empty
		if authURL == "" {
			authURL = DefaultAuthURL
		}
		if tokenURL == "" {
			tokenURL = DefaultTokenURL
		}
		cfg := &internal.Config{
			ClientID:     f.ClientID,
			ClientSecret: f.ClientSecret,
			Scopes:       scopes,
			Endpoint:     internal.Endpoint{
				AuthURL: authURL,
				TokenURL: tokenURL,
			},
		}
		tok := &internal.Token{RefreshToken: f.RefreshToken}
		return cfg.TokenSource(ctx, tok), nil
	case OAuthClientKey:
		var oauth OAuthClient
		if (f.Web.ProjectID != "") {
			oauth = f.Web
		} else {
			oauth = f.Installed
		}

		if len(oauth.RedirectURL) == 0 {
			// Redirect URL is a required field for end-user oauth flow.
			return nil, errors.New("incomplete client key: redirect url is missing")
		}

		if handler == nil {
			handler = defaultAuthorizeFlowHandler
		}

		cfg := &internal.Config{
			ClientID:     oauth.ClientID,
			ClientSecret: oauth.ClientSecret,
			Scopes:       scopes,
			Endpoint:     internal.Endpoint{
				AuthURL: oauth.AuthURL,
				TokenURL: oauth.TokenURL,
			},
			FlowHandler: handler,
			State: state,
			RedirectURL: oauth.RedirectURL[0],
		}
		return cfg.TokenSource(ctx, nil), nil
	default:
		return nil, fmt.Errorf("unknown credential type: %q", f.Type)
	}
}

// Default 3LO authorization handler. Prints the authorization URL on stdout
// and reads the verification code from stdin.
func defaultAuthorizeFlowHandler(authorizeUrl string) (string, error) {
	// Print the url on console, let user authorize and paste the token back.
	fmt.Printf("Go to the following link in your browser:\n\n   %s\n\n", authorizeUrl)
	fmt.Println("Enter verification code: ")
	var code string
	fmt.Scanln(&code)
	return code, nil
}
<|endoftext|>


<commit_before>package proto

import(
	pb "goprotobuf.googlecode.com/hg/proto"
	"io/ioutil"
	"path/filepath"
	"fmt"
	"os"
	"strings"
)

func NewMixer(path string) (*Mixer) {
	version, err := ioutil.ReadFile(path + "/VERSION")

	if err != nil {
		panic("Couldn't find a version for mixer:" + path + ":" + err.String() )
	}

	_, name := filepath.Split(path)

	return &Mixer{
	Name: pb.String(name),
	Version: pb.String( string(version) ),
	Assets: make([]*File,0),
	Features: make([]*Feature,0),
	Recipes: make([]*Recipe,0),
	Rewriters: make([]*File,0),
	Package: nil,
	}

}

func ResolveMixer(name string) (*Mixer) {
	path := os.ShellExpand("$GOHATTAN_DATA/mixers/")

	if !strings.Contains(name, ".mxr") {
		name = name + ".mxr"
	}

	fullPath := filepath.Join(path, name)
	return OpenMixer(fullPath)
}

func (m *Mixer) Write(path string) (outputPath string) {

	name := pb.GetString(m.Name)
	version := pb.GetString(m.Version)
	outputFilename := filepath.Join(path, name + "-" + version + ".mxr")

	bytes, err := pb.Marshal(m)

	if err != nil {
		panic("Could not marshal mixer (" + name + "), " + err.String())
	}

	bytes = Encrypt(bytes)

	ioutil.WriteFile(outputFilename, bytes, uint32(0666) )

	return outputFilename
}

func OpenMixer(location string) (m *Mixer) {

	data, err := ioutil.ReadFile(location)

	if err != nil {
		panic("Could not find mixer file:" + location)
	}

	data = Decrypt(data)

	thisMixer := &Mixer{}
	err = pb.Unmarshal(data, thisMixer)

	if err != nil {
		panic("Error unmarshalling mixer at:" + location)
	}

	return thisMixer
}

func (m *Mixer) Inspect(printFunctions bool) {
	fmt.Printf(" \tFeatures: \n\t\t%v\n", m.Features)

	println("\tAssets:")
	for _, asset := range(m.Assets) {
		fmt.Printf("\t\t -- %v\n", pb.GetString(asset.Path) )
	}

	println("\tRecipes:")
	for _, recipe := range(m.Recipes) {
		fmt.Printf("\t\t -- Recipe (%v) contains (%v) files.\n", pb.GetString(recipe.Name), len(recipe.Files) )
	}

	println("\tRewriters:")
	for _, rewriter := range(m.Rewriters) {
		fmt.Printf("\t\t -- %v\n", pb.GetString(rewriter.Path) )
	}

	println("\tRoot Package:")
	if m.Package != nil {
		fmt.Printf("\t\t -- Name: %v\n", pb.GetString(m.Package.Name) )
		fmt.Printf("\t\t -- Types: %v\n", m.Package.Types )
		fmt.Printf("\t\t -- Dependencies:  %v\n", m.Package.Dependencies )
		if printFunctions {
			fmt.Printf("\t\t -- Functions (%v):\n", len(m.Package.Functions))
			for _, function := range(m.Package.Functions) {
				fmt.Printf("\t\t\t %v\n", function.Stub(m.Package) )
			}
		}
	}

}

/* Dummy encryption for now */

func Encrypt(data []byte) []byte{
	for index, b := range(data) {
		data[index] = rot13(b)
	}
	return data
}

func Decrypt(data []byte) []byte{
	for index, b := range(data) {
		data[index] = rot13(b)
	}
	return data
}

func rot13(b byte) byte {
    	if 'a' <= b && b <= 'z' {
    		b = 'a' + ((b-'a')+13)%26
    	}
    	if 'A' <= b && b <= 'Z' {
    		b = 'A' + ((b-'A')+13)%26
    	}
    	return b
}<commit_msg>Ad mixer.Unpack stub<commit_after>package proto

import(
	pb "goprotobuf.googlecode.com/hg/proto"
	"io/ioutil"
	"path/filepath"
	"fmt"
	"os"
	"strings"
)

func NewMixer(path string) (*Mixer) {
	version, err := ioutil.ReadFile(path + "/VERSION")

	if err != nil {
		panic("Couldn't find a version for mixer:" + path + ":" + err.String() )
	}

	_, name := filepath.Split(path)

	return &Mixer{
	Name: pb.String(name),
	Version: pb.String( string(version) ),
	Assets: make([]*File,0),
	Features: make([]*Feature,0),
	Recipes: make([]*Recipe,0),
	Rewriters: make([]*File,0),
	Package: nil,
	}

}

func ResolveMixer(name string) (*Mixer) {
	path := os.ShellExpand("$GOHATTAN_DATA/mixers/")

	if !strings.Contains(name, ".mxr") {
		name = name + ".mxr"
	}

	fullPath := filepath.Join(path, name)
	return OpenMixer(fullPath)
}

func (m *Mixer) Write(path string) (outputPath string) {

	name := pb.GetString(m.Name)
	version := pb.GetString(m.Version)
	outputFilename := filepath.Join(path, name + "-" + version + ".mxr")

	bytes, err := pb.Marshal(m)

	if err != nil {
		panic("Could not marshal mixer (" + name + "), " + err.String())
	}

	bytes = Encrypt(bytes)

	ioutil.WriteFile(outputFilename, bytes, uint32(0666) )

	return outputFilename
}

func OpenMixer(location string) (m *Mixer) {

	data, err := ioutil.ReadFile(location)

	if err != nil {
		panic("Could not find mixer file:" + location)
	}

	data = Decrypt(data)

	thisMixer := &Mixer{}
	err = pb.Unmarshal(data, thisMixer)

	if err != nil {
		panic("Error unmarshalling mixer at:" + location)
	}

	return thisMixer
}

func (m *Mixer) Inspect(printFunctions bool) {
	fmt.Printf(" \tFeatures: \n\t\t%v\n", m.Features)

	println("\tAssets:")
	for _, asset := range(m.Assets) {
		fmt.Printf("\t\t -- %v\n", pb.GetString(asset.Path) )
	}

	println("\tRecipes:")
	for _, recipe := range(m.Recipes) {
		fmt.Printf("\t\t -- Recipe (%v) contains (%v) files.\n", pb.GetString(recipe.Name), len(recipe.Files) )
	}

	println("\tRewriters:")
	for _, rewriter := range(m.Rewriters) {
		fmt.Printf("\t\t -- %v\n", pb.GetString(rewriter.Path) )
	}

	println("\tRoot Package:")
	fmt.Printf(m.packageSummary())
}

func (m *Mixer) Unpack(path string) {
	m.Assets.Unpack(path)
	m.Recipes.Unpack(path)
	m.Rewriters.Unpack(path)

	summary := m.packageSummary(path)

	dir, _ := filepath.Split(path)
	err := ioutil.WriteFile(filepath.Join(summary, "package-summary.txt"), uint32(0644))

	if err != nil {
		fmt.Printf("Warning : Couldn't write package summary")
	}
}

func (m *Mixer) packageSummary() string {
	summary := ""
    if m.Package != nil {
		fmt.Sprintf("\t\t -- Name: %v\n", pb.GetString(m.Package.Name) )
		fmt.Sprintf("\t\t -- Types: %v\n", m.Package.Types )
		fmt.Sprintf("\t\t -- Dependencies:  %v\n", m.Package.Dependencies )
		if printFunctions {
			fmt.Sprintf("\t\t -- Functions (%v):\n", len(m.Package.Functions))
			for _, function := range(m.Package.Functions) {
				fmt.Sprintf("\t\t\t %v\n", function.Stub(m.Package) )
			}
		}
	}
	return summary
}

/* Dummy encryption for now */

func Encrypt(data []byte) []byte{
	for index, b := range(data) {
		data[index] = rot13(b)
	}
	return data
}

func Decrypt(data []byte) []byte{
	for index, b := range(data) {
		data[index] = rot13(b)
	}
	return data
}

func rot13(b byte) byte {
    	if 'a' <= b && b <= 'z' {
    		b = 'a' + ((b-'a')+13)%26
    	}
    	if 'A' <= b && b <= 'Z' {
    		b = 'A' + ((b-'A')+13)%26
    	}
    	return b
}<|endoftext|>


<commit_before>package log

import (
	"fmt"
	"io"
	"os"
	"time"
)

type Logger struct {
	PrintDebug bool
	Output     io.Writer
	DateFormat string
}

func NewLogger(opts ...func(*Logger)) *Logger {
	l := &Logger{
		PrintDebug: false,
		Output:     os.Stdout,
		DateFormat: "15:04",
	}

	for _, opt := range opts {
		opt(l)
	}

	return l
}

func (l *Logger) log(lvl, msg string, args ...interface{}) {
	fmt.Fprintf(l.Output, "%s %s %s\n", time.Now().Format(l.DateFormat), lvl, fmt.Sprintf(msg, args...))
}

func (l *Logger) Debug(msg string, args ...interface{}) {
	l.log("debug", msg, args...)
}

func (l *Logger) Info(msg string, args ...interface{}) {
	l.log("info ", msg, args...)
}
<commit_msg>Fix the debug logging flag<commit_after>package log

import (
	"fmt"
	"io"
	"os"
	"time"
)

type Logger struct {
	PrintDebug bool
	Output     io.Writer
	DateFormat string
}

func NewLogger(opts ...func(*Logger)) *Logger {
	l := &Logger{
		PrintDebug: false,
		Output:     os.Stdout,
		DateFormat: "15:04",
	}

	for _, opt := range opts {
		opt(l)
	}

	return l
}

func (l *Logger) log(lvl, msg string, args ...interface{}) {
	fmt.Fprintf(l.Output, "%s %s %s\n", time.Now().Format(l.DateFormat), lvl, fmt.Sprintf(msg, args...))
}

func (l *Logger) Debug(msg string, args ...interface{}) {
	if l.PrintDebug {
		l.log("debug", msg, args...)
	}
}

func (l *Logger) Info(msg string, args ...interface{}) {
	l.log("info ", msg, args...)
}
<|endoftext|>


<commit_before>package logger

import (
	"github.com/buzzxu/ironman/conf"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
	"gopkg.in/natefinch/lumberjack.v2"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

func zapLogger(level string) (logger *zap.Logger) {
	loggerConfig := newLoggerConfig(level, nil, nil)
	logger, err := loggerConfig.Build()
	if err != nil {
		panic(err)
	}
	return
}

func newZapLogger(level string, te zapcore.TimeEncoder, de zapcore.DurationEncoder) (logger *zap.Logger) {
	loggerConfig := newLoggerConfig(level, nil, nil)
	logger, err := loggerConfig.Build()
	if err != nil {
		panic(err)
	}
	return
}

func newLoggerConfig(level string, te zapcore.TimeEncoder, de zapcore.DurationEncoder) (loggerConfig zap.Config) {
	loggerConfig = zap.NewProductionConfig()
	if te == nil {
		loggerConfig.EncoderConfig.EncodeTime = timeEncoder
	} else {
		loggerConfig.EncoderConfig.EncodeTime = te
	}
	if de == nil {
		loggerConfig.EncoderConfig.EncodeDuration = milliSecondsDurationEncoder
	} else {
		loggerConfig.EncoderConfig.EncodeDuration = de
	}
	loggerConfig.EncoderConfig.EncodeCaller = callerEncoder
	switch level {
	case "debug":
		loggerConfig.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
		break
	case "info":
		loggerConfig.Level = zap.NewAtomicLevelAt(zap.InfoLevel)
		break
	case "warn":
		loggerConfig.Level = zap.NewAtomicLevelAt(zap.WarnLevel)
		break
	case "error":
		loggerConfig.Level = zap.NewAtomicLevelAt(zap.ErrorLevel)
		break
	default:
		loggerConfig.Level = zap.NewAtomicLevelAt(zap.InfoLevel)
	}
	return
}

func newLogger(level string, json, console, caller bool, lumberjack *lumberjack.Logger) (logger *zap.Logger) {
	writeSyncer := getLogWriter(console, lumberjack)
	encoder := getEncoder(json, caller)
	var logLevel zapcore.Level
	switch strings.ToLower(level) {
	case "debug":
		logLevel = zap.DebugLevel
		break
	case "info":
		logLevel = zap.InfoLevel
		break
	case "warn":
		logLevel = zap.WarnLevel
		break
	case "error":
		logLevel = zap.ErrorLevel
		break
	default:
		logLevel = zap.DebugLevel
	}
	core := zapcore.NewCore(encoder, writeSyncer, logLevel)
	logger = zap.New(core, zap.AddCaller())
	return
}

func defaultLogger(json, console, caller bool) (logger *zap.Logger) {
	highPriority := zap.LevelEnablerFunc(func(lev zapcore.Level) bool {
		return lev >= zap.ErrorLevel
	})
	lowPriority := zap.LevelEnablerFunc(func(lev zapcore.Level) bool {
		//不显示debug
		return lev < zap.ErrorLevel && lev > zap.DebugLevel
	})
	encoder := getEncoder(json, caller)
	lowWriteSyncer := getLogWriter(console, lumberJack(conf.ServerConf.Logger.Dir+string(filepath.Separator)+"info.log"))
	highWriteSyncer := getLogWriter(console, lumberJack(conf.ServerConf.Logger.Dir+string(filepath.Separator)+"error.log"))

	lowCore := zapcore.NewCore(encoder, lowWriteSyncer, lowPriority)
	highCore := zapcore.NewCore(encoder, highWriteSyncer, highPriority)
	logger = zap.New(zapcore.NewTee(highCore, lowCore), zap.AddCaller())
	return
}

func getEncoder(json, caller bool) zapcore.Encoder {
	encoderConfig := zap.NewProductionEncoderConfig()
	encoderConfig.TimeKey = "time"
	encoderConfig.StacktraceKey = "stacktrace"
	encoderConfig.CallerKey = "line"
	encoderConfig.EncodeTime = timeEncoder
	encoderConfig.EncodeLevel = zapcore.CapitalLevelEncoder
	if caller {
		encoderConfig.EncodeCaller = callerEncoder
	}
	encoderConfig.EncodeDuration = zapcore.SecondsDurationEncoder
	encoderConfig.EncodeName = zapcore.FullNameEncoder
	if json {
		return zapcore.NewJSONEncoder(encoderConfig)
	} else {
		return zapcore.NewConsoleEncoder(encoderConfig)
	}
}
func getLogWriter(console bool, hook *lumberjack.Logger) zapcore.WriteSyncer {
	if console {
		return zapcore.NewMultiWriteSyncer(zapcore.AddSync(os.Stdout), zapcore.AddSync(hook))
	}
	return zapcore.AddSync(hook)
}

// callerEncoder will add caller to log. format is "filename:lineNum:funcName", e.g:"zaplog/zaplog_test.go:15:zaplog.TestNewLogger"
func callerEncoder(caller zapcore.EntryCaller, enc zapcore.PrimitiveArrayEncoder) {
	enc.AppendString(strings.Join([]string{caller.TrimmedPath(), runtime.FuncForPC(caller.PC).Name()}, ":"))
}

// timeEncoder specifics the time format
func timeEncoder(t time.Time, enc zapcore.PrimitiveArrayEncoder) {
	enc.AppendString(t.Format("2006-01-02 15:04:05"))
}

// milliSecondsDurationEncoder serializes a time.Duration to a floating-point number of micro seconds elapsed.
func milliSecondsDurationEncoder(d time.Duration, enc zapcore.PrimitiveArrayEncoder) {
	enc.AppendFloat64(float64(d) / float64(time.Millisecond))
}
<commit_msg>logger skip caller<commit_after>package logger

import (
	"github.com/buzzxu/ironman/conf"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
	"gopkg.in/natefinch/lumberjack.v2"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

func zapLogger(level string) (logger *zap.Logger) {
	loggerConfig := newLoggerConfig(level, nil, nil)
	logger, err := loggerConfig.Build()
	if err != nil {
		panic(err)
	}
	return
}

func newZapLogger(level string, te zapcore.TimeEncoder, de zapcore.DurationEncoder) (logger *zap.Logger) {
	loggerConfig := newLoggerConfig(level, nil, nil)
	logger, err := loggerConfig.Build()
	if err != nil {
		panic(err)
	}
	return
}

func newLoggerConfig(level string, te zapcore.TimeEncoder, de zapcore.DurationEncoder) (loggerConfig zap.Config) {
	loggerConfig = zap.NewProductionConfig()
	if te == nil {
		loggerConfig.EncoderConfig.EncodeTime = timeEncoder
	} else {
		loggerConfig.EncoderConfig.EncodeTime = te
	}
	if de == nil {
		loggerConfig.EncoderConfig.EncodeDuration = milliSecondsDurationEncoder
	} else {
		loggerConfig.EncoderConfig.EncodeDuration = de
	}
	loggerConfig.EncoderConfig.EncodeCaller = callerEncoder
	switch level {
	case "debug":
		loggerConfig.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
		break
	case "info":
		loggerConfig.Level = zap.NewAtomicLevelAt(zap.InfoLevel)
		break
	case "warn":
		loggerConfig.Level = zap.NewAtomicLevelAt(zap.WarnLevel)
		break
	case "error":
		loggerConfig.Level = zap.NewAtomicLevelAt(zap.ErrorLevel)
		break
	default:
		loggerConfig.Level = zap.NewAtomicLevelAt(zap.InfoLevel)
	}
	return
}

func newLogger(level string, json, console, caller bool, lumberjack *lumberjack.Logger) (logger *zap.Logger) {
	writeSyncer := getLogWriter(console, lumberjack)
	encoder := getEncoder(json)
	var logLevel zapcore.Level
	switch strings.ToLower(level) {
	case "debug":
		logLevel = zap.DebugLevel
		break
	case "info":
		logLevel = zap.InfoLevel
		break
	case "warn":
		logLevel = zap.WarnLevel
		break
	case "error":
		logLevel = zap.ErrorLevel
		break
	default:
		logLevel = zap.DebugLevel
	}
	core := zapcore.NewCore(encoder, writeSyncer, logLevel)
	if caller {
		logger = zap.New(core, zap.AddCaller())
	} else {
		logger = zap.New(core)
	}

	return
}

func defaultLogger(json, console, caller bool) (logger *zap.Logger) {
	highPriority := zap.LevelEnablerFunc(func(lev zapcore.Level) bool {
		return lev >= zap.ErrorLevel
	})
	lowPriority := zap.LevelEnablerFunc(func(lev zapcore.Level) bool {
		//不显示debug
		return lev < zap.ErrorLevel && lev > zap.DebugLevel
	})
	encoder := getEncoder(json)
	lowWriteSyncer := getLogWriter(console, lumberJack(conf.ServerConf.Logger.Dir+string(filepath.Separator)+"info.log"))
	highWriteSyncer := getLogWriter(console, lumberJack(conf.ServerConf.Logger.Dir+string(filepath.Separator)+"error.log"))

	lowCore := zapcore.NewCore(encoder, lowWriteSyncer, lowPriority)
	highCore := zapcore.NewCore(encoder, highWriteSyncer, highPriority)
	if caller {
		logger = zap.New(zapcore.NewTee(highCore, lowCore), zap.AddCaller())
	} else {
		logger = zap.New(zapcore.NewTee(highCore, lowCore))
	}
	return
}

func getEncoder(json bool) zapcore.Encoder {
	encoderConfig := zap.NewProductionEncoderConfig()
	encoderConfig.TimeKey = "time"
	encoderConfig.StacktraceKey = "stacktrace"
	encoderConfig.CallerKey = "line"
	encoderConfig.EncodeTime = timeEncoder
	encoderConfig.EncodeLevel = zapcore.CapitalLevelEncoder
	encoderConfig.EncodeCaller = callerEncoder
	encoderConfig.EncodeDuration = zapcore.SecondsDurationEncoder
	encoderConfig.EncodeName = zapcore.FullNameEncoder
	if json {
		return zapcore.NewJSONEncoder(encoderConfig)
	} else {
		return zapcore.NewConsoleEncoder(encoderConfig)
	}
}
func getLogWriter(console bool, hook *lumberjack.Logger) zapcore.WriteSyncer {
	if console {
		return zapcore.NewMultiWriteSyncer(zapcore.AddSync(os.Stdout), zapcore.AddSync(hook))
	}
	return zapcore.AddSync(hook)
}

// callerEncoder will add caller to log. format is "filename:lineNum:funcName", e.g:"zaplog/zaplog_test.go:15:zaplog.TestNewLogger"
func callerEncoder(caller zapcore.EntryCaller, enc zapcore.PrimitiveArrayEncoder) {
	enc.AppendString(strings.Join([]string{caller.TrimmedPath(), runtime.FuncForPC(caller.PC).Name()}, ":"))
}

// timeEncoder specifics the time format
func timeEncoder(t time.Time, enc zapcore.PrimitiveArrayEncoder) {
	enc.AppendString(t.Format("2006-01-02 15:04:05"))
}

// milliSecondsDurationEncoder serializes a time.Duration to a floating-point number of micro seconds elapsed.
func milliSecondsDurationEncoder(d time.Duration, enc zapcore.PrimitiveArrayEncoder) {
	enc.AppendFloat64(float64(d) / float64(time.Millisecond))
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/ymotongpoo/goltsv"
	"reflect"
	"strings"
	"testing"
)

var ParseKeysListTests = []struct {
	list string
	keys []string
}{
	// normal
	{`host`, []string{`host`}},
	{`host,status`, []string{`host`, `status`}},
	{`host,status,size`, []string{`host`, `status`, `size`}},

	// include empty keys
	{``, []string{``}},
	{`,`, []string{``, ``}},
	{`,,`, []string{``, ``, ``}},
	{`,host`, []string{``, `host`}},
	{`,,host`, []string{``, ``, `host`}},
	{`host,`, []string{`host`, ``}},
	{`host,,`, []string{`host`, ``, ``}},
	{`,,host,,status,,`, []string{``, ``, `host`, ``, `status`, ``, ``}},

	// include escaped comma
	{`a\,b`, []string{`a,b`}},
	{`a\,\,b`, []string{`a,,b`}},
	{`a\,,b\,`, []string{`a,`, `b,`}},
	{`\,a,\,b`, []string{`,a`, `,b`}},
	{`\,a\,,\,b\,`, []string{`,a,`, `,b,`}},
	{`a\,b,c\,d\,e`, []string{`a,b`, `c,d,e`}},
	{`a\,b,c\,d\,e,f\,g\,h\,i`, []string{`a,b`, `c,d,e`, `f,g,h,i`}},

	// include escaped backslash
	{`a\\b`, []string{`a\b`}},
	{`a\\\\b`, []string{`a\\b`}},
	{`a\\,b\\`, []string{`a\`, `b\`}},
	{`\\a,\\b`, []string{`\a`, `\b`}},
	{`\\a\\,\\b\\`, []string{`\a\`, `\b\`}},
	{`a\\b,c\\d\\e`, []string{`a\b`, `c\d\e`}},
	{`a\\b,c\\d\\e,f\\g\\h\\i`, []string{`a\b`, `c\d\e`, `f\g\h\i`}},
}

func TestParseKeysList(t *testing.T) {
	for _, test := range ParseKeysListTests {
		expect := test.keys
		actual := ParseKeysList(test.list)
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("ParseKeysList(%q) = %q, want %q",
				test.list, actual, expect)
		}
	}
}

func TestNewLTSVScanner(t *testing.T) {
	keys := []string{"host", "time"}
	reader := strings.NewReader(``)

	expect := &LTSVScanner{
		Delimiter: "\t",
		keys:      keys,
		reader:    goltsv.NewReader(reader),
	}
	actual := NewLTSVScanner(keys, reader)
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("NewLTSVScanner(%q, %q) = %q, want %q",
			keys, reader, actual, expect)
	}
}

type ScanResult struct {
	scan bool
	text string
	err  error
}

func TestScan(t *testing.T) {
	keys := []string{"host"}
	reader := strings.NewReader(`
host:192.168.0.1	status:200
host:172.16.0.12	status:404
`[1:])
	l := NewLTSVScanner(keys, reader)

	expects := []ScanResult{
		{scan: true, text: "192.168.0.1", err: nil},
		{scan: true, text: "172.16.0.12", err: nil},
		{scan: false, text: "", err: nil},
	}
	for i := 0; i < len(expects); i++ {
		expect := expects[i]
		actual := ScanResult{}
		actual.scan = l.Scan()
		actual.text = l.Text()
		actual.err = l.Err()
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("Scan %v time: got %v, want %v",
				i+1, actual, expect)
		}
	}
}

func TestScanError(t *testing.T) {
	keys := []string{"host"}
	reader := strings.NewReader(`
host:192.168.0.1	status:200
a	b	c
host:172.16.0.12	status:404
`[1:])
	l := NewLTSVScanner(keys, reader)

	expects := []ScanResult{
		{scan: true, text: "192.168.0.1", err: nil},
		{scan: false, text: "", err: goltsv.ErrLabelName},
		{scan: false, text: "", err: goltsv.ErrLabelName},
	}
	for i := 0; i < len(expects); i++ {
		expect := expects[i]
		actual := ScanResult{}
		actual.scan = l.Scan()
		actual.text = l.Text()
		actual.err = l.Err()
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("Scan %v time: got %v, want %v",
				i+1, actual, expect)
		}
	}
}

var DelimiterTests = []struct {
	keys      []string
	delimiter string
	src       string
	dst       []string
}{
	{
		keys:      []string{"host", "status"},
		delimiter: ",",
		src: `
host:192.168.0.1	status:200
host:172.16.0.12	status:404
`[1:],
		dst: []string{
			"192.168.0.1,200",
			"172.16.0.12,404",
		},
	},
	{
		keys:      []string{"host", "status"},
		delimiter: "--",
		src: `
host:192.168.0.1	status:200
host:172.16.0.12	status:404
`[1:],
		dst: []string{
			"192.168.0.1--200",
			"172.16.0.12--404",
		},
	},
}

func TestDelimiter(t *testing.T) {
	for _, test := range DelimiterTests {
		l := NewLTSVScanner(test.keys, strings.NewReader(test.src))
		l.Delimiter = test.delimiter

		expect := test.dst
		actual := []string{}
		for l.Scan() {
			actual = append(actual, l.Text())
		}
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("(keys: %q, delimiter: %q) got %q, want %q",
				test.keys, test.delimiter, actual, expect)
		}
	}
}
<commit_msg>Add test to be able to remain LTSV<commit_after>package main

import (
	"github.com/ymotongpoo/goltsv"
	"reflect"
	"strings"
	"testing"
)

var ParseKeysListTests = []struct {
	list string
	keys []string
}{
	// normal
	{`host`, []string{`host`}},
	{`host,status`, []string{`host`, `status`}},
	{`host,status,size`, []string{`host`, `status`, `size`}},

	// include empty keys
	{``, []string{``}},
	{`,`, []string{``, ``}},
	{`,,`, []string{``, ``, ``}},
	{`,host`, []string{``, `host`}},
	{`,,host`, []string{``, ``, `host`}},
	{`host,`, []string{`host`, ``}},
	{`host,,`, []string{`host`, ``, ``}},
	{`,,host,,status,,`, []string{``, ``, `host`, ``, `status`, ``, ``}},

	// include escaped comma
	{`a\,b`, []string{`a,b`}},
	{`a\,\,b`, []string{`a,,b`}},
	{`a\,,b\,`, []string{`a,`, `b,`}},
	{`\,a,\,b`, []string{`,a`, `,b`}},
	{`\,a\,,\,b\,`, []string{`,a,`, `,b,`}},
	{`a\,b,c\,d\,e`, []string{`a,b`, `c,d,e`}},
	{`a\,b,c\,d\,e,f\,g\,h\,i`, []string{`a,b`, `c,d,e`, `f,g,h,i`}},

	// include escaped backslash
	{`a\\b`, []string{`a\b`}},
	{`a\\\\b`, []string{`a\\b`}},
	{`a\\,b\\`, []string{`a\`, `b\`}},
	{`\\a,\\b`, []string{`\a`, `\b`}},
	{`\\a\\,\\b\\`, []string{`\a\`, `\b\`}},
	{`a\\b,c\\d\\e`, []string{`a\b`, `c\d\e`}},
	{`a\\b,c\\d\\e,f\\g\\h\\i`, []string{`a\b`, `c\d\e`, `f\g\h\i`}},
}

func TestParseKeysList(t *testing.T) {
	for _, test := range ParseKeysListTests {
		expect := test.keys
		actual := ParseKeysList(test.list)
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("ParseKeysList(%q) = %q, want %q",
				test.list, actual, expect)
		}
	}
}

func TestNewLTSVScanner(t *testing.T) {
	keys := []string{"host", "time"}
	reader := strings.NewReader(``)

	expect := &LTSVScanner{
		Delimiter:  "\t",
		RemainLTSV: false,
		keys:       keys,
		reader:     goltsv.NewReader(reader),
	}
	actual := NewLTSVScanner(keys, reader)
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("NewLTSVScanner(%q, %q) = %q, want %q",
			keys, reader, actual, expect)
	}
}

type ScanResult struct {
	scan bool
	text string
	err  error
}

func TestScan(t *testing.T) {
	keys := []string{"host"}
	reader := strings.NewReader(`
host:192.168.0.1	status:200
host:172.16.0.12	status:404
`[1:])
	l := NewLTSVScanner(keys, reader)

	expects := []ScanResult{
		{scan: true, text: "192.168.0.1", err: nil},
		{scan: true, text: "172.16.0.12", err: nil},
		{scan: false, text: "", err: nil},
	}
	for i := 0; i < len(expects); i++ {
		expect := expects[i]
		actual := ScanResult{}
		actual.scan = l.Scan()
		actual.text = l.Text()
		actual.err = l.Err()
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("Scan %v time: got %v, want %v",
				i+1, actual, expect)
		}
	}
}

func TestScanError(t *testing.T) {
	keys := []string{"host"}
	reader := strings.NewReader(`
host:192.168.0.1	status:200
a	b	c
host:172.16.0.12	status:404
`[1:])
	l := NewLTSVScanner(keys, reader)

	expects := []ScanResult{
		{scan: true, text: "192.168.0.1", err: nil},
		{scan: false, text: "", err: goltsv.ErrLabelName},
		{scan: false, text: "", err: goltsv.ErrLabelName},
	}
	for i := 0; i < len(expects); i++ {
		expect := expects[i]
		actual := ScanResult{}
		actual.scan = l.Scan()
		actual.text = l.Text()
		actual.err = l.Err()
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("Scan %v time: got %v, want %v",
				i+1, actual, expect)
		}
	}
}

var DelimiterTests = []struct {
	keys      []string
	delimiter string
	src       string
	dst       []string
}{
	{
		keys:      []string{"host", "status"},
		delimiter: ",",
		src: `
host:192.168.0.1	status:200
host:172.16.0.12	status:404
`[1:],
		dst: []string{
			"192.168.0.1,200",
			"172.16.0.12,404",
		},
	},
	{
		keys:      []string{"host", "status"},
		delimiter: "--",
		src: `
host:192.168.0.1	status:200
host:172.16.0.12	status:404
`[1:],
		dst: []string{
			"192.168.0.1--200",
			"172.16.0.12--404",
		},
	},
}

func TestDelimiter(t *testing.T) {
	for _, test := range DelimiterTests {
		l := NewLTSVScanner(test.keys, strings.NewReader(test.src))
		l.Delimiter = test.delimiter

		expect := test.dst
		actual := []string{}
		for l.Scan() {
			actual = append(actual, l.Text())
		}
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("(keys: %q, delimiter: %q) got %q, want %q",
				test.keys, test.delimiter, actual, expect)
		}
	}
}

var RemainLTSVTests = []struct {
	keys      []string
	delimiter string
	src       string
	dst       []string
}{
	{
		keys:      []string{"host"},
		delimiter: "\t",
		src: `
host:192.168.0.1	status:200
host:172.16.0.12	status:404
`[1:],
		dst: []string{
			"host:192.168.0.1",
			"host:172.16.0.12",
		},
	},
	{
		keys:      []string{"status", "host"},
		delimiter: "\t",
		src: `
host:192.168.0.1	status:200
host:172.16.0.12	status:404
`[1:],
		dst: []string{
			"status:200\thost:192.168.0.1",
			"status:404\thost:172.16.0.12",
		},
	},
	{
		keys:      []string{"status", "host"},
		delimiter: "---",
		src: `
host:192.168.0.1	status:200
host:172.16.0.12	status:404
`[1:],
		dst: []string{
			"status:200\thost:192.168.0.1",
			"status:404\thost:172.16.0.12",
		},
	},
}

func TestRemainLTSV(t *testing.T) {
	for _, test := range RemainLTSVTests {
		l := NewLTSVScanner(test.keys, strings.NewReader(test.src))
		l.RemainLTSV = true

		expect := test.dst
		actual := []string{}
		for l.Scan() {
			actual = append(actual, l.Text())
		}
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("(keys: %q, remainLTSV: true) got %q, want %q",
				test.keys, actual, expect)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package io

import (
	"os";
	"syscall";
)

export var ErrEOF = os.NewError("EOF")

export type Read interface {
	Read(p *[]byte) (n int, err *os.Error);
}

export type Write interface {
	Write(p *[]byte) (n int, err *os.Error);
}

export type ReadWrite interface {
	Read(p *[]byte) (n int, err *os.Error);
	Write(p *[]byte) (n int, err *os.Error);
}

export type ReadWriteClose interface {
	Read(p *[]byte) (n int, err *os.Error);
	Write(p *[]byte) (n int, err *os.Error);
	Close() *os.Error;
}

export func WriteString(w Write, s string) (n int, err *os.Error) {
	b := new([]byte, len(s)+1);
	if !syscall.StringToBytes(b, s) {
		return -1, os.EINVAL
	}
	// BUG return w.Write(b[0:len(s)])
	r, e := w.Write(b[0:len(s)]);
	return r, e
}

// Read until buffer is full,  EOF, or error
export func Readn(fd Read, buf *[]byte) (n int, err *os.Error) {
	n = 0;
	for n < len(buf) {
		nn, e := fd.Read(buf[n:len(buf)]);
		if nn > 0 {
			n += nn
		}
		if e != nil {
			return n, e
		}
		if nn <= 0 {
			return n, ErrEOF	// no error but insufficient data
		}
	}
	return n, nil
}

// Convert something that implements Read into something
// whose Reads are always Readn
type FullRead struct {
	fd	Read;
}

func (fd *FullRead) Read(p *[]byte) (n int, err *os.Error) {
	n, err = Readn(fd.fd, p);
	return n, err
}

export func MakeFullReader(fd Read) Read {
	if fr, ok := fd.(*FullRead); ok {
		// already a FullRead
		return fd
	}
	return &FullRead{fd}
}
<commit_msg>Copyn<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package io

import (
	"os";
	"syscall";
)

export var ErrEOF = os.NewError("EOF")

export type Read interface {
	Read(p *[]byte) (n int, err *os.Error);
}

export type Write interface {
	Write(p *[]byte) (n int, err *os.Error);
}

export type ReadWrite interface {
	Read(p *[]byte) (n int, err *os.Error);
	Write(p *[]byte) (n int, err *os.Error);
}

export type ReadWriteClose interface {
	Read(p *[]byte) (n int, err *os.Error);
	Write(p *[]byte) (n int, err *os.Error);
	Close() *os.Error;
}

export func WriteString(w Write, s string) (n int, err *os.Error) {
	b := new([]byte, len(s)+1);
	if !syscall.StringToBytes(b, s) {
		return -1, os.EINVAL
	}
	// BUG return w.Write(b[0:len(s)])
	r, e := w.Write(b[0:len(s)]);
	return r, e
}

// Read until buffer is full,  EOF, or error
export func Readn(fd Read, buf *[]byte) (n int, err *os.Error) {
	n = 0;
	for n < len(buf) {
		nn, e := fd.Read(buf[n:len(buf)]);
		if nn > 0 {
			n += nn
		}
		if e != nil {
			return n, e
		}
		if nn <= 0 {
			return n, ErrEOF	// no error but insufficient data
		}
	}
	return n, nil
}

// Convert something that implements Read into something
// whose Reads are always Readn
type FullRead struct {
	fd	Read;
}

func (fd *FullRead) Read(p *[]byte) (n int, err *os.Error) {
	n, err = Readn(fd.fd, p);
	return n, err
}

export func MakeFullReader(fd Read) Read {
	if fr, ok := fd.(*FullRead); ok {
		// already a FullRead
		return fd
	}
	return &FullRead{fd}
}

// Copies n bytes (or until EOF is reached) from src to dst.
// Returns the number of bytes copied and the error, if any.
export func Copyn(src Read, dst Write, n int) (c int, err *os.Error) {
	buf := new([]byte, 32*1024);  // BUG 6g crashes on non-pointer array slices
	c = 0;
	for c < n {
		l := n - c;
		if l > len(buf) {
			l = len(buf)
		}
		nr, er := src.Read(buf[0 : l]);
		if nr > 0 {
			nw, ew := dst.Write(buf[0 : nr]);
			if nw != nr || ew != nil {
				c += nw;
				if ew == nil {
					ew = os.EIO
				}
				err = ew;
				break;
			}
			c += nr;
		}
		if er != nil {
			err = er;
			break;
		}
		if nr == 0 {
			break;
		}
	}
	return c, err
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"path/filepath"

	"github.com/spf13/cobra"

	"restic"
	"restic/errors"
	"restic/repository"
)

var cmdLs = &cobra.Command{
	Use:   "ls [flags] [snapshot-ID ...]",
	Short: "list files in a snapshot",
	Long: `
The "ls" command allows listing files and directories in a snapshot.

The special snapshot-ID "latest" can be used to list files and directories of the latest snapshot in the repository.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runLs(lsOptions, globalOptions, args)
	},
}

// LsOptions collects all options for the ls command.
type LsOptions struct {
	ListLong bool
	Host     string
	Tags     []string
	Paths    []string
}

var lsOptions LsOptions

func init() {
	cmdRoot.AddCommand(cmdLs)

	flags := cmdLs.Flags()
	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")

	flags.StringVarP(&lsOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
	flags.StringSliceVar(&lsOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot ID is given")
	flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
}

func printTree(repo *repository.Repository, id *restic.ID, prefix string) error {
	tree, err := repo.LoadTree(context.TODO(), *id)
	if err != nil {
		return err
	}

	for _, entry := range tree.Nodes {
		Printf(formatNode(prefix, entry, lsOptions.ListLong) + "\n")

		if entry.Type == "dir" && entry.Subtree != nil {
			if err = printTree(repo, entry.Subtree, filepath.Join(prefix, entry.Name)); err != nil {
				return err
			}
		}
	}

	return nil
}

func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
	if len(args) == 0 && opts.Host == "" && len(opts.Tags) == 0 && len(opts.Paths) == 0 {
		return errors.Fatal("Invalid arguments, either give one or more snapshot IDs or set filters.")
	}

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if err = repo.LoadIndex(context.TODO()); err != nil {
		return err
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()
	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
		Verbosef("snapshot %s of %v at %s):\n", sn.ID().Str(), sn.Paths, sn.Time)

		if err = printTree(repo, sn.Tree, string(filepath.Separator)); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>ls: Print names with percent correctly<commit_after>package main

import (
	"context"
	"path/filepath"

	"github.com/spf13/cobra"

	"restic"
	"restic/errors"
	"restic/repository"
)

var cmdLs = &cobra.Command{
	Use:   "ls [flags] [snapshot-ID ...]",
	Short: "list files in a snapshot",
	Long: `
The "ls" command allows listing files and directories in a snapshot.

The special snapshot-ID "latest" can be used to list files and directories of the latest snapshot in the repository.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runLs(lsOptions, globalOptions, args)
	},
}

// LsOptions collects all options for the ls command.
type LsOptions struct {
	ListLong bool
	Host     string
	Tags     []string
	Paths    []string
}

var lsOptions LsOptions

func init() {
	cmdRoot.AddCommand(cmdLs)

	flags := cmdLs.Flags()
	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")

	flags.StringVarP(&lsOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
	flags.StringSliceVar(&lsOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot ID is given")
	flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
}

func printTree(repo *repository.Repository, id *restic.ID, prefix string) error {
	tree, err := repo.LoadTree(context.TODO(), *id)
	if err != nil {
		return err
	}

	for _, entry := range tree.Nodes {
		Printf("%s\n", formatNode(prefix, entry, lsOptions.ListLong))

		if entry.Type == "dir" && entry.Subtree != nil {
			if err = printTree(repo, entry.Subtree, filepath.Join(prefix, entry.Name)); err != nil {
				return err
			}
		}
	}

	return nil
}

func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
	if len(args) == 0 && opts.Host == "" && len(opts.Tags) == 0 && len(opts.Paths) == 0 {
		return errors.Fatal("Invalid arguments, either give one or more snapshot IDs or set filters.")
	}

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if err = repo.LoadIndex(context.TODO()); err != nil {
		return err
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()
	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
		Verbosef("snapshot %s of %v at %s):\n", sn.ID().Str(), sn.Paths, sn.Time)

		if err = printTree(repo, sn.Tree, string(filepath.Separator)); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	Pagelimit = 40
)

/*
 * Helper function to easily read input from console
 */
func readFromConsole() (stringstore string) {
	scanner := bufio.NewScanner(os.Stdin)
	if scanner.Scan() {
		stringstore = scanner.Text()
	}
	return
}

// Given an array of statuses, returns an array of their ids
func getStatusIds(vs []Status) []uint64 {
	vsm := make([]uint64, len(vs))
	for i, v := range vs {
		vsm[i] = v.ID
	}
	return vsm
}

// Returns true if `a` is an element of the array
func idInSlice(a uint64, list []uint64) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

/*
 * Various data structs
 */

type MastoPurgeSettings struct {
	Server       string `json:"server"`
	ClientID     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
	AccessToken  string `json:"access_token"`
}

type AccountInfo struct {
	ID       int    `json:"id,string"`
	Username string `json:"username"`
}

type RespAppRegister struct {
	ClientID     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
}

type RespAccessToken struct {
	AccessToken string `json:"access_token"`
}

type Status struct {
	ID uint64 `json:"id,string"`
	//Content     string  `json:"content"`
	CreatedAt time.Time `json:"created_at"`
}

var (
	noninteractiveMode = flag.Bool("noninteractive", false, "Run in non-interactive mode, suitable for eg. cron jobs. (When run with a missing settings file, the config process will run interactively.)")
	maxAgeArgument     = flag.String("maxage", "", "Max age of posts you want to keep. Required when running in non-interactive mode. Allowed units: hours, days, weeks, months, years. Example: \"6 months\".")
	configFile         = flag.String("config", ".mastopurgesettings", "Path + filename for the settings file.")
	printVersion       = flag.Bool("version", false, "Print version, and exit.")
	quietMode          = flag.Bool("quiet", false, "Reduce output to the most important messages only.")
	dryRun             = flag.Bool("dryrun", false, "Run MastoPurge to preview its results, but without actually deleting any statuses.")
)

var versionString string = "0.0.0"

func main() {
	flag.Parse()

	if *printVersion {
		fmt.Printf("MastoPurge version %s\n", versionString)
		os.Exit(0)
	}

	interactiveMode := !(*noninteractiveMode)

	if interactiveMode && !*quietMode {
		/*
		 * Add fancyness!
		 */
		fmt.Println("\nWelcome to ...")
		fmt.Println("                     _                                   ")
		fmt.Println(" _ __ ___   __ _ ___| |_ ___  _ __  _   _ _ __ __ _  ___ ")
		fmt.Println("| '_ ` _ \\ / _` / __| __/ _ \\| '_ \\| | | | '__/ _` |/ _ \\")
		fmt.Println("| | | | | | (_| \\__ \\ || (_) | |_) | |_| | | | (_| |  __/")
		fmt.Println("|_| |_| |_|\\__,_|___/\\__\\___/| .__/ \\__,_|_|  \\__, |\\___|")
		fmt.Println("                             |_|              |___/      ")
		fmt.Println("    ... add German Datenhygiene to your Mastodon-Account!")
		fmt.Print("\n\n")
		fmt.Printf("Version %s\n\n", versionString)
	}

	/*
	 * Set up settings and Httpclient
	 */
	settings := MastoPurgeSettings{}
	hc := &APIClient{}
	hc.Timeout = time.Second * 5
	hc.Init()

	/*
	 * Check if configuration file (*configFile, default .mastopurgesettings) exists
	 * - Create if it does not exist
	 * - Use if exists
	 */
	configRaw, readerr := ioutil.ReadFile(*configFile)
	if readerr != nil {
		log.Printf("MastoPurge configuration file %s does not exist or is not accessible.\n", *configFile)
		fmt.Println("\nFirst we need to connect MastoPurge to your Mastodon account.")
		fmt.Println("Enter the domain of your Mastodon home instance: (e.g. \"metalhead.club\")")
		fmt.Print("[Mastodon home instance]: ")
		settings.Server = readFromConsole()
		hc.Server = "https://" + settings.Server

		// Register application for user on server
		if !*quietMode {
			log.Println(">>>>>> Registering MastoPurge App on " + settings.Server)
		}
		params := url.Values{}
		params.Add("client_name", "MastoPurge")
		params.Add("redirect_uris", "urn:ietf:wg:oauth:2.0:oob")
		params.Add("scopes", "read write")
		body, registerErr := hc.Request(http.MethodPost, "/api/v1/apps", params)
		if registerErr != nil {
			log.Fatal(registerErr)
		}

		// Parse response JSON
		respAppRegister := RespAppRegister{}
		err := json.Unmarshal(body, &respAppRegister)
		if err != nil {
			log.Fatal(err)
		}
		settings.ClientID = respAppRegister.ClientID
		settings.ClientSecret = respAppRegister.ClientSecret

		// User must manually authenticate app
		authurl := "https://" + settings.Server + "/oauth/authorize?scope=read%20write&response_type=code&redirect_uri=urn:ietf:wg:oauth:2.0:oob&client_id=" + settings.ClientID
		fmt.Println("\nPlease visit this URL in your web browser:")
		fmt.Println(authurl)
		fmt.Println("\n... and enter the code here:")
		fmt.Print("[Auth code]: ")
		code := readFromConsole()

		// Request auth token via auth code ...
		params = url.Values{}
		params.Add("client_id", settings.ClientID)
		params.Add("client_secret", settings.ClientSecret)
		params.Add("grant_type", "authorization_code")
		params.Add("redirect_uri", "urn:ietf:wg:oauth:2.0:oob")
		params.Add("code", code)
		body, err = hc.Request(http.MethodPost, "/oauth/token", params)
		if err != nil {
			log.Fatal(err)
		}

		respAccessToken := RespAccessToken{}
		err = json.Unmarshal(body, &respAccessToken)
		if err != nil {
			log.Fatal(err)
		}
		settings.AccessToken = respAccessToken.AccessToken
		hc.AccessToken = settings.AccessToken

		// Write settings to config file
		configRaw, _ = json.Marshal(settings)
		err = ioutil.WriteFile(*configFile, configRaw, 0600)
		if err != nil {
			log.Fatal(err)
		}

	} else {
		if !*quietMode {
			log.Println("MastoPurge configuration found! Reading config.")
		}

		/*
		 * Load settings
		 */
		err := json.Unmarshal(configRaw, &settings)
		if err != nil {
			log.Fatalf("Config file is malformed :(\nPlease consider deleting '%s' from your file system and starting MastoPurge again.", *configFile)
		}

		hc.Server = "https://" + settings.Server
		hc.AccessToken = settings.AccessToken
	}

	/*
	 * Check if account access is okay
	 */

	if !*quietMode {
		log.Println("Requesting access to Mastodon account")
	}
	body, accessErr := hc.Request(http.MethodGet, "/api/v1/accounts/verify_credentials", nil)
	if accessErr != nil {
		log.Fatal(accessErr)
	} else {
		accountinfo := AccountInfo{}
		err := json.Unmarshal(body, &accountinfo)

		if accountinfo.ID == 0 {
			if !*quietMode {
				log.Println("Access DENIED :-(")
			}
			log.Fatalf("Unfortunately API access was not granted. Consider deleting '%s' and starting MastoPurge again!", *configFile)
		} else {
			if !*quietMode {
				log.Println("Access GRANTED :-)")
				log.Println(">>> Account ID:", accountinfo.ID)
				log.Println(">>> Username:", accountinfo.Username)
			}

			// Do some date calculations ...
			var maxage time.Duration
			var maxtime time.Time

			for {
				maxagestring := *maxAgeArgument
				if maxagestring == "" {
					if interactiveMode {
						fmt.Println("\nEnter the maximum age of the posts you want to KEEP, e.g. \"30 days\". Older posts will be deleted. Allowed units: hours, days, weeks, months, years.")
						fmt.Print("[Maximum post age]: ")
						maxagestring = readFromConsole()
					} else {
						log.Println("missing required argument -maxAge")
						flag.PrintDefaults()
						os.Exit(1)
					}
				}

				parts := strings.Split(maxagestring, " ")

				if len(parts) == 2 {
					maxagenum, converr := strconv.Atoi(parts[0])
					if converr == nil {
						var factor time.Duration

						switch parts[1] {
						case "hours", "hour":
							factor = time.Hour
						case "days", "day":
							factor = time.Hour * time.Duration(24)
						case "weeks", "week":
							factor = time.Hour * time.Duration(24*7)
						case "months", "month":
							factor = time.Hour * time.Duration(24*30)
						case "years", "year":
							factor = time.Hour * time.Duration(24*365)
						default:
							factor = 0
						}

						if factor != 0 {
							maxage = factor * time.Duration(maxagenum)
							break
						}
					}
				}

				if interactiveMode {
					fmt.Println("Error: Invalid age format.")
				} else {
					log.Fatalf("Invalid maximum age \"%s\".", maxagestring)
				}
			}

			maxtime = time.Now().Add(-maxage)
			if interactiveMode {
				fmt.Println("Okay, let's do it! ")
				fmt.Println("Posts older than", maxtime, "will be deleted!")
				fmt.Print("Loading gun ")
				for i := 0; i < 40; i++ {
					fmt.Print(".")
					time.Sleep(time.Duration(50) * time.Millisecond)
				}
				time.Sleep(time.Duration(2) * time.Second)
				fmt.Print("\n\n")
			} else {
				log.Println("Posts older than", maxtime.Format("Jan 2, 2006 at 3:04:05 PM MST"), "will be deleted!")
			}

			// Get IDs of pinned posts (these won't be deleted)
			if !*quietMode {
				log.Printf("========== Fetching pinned statuses ==========\n")
			}
			params := url.Values{}
			params.Add("pinned", "true")
			resp, fetchErr := hc.Request(http.MethodGet, "/api/v1/accounts/"+strconv.Itoa(accountinfo.ID)+"/statuses", params)
			if fetchErr != nil {
				log.Fatal(fetchErr)
			}
			var pinnedStatuses []Status
			err = json.Unmarshal(resp, &pinnedStatuses)
			if err != nil {
				// Maybe server response is an error message?
				log.Println(string(resp))
				log.Fatal(err)
			}
			pinnedStatusIds := getStatusIds(pinnedStatuses)
			log.Printf("Found %d pinned statuses, which will not be deleted.", len(pinnedStatusIds))

			var maxid uint64 = 0
			var deletedcount uint16

			// Fetch new pages until there are no more pages
			for {
				if !*quietMode {
					log.Printf("========== Fetching new statuses until status %d ==========\n", maxid)
				}

				nodeletions := true

				// Fetch posts
				params := url.Values{}
				params.Add("limit", strconv.Itoa(Pagelimit))
				if maxid != 0 {
					params.Add("max_id", fmt.Sprint(maxid))
				}
				resp, fetchErr := hc.Request(http.MethodGet, "/api/v1/accounts/"+strconv.Itoa(accountinfo.ID)+"/statuses", params)
				if fetchErr != nil {
					log.Fatal(fetchErr)
				}

				var statuses []Status
				err = json.Unmarshal(resp, &statuses)
				if err != nil {
					// Maybe server response is an error message?
					log.Println(string(resp))
					log.Fatal(err)
				}

				// Exit killer loop if there are no more statuses
				if len(statuses) == 0 {
					break
				}

				for _, status := range statuses {
					// Parse time
					if status.CreatedAt.Before(maxtime) {
						if idInSlice(status.ID, pinnedStatusIds) {
							if !*quietMode {
								log.Println("Status " + fmt.Sprint(status.ID) + " is pinned; not deleting.")
							}
							continue
						}

						// Delete post
						nodeletions = false

						if !*dryRun {
							delResp, delErr := hc.Request(http.MethodDelete, "/api/v1/statuses/"+fmt.Sprint(status.ID), nil)
							if delErr != nil {
								log.Println("!!! Could not delete status " + fmt.Sprint(status.ID) + " !!!")
							}

							var delStatus Status
							err = json.Unmarshal(delResp, &delStatus)
							if err != nil {
								log.Println(string(delResp))
								log.Fatal(err)
							}

							if delStatus.ID == status.ID {
								deletedcount++
							} else {
								log.Println("Status " + fmt.Sprint(status.ID) + " could not be deleted :( \nResponse: " + string(delResp))
							}
						}
					}

					if status.ID < maxid || maxid == 0 {
						maxid = status.ID - 1
					}
				}

				if nodeletions {
					if !*quietMode {
						log.Println("No posts to be deleted on this page. Trying next page ...")
					}
				} else {
					if deletedcount == 0 && *dryRun {
						if !*quietMode {
							log.Println("0 statuses deleted, because -dryRun was passed.")
						}
					} else {
						if !*quietMode {
							log.Println(deletedcount, "statuses deleted.")
						}
					}
					// Wait before fetching a new page. Give server time to re-assemble pages.
					time.Sleep(time.Duration(1) * time.Second)
				}
			}

			if deletedcount == 0 && *dryRun {
				log.Println("[dryRun] 0 statuses deleted in total, because -dryRun was passed.")
			}

			// No more pages, done! :-)
			if interactiveMode {
				fmt.Println(">>>>>> ", deletedcount, "statuses were successfully deleted.")
			} else {
				log.Println(deletedcount, "statuses were successfully deleted.")
			}
		}
	}
}
<commit_msg>Fix help text regarding missing --maxage flag<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	Pagelimit = 40
)

/*
 * Helper function to easily read input from console
 */
func readFromConsole() (stringstore string) {
	scanner := bufio.NewScanner(os.Stdin)
	if scanner.Scan() {
		stringstore = scanner.Text()
	}
	return
}

// Given an array of statuses, returns an array of their ids
func getStatusIds(vs []Status) []uint64 {
	vsm := make([]uint64, len(vs))
	for i, v := range vs {
		vsm[i] = v.ID
	}
	return vsm
}

// Returns true if `a` is an element of the array
func idInSlice(a uint64, list []uint64) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

/*
 * Various data structs
 */

type MastoPurgeSettings struct {
	Server       string `json:"server"`
	ClientID     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
	AccessToken  string `json:"access_token"`
}

type AccountInfo struct {
	ID       int    `json:"id,string"`
	Username string `json:"username"`
}

type RespAppRegister struct {
	ClientID     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
}

type RespAccessToken struct {
	AccessToken string `json:"access_token"`
}

type Status struct {
	ID uint64 `json:"id,string"`
	//Content     string  `json:"content"`
	CreatedAt time.Time `json:"created_at"`
}

var (
	noninteractiveMode = flag.Bool("noninteractive", false, "Run in non-interactive mode, suitable for eg. cron jobs. (When run with a missing settings file, the config process will run interactively.)")
	maxAgeArgument     = flag.String("maxage", "", "Max age of posts you want to keep. Required when running in non-interactive mode. Allowed units: hours, days, weeks, months, years. Example: \"6 months\".")
	configFile         = flag.String("config", ".mastopurgesettings", "Path + filename for the settings file.")
	printVersion       = flag.Bool("version", false, "Print version, and exit.")
	quietMode          = flag.Bool("quiet", false, "Reduce output to the most important messages only.")
	dryRun             = flag.Bool("dryrun", false, "Run MastoPurge to preview its results, but without actually deleting any statuses.")
)

var versionString string = "0.0.0"

func main() {
	flag.Parse()

	if *printVersion {
		fmt.Printf("MastoPurge version %s\n", versionString)
		os.Exit(0)
	}

	interactiveMode := !(*noninteractiveMode)

	if interactiveMode && !*quietMode {
		/*
		 * Add fancyness!
		 */
		fmt.Println("\nWelcome to ...")
		fmt.Println("                     _                                   ")
		fmt.Println(" _ __ ___   __ _ ___| |_ ___  _ __  _   _ _ __ __ _  ___ ")
		fmt.Println("| '_ ` _ \\ / _` / __| __/ _ \\| '_ \\| | | | '__/ _` |/ _ \\")
		fmt.Println("| | | | | | (_| \\__ \\ || (_) | |_) | |_| | | | (_| |  __/")
		fmt.Println("|_| |_| |_|\\__,_|___/\\__\\___/| .__/ \\__,_|_|  \\__, |\\___|")
		fmt.Println("                             |_|              |___/      ")
		fmt.Println("    ... add German Datenhygiene to your Mastodon-Account!")
		fmt.Print("\n\n")
		fmt.Printf("Version %s\n\n", versionString)
	}

	/*
	 * Set up settings and Httpclient
	 */
	settings := MastoPurgeSettings{}
	hc := &APIClient{}
	hc.Timeout = time.Second * 5
	hc.Init()

	/*
	 * Check if configuration file (*configFile, default .mastopurgesettings) exists
	 * - Create if it does not exist
	 * - Use if exists
	 */
	configRaw, readerr := ioutil.ReadFile(*configFile)
	if readerr != nil {
		log.Printf("MastoPurge configuration file %s does not exist or is not accessible.\n", *configFile)
		fmt.Println("\nFirst we need to connect MastoPurge to your Mastodon account.")
		fmt.Println("Enter the domain of your Mastodon home instance: (e.g. \"metalhead.club\")")
		fmt.Print("[Mastodon home instance]: ")
		settings.Server = readFromConsole()
		hc.Server = "https://" + settings.Server

		// Register application for user on server
		if !*quietMode {
			log.Println(">>>>>> Registering MastoPurge App on " + settings.Server)
		}
		params := url.Values{}
		params.Add("client_name", "MastoPurge")
		params.Add("redirect_uris", "urn:ietf:wg:oauth:2.0:oob")
		params.Add("scopes", "read write")
		body, registerErr := hc.Request(http.MethodPost, "/api/v1/apps", params)
		if registerErr != nil {
			log.Fatal(registerErr)
		}

		// Parse response JSON
		respAppRegister := RespAppRegister{}
		err := json.Unmarshal(body, &respAppRegister)
		if err != nil {
			log.Fatal(err)
		}
		settings.ClientID = respAppRegister.ClientID
		settings.ClientSecret = respAppRegister.ClientSecret

		// User must manually authenticate app
		authurl := "https://" + settings.Server + "/oauth/authorize?scope=read%20write&response_type=code&redirect_uri=urn:ietf:wg:oauth:2.0:oob&client_id=" + settings.ClientID
		fmt.Println("\nPlease visit this URL in your web browser:")
		fmt.Println(authurl)
		fmt.Println("\n... and enter the code here:")
		fmt.Print("[Auth code]: ")
		code := readFromConsole()

		// Request auth token via auth code ...
		params = url.Values{}
		params.Add("client_id", settings.ClientID)
		params.Add("client_secret", settings.ClientSecret)
		params.Add("grant_type", "authorization_code")
		params.Add("redirect_uri", "urn:ietf:wg:oauth:2.0:oob")
		params.Add("code", code)
		body, err = hc.Request(http.MethodPost, "/oauth/token", params)
		if err != nil {
			log.Fatal(err)
		}

		respAccessToken := RespAccessToken{}
		err = json.Unmarshal(body, &respAccessToken)
		if err != nil {
			log.Fatal(err)
		}
		settings.AccessToken = respAccessToken.AccessToken
		hc.AccessToken = settings.AccessToken

		// Write settings to config file
		configRaw, _ = json.Marshal(settings)
		err = ioutil.WriteFile(*configFile, configRaw, 0600)
		if err != nil {
			log.Fatal(err)
		}

	} else {
		if !*quietMode {
			log.Println("MastoPurge configuration found! Reading config.")
		}

		/*
		 * Load settings
		 */
		err := json.Unmarshal(configRaw, &settings)
		if err != nil {
			log.Fatalf("Config file is malformed :(\nPlease consider deleting '%s' from your file system and starting MastoPurge again.", *configFile)
		}

		hc.Server = "https://" + settings.Server
		hc.AccessToken = settings.AccessToken
	}

	/*
	 * Check if account access is okay
	 */

	if !*quietMode {
		log.Println("Requesting access to Mastodon account")
	}
	body, accessErr := hc.Request(http.MethodGet, "/api/v1/accounts/verify_credentials", nil)
	if accessErr != nil {
		log.Fatal(accessErr)
	} else {
		accountinfo := AccountInfo{}
		err := json.Unmarshal(body, &accountinfo)

		if accountinfo.ID == 0 {
			if !*quietMode {
				log.Println("Access DENIED :-(")
			}
			log.Fatalf("Unfortunately API access was not granted. Consider deleting '%s' and starting MastoPurge again!", *configFile)
		} else {
			if !*quietMode {
				log.Println("Access GRANTED :-)")
				log.Println(">>> Account ID:", accountinfo.ID)
				log.Println(">>> Username:", accountinfo.Username)
			}

			// Do some date calculations ...
			var maxage time.Duration
			var maxtime time.Time

			for {
				maxagestring := *maxAgeArgument
				if maxagestring == "" {
					if interactiveMode {
						fmt.Println("\nEnter the maximum age of the posts you want to KEEP, e.g. \"30 days\". Older posts will be deleted. Allowed units: hours, days, weeks, months, years.")
						fmt.Print("[Maximum post age]: ")
						maxagestring = readFromConsole()
					} else {
						log.Println("missing required argument --maxage")
						flag.PrintDefaults()
						os.Exit(1)
					}
				}

				parts := strings.Split(maxagestring, " ")

				if len(parts) == 2 {
					maxagenum, converr := strconv.Atoi(parts[0])
					if converr == nil {
						var factor time.Duration

						switch parts[1] {
						case "hours", "hour":
							factor = time.Hour
						case "days", "day":
							factor = time.Hour * time.Duration(24)
						case "weeks", "week":
							factor = time.Hour * time.Duration(24*7)
						case "months", "month":
							factor = time.Hour * time.Duration(24*30)
						case "years", "year":
							factor = time.Hour * time.Duration(24*365)
						default:
							factor = 0
						}

						if factor != 0 {
							maxage = factor * time.Duration(maxagenum)
							break
						}
					}
				}

				if interactiveMode {
					fmt.Println("Error: Invalid age format.")
				} else {
					log.Fatalf("Invalid maximum age \"%s\".", maxagestring)
				}
			}

			maxtime = time.Now().Add(-maxage)
			if interactiveMode {
				fmt.Println("Okay, let's do it! ")
				fmt.Println("Posts older than", maxtime, "will be deleted!")
				fmt.Print("Loading gun ")
				for i := 0; i < 40; i++ {
					fmt.Print(".")
					time.Sleep(time.Duration(50) * time.Millisecond)
				}
				time.Sleep(time.Duration(2) * time.Second)
				fmt.Print("\n\n")
			} else {
				log.Println("Posts older than", maxtime.Format("Jan 2, 2006 at 3:04:05 PM MST"), "will be deleted!")
			}

			// Get IDs of pinned posts (these won't be deleted)
			if !*quietMode {
				log.Printf("========== Fetching pinned statuses ==========\n")
			}
			params := url.Values{}
			params.Add("pinned", "true")
			resp, fetchErr := hc.Request(http.MethodGet, "/api/v1/accounts/"+strconv.Itoa(accountinfo.ID)+"/statuses", params)
			if fetchErr != nil {
				log.Fatal(fetchErr)
			}
			var pinnedStatuses []Status
			err = json.Unmarshal(resp, &pinnedStatuses)
			if err != nil {
				// Maybe server response is an error message?
				log.Println(string(resp))
				log.Fatal(err)
			}
			pinnedStatusIds := getStatusIds(pinnedStatuses)
			log.Printf("Found %d pinned statuses, which will not be deleted.", len(pinnedStatusIds))

			var maxid uint64 = 0
			var deletedcount uint16

			// Fetch new pages until there are no more pages
			for {
				if !*quietMode {
					log.Printf("========== Fetching new statuses until status %d ==========\n", maxid)
				}

				nodeletions := true

				// Fetch posts
				params := url.Values{}
				params.Add("limit", strconv.Itoa(Pagelimit))
				if maxid != 0 {
					params.Add("max_id", fmt.Sprint(maxid))
				}
				resp, fetchErr := hc.Request(http.MethodGet, "/api/v1/accounts/"+strconv.Itoa(accountinfo.ID)+"/statuses", params)
				if fetchErr != nil {
					log.Fatal(fetchErr)
				}

				var statuses []Status
				err = json.Unmarshal(resp, &statuses)
				if err != nil {
					// Maybe server response is an error message?
					log.Println(string(resp))
					log.Fatal(err)
				}

				// Exit killer loop if there are no more statuses
				if len(statuses) == 0 {
					break
				}

				for _, status := range statuses {
					// Parse time
					if status.CreatedAt.Before(maxtime) {
						if idInSlice(status.ID, pinnedStatusIds) {
							if !*quietMode {
								log.Println("Status " + fmt.Sprint(status.ID) + " is pinned; not deleting.")
							}
							continue
						}

						// Delete post
						nodeletions = false

						if !*dryRun {
							delResp, delErr := hc.Request(http.MethodDelete, "/api/v1/statuses/"+fmt.Sprint(status.ID), nil)
							if delErr != nil {
								log.Println("!!! Could not delete status " + fmt.Sprint(status.ID) + " !!!")
							}

							var delStatus Status
							err = json.Unmarshal(delResp, &delStatus)
							if err != nil {
								log.Println(string(delResp))
								log.Fatal(err)
							}

							if delStatus.ID == status.ID {
								deletedcount++
							} else {
								log.Println("Status " + fmt.Sprint(status.ID) + " could not be deleted :( \nResponse: " + string(delResp))
							}
						}
					}

					if status.ID < maxid || maxid == 0 {
						maxid = status.ID - 1
					}
				}

				if nodeletions {
					if !*quietMode {
						log.Println("No posts to be deleted on this page. Trying next page ...")
					}
				} else {
					if deletedcount == 0 && *dryRun {
						if !*quietMode {
							log.Println("0 statuses deleted, because -dryRun was passed.")
						}
					} else {
						if !*quietMode {
							log.Println(deletedcount, "statuses deleted.")
						}
					}
					// Wait before fetching a new page. Give server time to re-assemble pages.
					time.Sleep(time.Duration(1) * time.Second)
				}
			}

			if deletedcount == 0 && *dryRun {
				log.Println("[dryRun] 0 statuses deleted in total, because -dryRun was passed.")
			}

			// No more pages, done! :-)
			if interactiveMode {
				fmt.Println(">>>>>> ", deletedcount, "statuses were successfully deleted.")
			} else {
				log.Println(deletedcount, "statuses were successfully deleted.")
			}
		}
	}
}
<|endoftext|>


<commit_before>package common

import "github.com/couchbase/indexing/secondary/collatejson"
import mc "github.com/couchbase/gomemcached/client"
import qexpr "github.com/couchbaselabs/query/expression"
import qparser "github.com/couchbaselabs/query/expression/parser"
import qvalue "github.com/couchbaselabs/query/value"

// Evaluator interface for projector, to be implemented by
// secondary-index or other entities.
type Evaluator interface {
	// Return the bucket name for which this evaluator is applicable.
	Bucket() string

	// StreamBeginData is generated for downstream.
	StreamBeginData(vbno uint16, vbuuid, seqno uint64) (data interface{})

	// Sync is generated for downstream.
	SyncData(vbno uint16, vbuuid, seqno uint64) (data interface{})

	// SnapshotData is generated for downstream.
	SnapshotData(m *mc.UprEvent, vbno uint16, vbuuid, seqno uint64) interface{}

	// StreamEnd is generated for downstream.
	StreamEndData(vbno uint16, vbuuid, seqno uint64) (data interface{})

	// TransformRoute will transform document consumable by
	// downstream, returns data to be published to endpoints.
	TransformRoute(vbuuid uint64, m *mc.UprEvent) (endpoints map[string]interface{}, err error)
}

// CompileN1QLExpression will take expressions defined in N1QL's DDL statement
// and compile them for evaluation.
func CompileN1QLExpression(expressions []string) ([]interface{}, error) {
	cExprs := make([]interface{}, 0, len(expressions))
	for _, expr := range expressions {
		cExpr, err := qparser.Parse(expr)
		if err != nil {
			Errorf("CompileN1QLExpression() %v: %v", expr, err)
			return nil, err
		}
		cExprs = append(cExprs, cExpr)
	}
	return cExprs, nil
}

var missing = string(collatejson.MissingLiteral)

// N1QLTransform will use compile list of expression from N1QL's DDL
// statement and evaluate a document using them to return a secondary
// key as JSON object.
func N1QLTransform(docid, doc []byte, cExprs []interface{}) ([]byte, error) {
	arrValue := make([]qvalue.Value, 0, len(cExprs))
	context := qexpr.NewIndexContext()
	skip := true
	docval := qvalue.NewValueFromBytes(doc)
	for _, cExpr := range cExprs {
		expr := cExpr.(qexpr.Expression)
		key, err := expr.Evaluate(docval, context)
		if err != nil {
			return nil, err

		} else if key.Type() == qvalue.MISSING && skip {
			return nil, nil

		} else if key.Type() == qvalue.MISSING {
			arrValue = append(arrValue, key)
			continue
		}
		skip = false
		arrValue = append(arrValue, key)
	}

	if len(cExprs) == 1 && len(arrValue) == 1 && docid == nil {
		// used for partition-key evaluation and where predicate.
		return arrValue[0].Bytes(), nil

	} else if len(arrValue) > 0 {
		// A strictly internal hack to make secondary keys unique by
		// appending the docid. The shape of the secondary key looks like,
		//     [expr1, docid] - for simple key
		//     [expr1, expr2, ..., docid] - for composite key
		//
		// above hack is applicable only when docid is not `nil`
		if docid != nil {
			arrValue = append(arrValue, qvalue.NewValue(string(docid)))
		}
		secKey := qvalue.NewValue(make([]interface{}, len(arrValue)))
		for i, key := range arrValue {
			secKey.SetIndex(i, key)
		}
		return secKey.Bytes(), nil // return as JSON array
	}
	return nil, nil
}
<commit_msg>Missing literal, a unique string value, is used.<commit_after>package common

import "github.com/couchbase/indexing/secondary/collatejson"
import mc "github.com/couchbase/gomemcached/client"
import qexpr "github.com/couchbaselabs/query/expression"
import qparser "github.com/couchbaselabs/query/expression/parser"
import qvalue "github.com/couchbaselabs/query/value"

// Evaluator interface for projector, to be implemented by
// secondary-index or other entities.
type Evaluator interface {
	// Return the bucket name for which this evaluator is applicable.
	Bucket() string

	// StreamBeginData is generated for downstream.
	StreamBeginData(vbno uint16, vbuuid, seqno uint64) (data interface{})

	// Sync is generated for downstream.
	SyncData(vbno uint16, vbuuid, seqno uint64) (data interface{})

	// SnapshotData is generated for downstream.
	SnapshotData(m *mc.UprEvent, vbno uint16, vbuuid, seqno uint64) interface{}

	// StreamEnd is generated for downstream.
	StreamEndData(vbno uint16, vbuuid, seqno uint64) (data interface{})

	// TransformRoute will transform document consumable by
	// downstream, returns data to be published to endpoints.
	TransformRoute(vbuuid uint64, m *mc.UprEvent) (endpoints map[string]interface{}, err error)
}

// CompileN1QLExpression will take expressions defined in N1QL's DDL statement
// and compile them for evaluation.
func CompileN1QLExpression(expressions []string) ([]interface{}, error) {
	cExprs := make([]interface{}, 0, len(expressions))
	for _, expr := range expressions {
		cExpr, err := qparser.Parse(expr)
		if err != nil {
			Errorf("CompileN1QLExpression() %v: %v", expr, err)
			return nil, err
		}
		cExprs = append(cExprs, cExpr)
	}
	return cExprs, nil
}

var missing = qvalue.NewValue(string(collatejson.MissingLiteral))

// N1QLTransform will use compile list of expression from N1QL's DDL
// statement and evaluate a document using them to return a secondary
// key as JSON object.
func N1QLTransform(docid, doc []byte, cExprs []interface{}) ([]byte, error) {
	arrValue := make([]qvalue.Value, 0, len(cExprs))
	context := qexpr.NewIndexContext()
	skip := true
	docval := qvalue.NewValueFromBytes(doc)
	for _, cExpr := range cExprs {
		expr := cExpr.(qexpr.Expression)
		key, err := expr.Evaluate(docval, context)
		if err != nil {
			return nil, err

		} else if key.Type() == qvalue.MISSING && skip {
			return nil, nil

		} else if key.Type() == qvalue.MISSING {
			arrValue = append(arrValue, missing)
			continue
		}
		skip = false
		arrValue = append(arrValue, key)
	}

	if len(cExprs) == 1 && len(arrValue) == 1 && docid == nil {
		// used for partition-key evaluation and where predicate.
		return arrValue[0].Bytes(), nil

	} else if len(arrValue) > 0 {
		// A strictly internal hack to make secondary keys unique by
		// appending the docid. The shape of the secondary key looks like,
		//     [expr1, docid] - for simple key
		//     [expr1, expr2, ..., docid] - for composite key
		//
		// above hack is applicable only when docid is not `nil`
		if docid != nil {
			arrValue = append(arrValue, qvalue.NewValue(string(docid)))
		}
		secKey := qvalue.NewValue(make([]interface{}, len(arrValue)))
		for i, key := range arrValue {
			secKey.SetIndex(i, key)
		}
		return secKey.Bytes(), nil // return as JSON array
	}
	return nil, nil
}
<|endoftext|>