<commit_before>package flickr

import (
	"io"
	"os"
	"fmt"
	"net"
	"http"
	"sort"
	"bufio"
	"bytes"
	"io/ioutil"
	"crypto/md5"
)

const (
	endpoint        = "http://api.flickr.com/services/rest/?"
	upload_endpoint = "http://api.flickr.com/services/upload/"
	api_host        = "api.flickr.com"
)

type Request struct {
	ApiKey string
	Method string
	Args   map[string]string
}

type nopCloser struct {
	io.Reader
}

func (nopCloser) Close() os.Error { return nil }

// So we can return custom errors
type Error string

func (e Error) String() string {
	return string(e)
}

func (request *Request) Sign(secret string) {
	args := request.Args

	sorted_keys := make([]string, len(args)+2)

	args["api_key"] = request.ApiKey
	args["method"] = request.Method

	// Sort array keys
	i := 0
	for k, _ := range args {
		sorted_keys[i] = k
		i++
	}
	sort.SortStrings(sorted_keys)

	// Build out ordered key-value string prefixed by secret
	s := secret
	for _, key := range sorted_keys {
		if args[key] != "" {
			s += fmt.Sprintf("%s%s", key, args[key])
		}
	}

	// Since we're only adding two keys, it's easier
	// and more space-efficient to just delete them
	// them copy the whole map
	args["api_key"] = "", false
	args["method"] = "", false

	// Have the full string, now hash
	hash := md5.New()
	hash.Write([]byte(s))

	// Add api_sig as one of the args
	args["api_sig"] = fmt.Sprintf("%x", hash.Sum())
}

func (request *Request) Execute() (response string, ret os.Error) {
	args := request.Args

	if request.ApiKey == "" || request.Method == "" {
		return "", Error("Need both API key and method")
	}

	args["api_key"] = request.ApiKey
	args["method"] = request.Method

	s := endpoint + encodeQuery(args)

	res, _, err := http.Get(s)
	defer res.Body.Close()
	if err != nil {
		return "", err
	}

	body, _ := ioutil.ReadAll(res.Body)
	return string(body), nil
}

func encodeQuery(args map[string]string) string {
	s := ""
	i := 0
	for k, v := range args {
		if i != 0 {
			s += "&"
		}
		i++
		s += fmt.Sprintf("%s=%s", k, http.URLEscape(v))
	}
	return s
}

func (request *Request) Upload(filename string) (response string, err os.Error) {
	photo_file, error := ioutil.ReadFile(filename)
	if error != nil {
		return "", error
	}

	request.Args["api_key"] = request.ApiKey

	boundary := "----###---###--flickr-go-rules"
	end := "\r\n"

	post_body := ""
	for k, v := range request.Args {
		post_body += "--" + boundary + end
		post_body += "Content-Disposition: form-data; name=\"" + k + "\"" + end + end
		post_body += v + end
	}

	post_body += "--" + boundary + end
	post_body += "Content-Disposition: form-data; name=\"photo\"; filename=\"photo.jpg\"" + end
	post_body += "Content-Type: image/jpeg" + end + end
	post_body += string(photo_file) + end
	post_body += "--" + boundary + "--" + end

	post_req := new(http.Request)
	post_req.Method = "POST"
	post_req.RawURL = upload_endpoint
	post_req.Host = api_host
	post_req.Header = map[string]string{
		"Content-Type": "multipart/form-data; boundary=" + boundary + end,
	}

	post_req.Body = nopCloser{bytes.NewBufferString(post_body)}
	post_req.ContentLength = int64(len(post_body))

	// Create and use TCP connection
	conn, err := net.Dial("tcp", "", "api.flickr.com:80")
	defer conn.Close()

	if err != nil {
		return "", err
	}
	post_req.Write(conn)

	reader := bufio.NewReader(conn)
	resp, err := http.ReadResponse(reader, post_req.Method)
	if err != nil {
		return "", err
	}
	body, _ := ioutil.ReadAll(resp.Body)

	return string(body), nil
}
<commit_msg>Added code attribution for TCP talkin'<commit_after>package flickr

import (
	"io"
	"os"
	"fmt"
	"net"
	"http"
	"sort"
	"bufio"
	"bytes"
	"io/ioutil"
	"crypto/md5"
)

const (
	endpoint        = "http://api.flickr.com/services/rest/?"
	upload_endpoint = "http://api.flickr.com/services/upload/"
	api_host        = "api.flickr.com"
)

type Request struct {
	ApiKey string
	Method string
	Args   map[string]string
}

type nopCloser struct {
	io.Reader
}

func (nopCloser) Close() os.Error { return nil }

// So we can return custom errors
type Error string

func (e Error) String() string {
	return string(e)
}

func (request *Request) Sign(secret string) {
	args := request.Args

	sorted_keys := make([]string, len(args)+2)

	args["api_key"] = request.ApiKey
	args["method"] = request.Method

	// Sort array keys
	i := 0
	for k, _ := range args {
		sorted_keys[i] = k
		i++
	}
	sort.SortStrings(sorted_keys)

	// Build out ordered key-value string prefixed by secret
	s := secret
	for _, key := range sorted_keys {
		if args[key] != "" {
			s += fmt.Sprintf("%s%s", key, args[key])
		}
	}

	// Since we're only adding two keys, it's easier
	// and more space-efficient to just delete them
	// them copy the whole map
	args["api_key"] = "", false
	args["method"] = "", false

	// Have the full string, now hash
	hash := md5.New()
	hash.Write([]byte(s))

	// Add api_sig as one of the args
	args["api_sig"] = fmt.Sprintf("%x", hash.Sum())
}

func (request *Request) Execute() (response string, ret os.Error) {
	args := request.Args

	if request.ApiKey == "" || request.Method == "" {
		return "", Error("Need both API key and method")
	}

	args["api_key"] = request.ApiKey
	args["method"] = request.Method

	s := endpoint + encodeQuery(args)

	res, _, err := http.Get(s)
	defer res.Body.Close()
	if err != nil {
		return "", err
	}

	body, _ := ioutil.ReadAll(res.Body)
	return string(body), nil
}

func encodeQuery(args map[string]string) string {
	s := ""
	i := 0
	for k, v := range args {
		if i != 0 {
			s += "&"
		}
		i++
		s += fmt.Sprintf("%s=%s", k, http.URLEscape(v))
	}
	return s
}

func (request *Request) Upload(filename string) (response string, err os.Error) {
	photo_file, error := ioutil.ReadFile(filename)
	if error != nil {
		return "", error
	}

	request.Args["api_key"] = request.ApiKey

	boundary := "----###---###--flickr-go-rules"
	end := "\r\n"

	post_body := ""
	for k, v := range request.Args {
		post_body += "--" + boundary + end
		post_body += "Content-Disposition: form-data; name=\"" + k + "\"" + end + end
		post_body += v + end
	}

	post_body += "--" + boundary + end
	post_body += "Content-Disposition: form-data; name=\"photo\"; filename=\"photo.jpg\"" + end
	post_body += "Content-Type: image/jpeg" + end + end
	post_body += string(photo_file) + end
	post_body += "--" + boundary + "--" + end

	post_req := new(http.Request)
	post_req.Method = "POST"
	post_req.RawURL = upload_endpoint
	post_req.Host = api_host
	post_req.Header = map[string]string{
		"Content-Type": "multipart/form-data; boundary=" + boundary + end,
	}

	post_req.Body = nopCloser{bytes.NewBufferString(post_body)}
	post_req.ContentLength = int64(len(post_body))

	// Create and use TCP connection (lifted mostly wholesale from http.send)
	conn, err := net.Dial("tcp", "", "api.flickr.com:80")
	defer conn.Close()

	if err != nil {
		return "", err
	}
	post_req.Write(conn)

	reader := bufio.NewReader(conn)
	resp, err := http.ReadResponse(reader, post_req.Method)
	if err != nil {
		return "", err
	}
	body, _ := ioutil.ReadAll(resp.Body)

	return string(body), nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The following enables go generate to generate the doc.go file.
//go:generate go run $JIRI_ROOT/release/go/src/v.io/x/lib/cmdline/testdata/gendoc.go -env=CMDLINE_PREFIX=jiri .

package main

import (
	"flag"
	"fmt"
	"path/filepath"
	"strings"

	"v.io/jiri/jiri"
	"v.io/jiri/profiles"
	"v.io/jiri/tool"
	"v.io/x/devtools/jiri-v23-profile/v23_profile"
	"v.io/x/lib/cmdline"
)

var (
	manifestFlag      string
	verboseFlag       bool
	mergePoliciesFlag profiles.MergePolicies
)

func init() {
	mergePoliciesFlag = profiles.JiriMergePolicies()
	profiles.RegisterMergePoliciesFlag(&cmdGoExt.Flags, &mergePoliciesFlag)
	profiles.RegisterManifestFlag(&cmdGoExt.Flags, &manifestFlag, v23_profile.DefaultManifestFilename)
	flag.BoolVar(&verboseFlag, "v", false, "print verbose debugging information")
	tool.InitializeRunFlags(&cmdGoExt.Flags)
}

// cmdGoExt represents the "jiri goext" command.
var cmdGoExt = &cmdline.Command{
	Name:     "goext",
	Short:    "Vanadium extensions of the go tool",
	Long:     "Vanadium extensions of the go tool.",
	Children: []*cmdline.Command{cmdGoExtDistClean},
}

// cmdGoExtDistClean represents the "jiri goext distclean" command.
var cmdGoExtDistClean = &cmdline.Command{
	Runner: jiri.RunnerFunc(runGoExtDistClean),
	Name:   "distclean",
	Short:  "Restore the vanadium Go workspaces to their pristine state",
	Long: `
Unlike the 'go clean' command, which only removes object files for
packages in the source tree, the 'goext disclean' command removes all
object files from vanadium Go workspaces. This functionality is needed
to avoid accidental use of stale object files that correspond to
packages that no longer exist in the source tree.
`,
}

func runGoExtDistClean(jirix *jiri.X, _ []string) error {
	ch, err := profiles.NewConfigHelper(jirix, profiles.UseProfiles, manifestFlag)
	if err != nil {
		return err
	}
	ch.MergeEnvFromProfiles(mergePoliciesFlag, profiles.NativeTarget(), "jiri")
	failed := false
	if verboseFlag {
		fmt.Fprintf(jirix.Stdout(), "GOPATH:\n%s\n", strings.Join(ch.GetTokens("GOPATH", ":"), "\n"))
		fmt.Fprintf(jirix.Stdout(), "Jiri Root: %v\n", ch.Root())
	}
	for _, workspace := range ch.GetTokens("GOPATH", ":") {
		if !strings.HasPrefix(workspace, ch.Root()) {
			continue
		}
		for _, name := range []string{"bin", "pkg"} {
			dir := filepath.Join(workspace, name)
			if verboseFlag {
				fmt.Fprintf(jirix.Stdout(), "removing: %s\n", dir)
			}
			if err := jirix.Run().RemoveAll(dir); err != nil {
				failed = true
			}
		}
	}
	if failed {
		return cmdline.ErrExitCode(2)
	}
	return nil
}

func main() {
	cmdline.Main(cmdGoExt)
}
<commit_msg>v.io/x/devtools/jiri-goext: use runutil.Sequence.<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The following enables go generate to generate the doc.go file.
//go:generate go run $JIRI_ROOT/release/go/src/v.io/x/lib/cmdline/testdata/gendoc.go -env=CMDLINE_PREFIX=jiri .

package main

import (
	"flag"
	"fmt"
	"path/filepath"
	"strings"

	"v.io/jiri/jiri"
	"v.io/jiri/profiles"
	"v.io/jiri/tool"
	"v.io/x/devtools/jiri-v23-profile/v23_profile"
	"v.io/x/lib/cmdline"
)

var (
	manifestFlag      string
	verboseFlag       bool
	mergePoliciesFlag profiles.MergePolicies
)

func init() {
	mergePoliciesFlag = profiles.JiriMergePolicies()
	profiles.RegisterMergePoliciesFlag(&cmdGoExt.Flags, &mergePoliciesFlag)
	profiles.RegisterManifestFlag(&cmdGoExt.Flags, &manifestFlag, v23_profile.DefaultManifestFilename)
	flag.BoolVar(&verboseFlag, "v", false, "print verbose debugging information")
	tool.InitializeRunFlags(&cmdGoExt.Flags)
}

// cmdGoExt represents the "jiri goext" command.
var cmdGoExt = &cmdline.Command{
	Name:     "goext",
	Short:    "Vanadium extensions of the go tool",
	Long:     "Vanadium extensions of the go tool.",
	Children: []*cmdline.Command{cmdGoExtDistClean},
}

// cmdGoExtDistClean represents the "jiri goext distclean" command.
var cmdGoExtDistClean = &cmdline.Command{
	Runner: jiri.RunnerFunc(runGoExtDistClean),
	Name:   "distclean",
	Short:  "Restore the vanadium Go workspaces to their pristine state",
	Long: `
Unlike the 'go clean' command, which only removes object files for
packages in the source tree, the 'goext disclean' command removes all
object files from vanadium Go workspaces. This functionality is needed
to avoid accidental use of stale object files that correspond to
packages that no longer exist in the source tree.
`,
}

func runGoExtDistClean(jirix *jiri.X, _ []string) error {
	ch, err := profiles.NewConfigHelper(jirix, profiles.UseProfiles, manifestFlag)
	if err != nil {
		return err
	}
	ch.MergeEnvFromProfiles(mergePoliciesFlag, profiles.NativeTarget(), "jiri")
	failed := false
	if verboseFlag {
		fmt.Fprintf(jirix.Stdout(), "GOPATH:\n%s\n", strings.Join(ch.GetTokens("GOPATH", ":"), "\n"))
		fmt.Fprintf(jirix.Stdout(), "Jiri Root: %v\n", ch.Root())
	}
	for _, workspace := range ch.GetTokens("GOPATH", ":") {
		if !strings.HasPrefix(workspace, ch.Root()) {
			continue
		}
		for _, name := range []string{"bin", "pkg"} {
			dir := filepath.Join(workspace, name)
			if verboseFlag {
				fmt.Fprintf(jirix.Stdout(), "removing: %s\n", dir)
			}
			if err := jirix.NewSeq().RemoveAll(dir).Done(); err != nil {
				failed = true
			}
		}
	}
	if failed {
		return cmdline.ErrExitCode(2)
	}
	return nil
}

func main() {
	cmdline.Main(cmdGoExt)
}
<|endoftext|>


<commit_before>package flickr

import (
	"io"
	"os"
	"fmt"
	"net"
	"http"
	"sort"
	"bufio"
	"bytes"
	"io/ioutil"
	"crypto/md5"
)

const (
	endpoint        = "http://api.flickr.com/services/rest/?"
	uploadEndpoint  = "http://api.flickr.com/services/upload/"
	replaceEndpoint = "http://api.flickr.com/services/replace/"
	apiHost         = "api.flickr.com"
)

type Request struct {
	ApiKey string
	Method string
	Args   map[string]string
}

type nopCloser struct {
	io.Reader
}

func (nopCloser) Close() os.Error { return nil }

type Error string

func (e Error) String() string {
	return string(e)
}

func (request *Request) Sign(secret string) {
	args := request.Args

	// Remove api_sig
	args["api_sig"] = "", false

	sorted_keys := make([]string, len(args)+2)

	args["api_key"] = request.ApiKey
	args["method"] = request.Method

	// Sort array keys
	i := 0
	for k := range args {
		sorted_keys[i] = k
		i++
	}
	sort.SortStrings(sorted_keys)

	// Build out ordered key-value string prefixed by secret
	s := secret
	for _, key := range sorted_keys {
		if args[key] != "" {
			s += fmt.Sprintf("%s%s", key, args[key])
		}
	}

	// Since we're only adding two keys, it's easier
	// and more space-efficient to just delete them
	// them copy the whole map
	args["api_key"] = "", false
	args["method"] = "", false

	// Have the full string, now hash
	hash := md5.New()
	hash.Write([]byte(s))

	// Add api_sig as one of the args
	args["api_sig"] = fmt.Sprintf("%x", hash.Sum())
}

func (request *Request) URL() string {
	args := request.Args

	args["api_key"] = request.ApiKey
	args["method"] = request.Method

	s := endpoint + encodeQuery(args)
	return s
}

func (request *Request) Execute() (response string, ret os.Error) {
	if request.ApiKey == "" || request.Method == "" {
		return "", Error("Need both API key and method")
	}

	s := request.URL()

	res, _, err := http.Get(s)
	defer res.Body.Close()
	if err != nil {
		return "", err
	}

	body, _ := ioutil.ReadAll(res.Body)
	return string(body), nil
}

func encodeQuery(args map[string]string) string {
	i := 0
	s := bytes.NewBuffer(nil)
	for k, v := range args {
		if i != 0 {
			s.WriteString("&")
		}
		i++
		s.WriteString(k + "=" + http.URLEscape(v))
	}
	return s.String()
}

func (request *Request) buildPost(url string, filename string, filetype string) (*http.Request, os.Error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	stat, err := f.Stat()
	if err != nil {
		return nil, err
	}
	f_size := stat.Size

	request.Args["api_key"] = request.ApiKey

	boundary, end := "----###---###--flickr-go-rules", "\r\n"

	// Build out all of POST body sans file
	header := bytes.NewBuffer(nil)
	for k, v := range request.Args {
		header.WriteString("--" + boundary + end)
		header.WriteString("Content-Disposition: form-data; name=\"" + k + "\"" + end + end)
		header.WriteString(v + end)
	}
	header.WriteString("--" + boundary + end)
	header.WriteString("Content-Disposition: form-data; name=\"photo\"; filename=\"photo.jpg\"" + end)
	header.WriteString("Content-Type: " + filetype + end + end)

	footer := bytes.NewBufferString(end + "--" + boundary + "--" + end)

	body_len := int64(header.Len()) + int64(footer.Len()) + f_size

	r, w := io.Pipe()
	go func() {
		pieces := []io.Reader{header, f, footer}

		for _, k := range pieces {
			_, err = io.Copy(w, k)
			if err != nil {
				w.CloseWithError(nil)
				return
			}
		}
		f.Close()
		w.Close()
	}()

	http_header := make(http.Header)
	http_header.Add("Content-Type", "multipart/form-data; boundary="+boundary)

	postRequest := &http.Request{
		Method:        "POST",
		RawURL:        url,
		Host:          apiHost,
		Header:        http_header,
		Body:          r,
		ContentLength: body_len,
	}
	return postRequest, nil
}

// Example:
// r.Upload("thumb.jpg", "image/jpeg")
func (request *Request) Upload(filename string, filetype string) (response string, err os.Error) {
	postRequest, err := request.buildPost(uploadEndpoint, filename, filetype)
	if err != nil {
		return "", err
	}

	return sendPost(postRequest)
}

func (request *Request) Replace(filename string, filetype string) (response string, err os.Error) {
	postRequest, err := request.buildPost(replaceEndpoint, filename, filetype)
	if err != nil {
		return "", err
	}
	return sendPost(postRequest)
}

func sendPost(postRequest *http.Request) (body string, err os.Error) {
	// Create and use TCP connection (lifted mostly wholesale from http.send)
	conn, err := net.Dial("tcp", "api.flickr.com:80")
	defer conn.Close()

	if err != nil {
		return "", err
	}
	postRequest.Write(conn)

	reader := bufio.NewReader(conn)
	resp, err := http.ReadResponse(reader, postRequest.Method)
	if err != nil {
		return "", err
	}
	rawBody, _ := ioutil.ReadAll(resp.Body)

	return string(rawBody), nil
}
<commit_msg>Ran gofix<commit_after>package flickr

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"fmt"
	"http"
	"io"
	"io/ioutil"
	"net"
	"os"
	"sort"
	"url"
)

const (
	endpoint        = "http://api.flickr.com/services/rest/?"
	uploadEndpoint  = "http://api.flickr.com/services/upload/"
	replaceEndpoint = "http://api.flickr.com/services/replace/"
	apiHost         = "api.flickr.com"
)

type Request struct {
	ApiKey string
	Method string
	Args   map[string]string
}

type nopCloser struct {
	io.Reader
}

func (nopCloser) Close() os.Error { return nil }

type Error string

func (e Error) String() string {
	return string(e)
}

func (request *Request) Sign(secret string) {
	args := request.Args

	// Remove api_sig
	args["api_sig"] = "", false

	sorted_keys := make([]string, len(args)+2)

	args["api_key"] = request.ApiKey
	args["method"] = request.Method

	// Sort array keys
	i := 0
	for k := range args {
		sorted_keys[i] = k
		i++
	}
	sort.Strings(sorted_keys)

	// Build out ordered key-value string prefixed by secret
	s := secret
	for _, key := range sorted_keys {
		if args[key] != "" {
			s += fmt.Sprintf("%s%s", key, args[key])
		}
	}

	// Since we're only adding two keys, it's easier
	// and more space-efficient to just delete them
	// them copy the whole map
	args["api_key"] = "", false
	args["method"] = "", false

	// Have the full string, now hash
	hash := md5.New()
	hash.Write([]byte(s))

	// Add api_sig as one of the args
	args["api_sig"] = fmt.Sprintf("%x", hash.Sum())
}

func (request *Request) URL() string {
	args := request.Args

	args["api_key"] = request.ApiKey
	args["method"] = request.Method

	s := endpoint + encodeQuery(args)
	return s
}

func (request *Request) Execute() (response string, ret os.Error) {
	if request.ApiKey == "" || request.Method == "" {
		return "", Error("Need both API key and method")
	}

	s := request.URL()

	res, err := http.Get(s)
	defer res.Body.Close()
	if err != nil {
		return "", err
	}

	body, _ := ioutil.ReadAll(res.Body)
	return string(body), nil
}

func encodeQuery(args map[string]string) string {
	i := 0
	s := bytes.NewBuffer(nil)
	for k, v := range args {
		if i != 0 {
			s.WriteString("&")
		}
		i++
		s.WriteString(k + "=" + url.QueryEscape(v))
	}
	return s.String()
}

func (request *Request) buildPost(url_ string, filename string, filetype string) (*http.Request, os.Error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	stat, err := f.Stat()
	if err != nil {
		return nil, err
	}
	f_size := stat.Size

	request.Args["api_key"] = request.ApiKey

	boundary, end := "----###---###--flickr-go-rules", "\r\n"

	// Build out all of POST body sans file
	header := bytes.NewBuffer(nil)
	for k, v := range request.Args {
		header.WriteString("--" + boundary + end)
		header.WriteString("Content-Disposition: form-data; name=\"" + k + "\"" + end + end)
		header.WriteString(v + end)
	}
	header.WriteString("--" + boundary + end)
	header.WriteString("Content-Disposition: form-data; name=\"photo\"; filename=\"photo.jpg\"" + end)
	header.WriteString("Content-Type: " + filetype + end + end)

	footer := bytes.NewBufferString(end + "--" + boundary + "--" + end)

	body_len := int64(header.Len()) + int64(footer.Len()) + f_size

	r, w := io.Pipe()
	go func() {
		pieces := []io.Reader{header, f, footer}

		for _, k := range pieces {
			_, err = io.Copy(w, k)
			if err != nil {
				w.CloseWithError(nil)
				return
			}
		}
		f.Close()
		w.Close()
	}()

	http_header := make(http.Header)
	http_header.Add("Content-Type", "multipart/form-data; boundary="+boundary)

	postRequest := &http.Request{
		Method:        "POST",
		RawURL:        url_,
		Host:          apiHost,
		Header:        http_header,
		Body:          r,
		ContentLength: body_len,
	}
	return postRequest, nil
}

// Example:
// r.Upload("thumb.jpg", "image/jpeg")
func (request *Request) Upload(filename string, filetype string) (response string, err os.Error) {
	postRequest, err := request.buildPost(uploadEndpoint, filename, filetype)
	if err != nil {
		return "", err
	}

	return sendPost(postRequest)
}

func (request *Request) Replace(filename string, filetype string) (response string, err os.Error) {
	postRequest, err := request.buildPost(replaceEndpoint, filename, filetype)
	if err != nil {
		return "", err
	}
	return sendPost(postRequest)
}

func sendPost(postRequest *http.Request) (body string, err os.Error) {
	// Create and use TCP connection (lifted mostly wholesale from http.send)
	conn, err := net.Dial("tcp", "api.flickr.com:80")
	defer conn.Close()

	if err != nil {
		return "", err
	}
	postRequest.Write(conn)

	reader := bufio.NewReader(conn)
	resp, err := http.ReadResponse(reader, postRequest)
	if err != nil {
		return "", err
	}
	rawBody, _ := ioutil.ReadAll(resp.Body)

	return string(rawBody), nil
}
<|endoftext|>


<commit_before>package middleware

import (
	"github.com/gin-gonic/gin"
	"strings"

	"github.com/eirka/eirka-libs/redis"
)

var (
	expire        uint = 600
	RedisKeyIndex      = make(map[string]RedisKey)
	RedisKeys          = []RedisKey{
		{base: "index", fieldcount: 1, hash: true, expire: false},
		{base: "image", fieldcount: 1, hash: true, expire: false},
		{base: "tags", fieldcount: 1, hash: true, expire: false},
		{base: "tag", fieldcount: 2, hash: true, expire: true},
		{base: "thread", fieldcount: 2, hash: true, expire: false},
		{base: "post", fieldcount: 2, hash: true, expire: false},
		{base: "directory", fieldcount: 1, hash: false, expire: false},
		{base: "favorited", fieldcount: 1, hash: false, expire: true},
		{base: "new", fieldcount: 1, hash: false, expire: true},
		{base: "popular", fieldcount: 1, hash: false, expire: true},
		{base: "imageboards", fieldcount: 1, hash: false, expire: true},
	}
	cache = redis.RedisCache
)

func init() {
	// key index map
	for _, key := range RedisKeys {
		RedisKeyIndex[key.base] = key
	}
}

// Cache will check for the key in Redis and serve it. If not found, it will
// take the marshalled JSON from the controller and set it in Redis
func Cache() gin.HandlerFunc {
	return func(c *gin.Context) {
		var result []byte
		var err error

		// bool for analytics middleware
		c.Set("cached", false)

		// break cache if there is a query
		if c.Request.URL.RawQuery != "" {
			c.Next()
			return
		}

		// Trim leading / from path and split
		params := strings.Split(strings.Trim(c.Request.URL.Path, "/"), "/")

		// get the keyname
		key := RedisKeyIndex[params[0]]
		if key == nil {
			c.Next()
			return
		}

		// set the key minus the base
		key.SetKey(params[1:]...)

		result, err = key.Get()
		if err == redis.ErrCacheMiss {
			// go to the controller
			c.Next()

			// Check if there was an error from the controller
			_, controllerError := c.Get("controllerError")
			if controllerError {
				c.Abort()
				return
			}

			err = key.Set(c.MustGet("data").([]byte))
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

		} else if err != nil {
			c.Error(err)
			c.Abort()
			return
		}

		// if we made it this far then the page was cached
		c.Set("cached", true)

		c.Writer.Header().Set("Content-Type", "application/json")
		c.Writer.Write(result)
		c.Abort()
		return
	}

}

type RedisKey struct {
	base       string
	fieldcount int
	hash       bool
	expire     bool
	key        string
	hashid     string
}

func (r *RedisKey) SetKey(ids ...string) {

	// create our key
	r.key = strings.Join([]string{r.base, strings.Join(ids[:r.fieldcount], ":")}, ":")

	// get our hash id
	if r.hash {
		r.hashid = ids[r.fieldcount:]
	}

	return
}

func (r *RedisKey) Get() (result []byte, err error) {

	if r.hash {
		return cache.HGet(r.key, r.hashid)
	} else {
		return cache.Get(r.key)
	}

	return
}

func (r *RedisKey) Set(data []byte) (err error) {

	if r.hash {
		return cache.HMSet(r.key, r.hashid, data)
	} else {
		if r.expire {
			return cache.SetEx(r.key, expire, data)
		} else {
			return cache.Set(r.key, data)
		}
	}

	return
}
<commit_msg>use explicit keys<commit_after>package middleware

import (
	"github.com/gin-gonic/gin"
	"strings"

	"github.com/eirka/eirka-libs/redis"
)

var (
	expire        uint = 600
	cache              = redis.RedisCache
	RedisKeyIndex      = make(map[string]RedisKey)
	RedisKeys          = []RedisKey{
		{base: "index", fieldcount: 1, hash: true, expire: false},
		{base: "image", fieldcount: 1, hash: true, expire: false},
		{base: "tags", fieldcount: 1, hash: true, expire: false},
		{base: "tag", fieldcount: 2, hash: true, expire: true},
		{base: "thread", fieldcount: 2, hash: true, expire: false},
		{base: "post", fieldcount: 2, hash: true, expire: false},
		{base: "directory", fieldcount: 1, hash: false, expire: false},
		{base: "favorited", fieldcount: 1, hash: false, expire: true},
		{base: "new", fieldcount: 1, hash: false, expire: true},
		{base: "popular", fieldcount: 1, hash: false, expire: true},
		{base: "imageboards", fieldcount: 1, hash: false, expire: true},
	}
)

func init() {
	// key index map
	for _, key := range RedisKeys {
		RedisKeyIndex[key.base] = key
	}
}

// Cache will check for the key in Redis and serve it. If not found, it will
// take the marshalled JSON from the controller and set it in Redis
func Cache() gin.HandlerFunc {
	return func(c *gin.Context) {
		var result []byte
		var err error

		// bool for analytics middleware
		c.Set("cached", false)

		// break cache if there is a query
		if c.Request.URL.RawQuery != "" {
			c.Next()
			return
		}

		// Trim leading / from path and split
		params := strings.Split(strings.Trim(c.Request.URL.Path, "/"), "/")

		// get the keyname
		key, ok := RedisKeyIndex[params[0]]
		if !ok {
			c.Next()
			return
		}

		// set the key minus the base
		key.SetKey(params[1:]...)

		result, err = key.Get()
		if err == redis.ErrCacheMiss {
			// go to the controller
			c.Next()

			// Check if there was an error from the controller
			_, controllerError := c.Get("controllerError")
			if controllerError {
				c.Abort()
				return
			}

			err = key.Set(c.MustGet("data").([]byte))
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

		} else if err != nil {
			c.Error(err)
			c.Abort()
			return
		}

		// if we made it this far then the page was cached
		c.Set("cached", true)

		c.Writer.Header().Set("Content-Type", "application/json")
		c.Writer.Write(result)
		c.Abort()
		return
	}

}

type RedisKey struct {
	base       string
	fieldcount int
	hash       bool
	expire     bool
	key        string
	hashid     string
}

func (r *RedisKey) SetKey(ids ...string) {

	// create our key
	r.key = strings.Join([]string{r.base, strings.Join(ids[:r.fieldcount], ":")}, ":")

	// get our hash id
	if r.hash {
		r.hashid = ids[r.fieldcount:]
	}

	return
}

func (r *RedisKey) Get() (result []byte, err error) {

	if r.hash {
		return cache.HGet(r.key, r.hashid)
	} else {
		return cache.Get(r.key)
	}

	return
}

func (r *RedisKey) Set(data []byte) (err error) {

	if r.hash {
		return cache.HMSet(r.key, r.hashid, data)
	} else {
		if r.expire {
			return cache.SetEx(r.key, expire, data)
		} else {
			return cache.Set(r.key, data)
		}
	}

	return
}
<|endoftext|>


<commit_before>/*
Package framed provides an implementations of io.Writer and io.Reader that write
and read whole frames only.

Frames are length-prefixed.  The first two bytes are an unsigned 16 bit int
stored in little-endian byte order indicating the length of the content.  The
remaining bytes are the actual content of the frame.

The use of a uint16 means that the maximum possible frame size (MaxFrameSize)
is 65535.
*/
package framed

import (
	"encoding/binary"
	"fmt"
	"io"
	"sync"
)

const (
	// FrameHeaderBits is the size of the frame header in bits
	FrameHeaderBits = 16

	// FrameHeaderLength is the size of the frame header in bytes
	FrameHeaderLength = FrameHeaderBits / 8

	// MaxFrameLength is the maximum possible size of a frame (not including the
	// length prefix)
	MaxFrameLength = 1<<FrameHeaderBits - 1

	tooLongError = "Attempted to write frame of length %s which is longer than maximum allowed length of %s"
)

var endianness = binary.LittleEndian

/*
A Reader enhances an io.ReadCloser to read data in contiguous frames. It
implements the io.Reader interface, but unlike typical io.Readers it only
returns whole frames.

A Reader also supports the ability to read frames using dynamically allocated
buffers via the ReadFrame method.
*/
type Reader struct {
	Stream io.Reader // the raw underlying connection
	mutex  sync.Mutex
}

/*
A Writer enhances an io.WriteCLoser to write data in contiguous frames. It
implements the io.Writer interface, but unlike typical io.Writers, it includes
information that allows a corresponding Reader to read whole frames without them
being fragmented.

A Writer also supports a method that writes multiple buffers to the underlying
stream as a single frame.
*/
type Writer struct {
	Stream io.Writer // the raw underlying connection
	mutex  sync.Mutex
}

func NewReader(r io.Reader) *Reader {
	return &Reader{Stream: r}
}

func NewWriter(w io.Writer) *Writer {
	return &Writer{Stream: w}
}

/*
Read implements the function from io.Reader.  Unlike io.Reader.Read,
frame.Read only returns full frames of data (assuming that the data was written
by a framed.Writer).
*/
func (framed *Reader) Read(buffer []byte) (n int, err error) {
	framed.mutex.Lock()
	defer framed.mutex.Unlock()

	var nb uint16
	err = binary.Read(framed.Stream, endianness, &nb)
	if err != nil {
		return
	}

	n = int(nb)

	bufferSize := len(buffer)
	if n > bufferSize {
		return 0, fmt.Errorf("Buffer of size %d is too small to hold frame of size %d", bufferSize, n)
	}

	// Read into buffer
	n, err = io.ReadFull(framed.Stream, buffer[:n])
	return
}

// ReadFrame reads the next frame, using a new buffer sized to hold the frame.
func (framed *Reader) ReadFrame() (frame []byte, err error) {
	framed.mutex.Lock()
	defer framed.mutex.Unlock()

	var nb uint16
	err = binary.Read(framed.Stream, endianness, &nb)
	if err != nil {
		return
	}

	l := int(nb)
	frame = make([]byte, l)

	// Read into buffer
	_, err = io.ReadFull(framed.Stream, frame)
	return
}

/*
Write implements the Write method from io.Writer.  It prepends a frame length
header that allows the framed.Reader on the other end to read the whole frame.
*/
func (framed *Writer) Write(frame []byte) (n int, err error) {
	framed.mutex.Lock()
	defer framed.mutex.Unlock()

	n = len(frame)
	if n > MaxFrameLength {
		return 0, fmt.Errorf(tooLongError, n, MaxFrameLength)
	}

	// Write the length header
	if err = binary.Write(framed.Stream, endianness, uint16(n)); err != nil {
		return
	}

	// Write the data
	var written int
	if written, err = framed.Stream.Write(frame); err != nil {
		return
	}
	if written != n {
		err = fmt.Errorf("%d bytes written, expected to write %d", written, n)
	}
	return
}

func (framed *Writer) WritePieces(pieces ...[]byte) (n int, err error) {
	framed.mutex.Lock()
	defer framed.mutex.Unlock()

	for _, piece := range pieces {
		n = n + len(piece)
	}

	if n > MaxFrameLength {
		return 0, fmt.Errorf(tooLongError, n, MaxFrameLength)
	}

	// Write the length header
	if err = binary.Write(framed.Stream, endianness, uint16(n)); err != nil {
		return
	}

	// Write the data
	var written int
	for _, piece := range pieces {
		var nw int
		if nw, err = framed.Stream.Write(piece); err != nil {
			return
		}
		written = written + nw
	}
	if written != n {
		err = fmt.Errorf("%d bytes written, expected to write %d", written, n)
	}
	return
}
<commit_msg>Cleaned up go vet issues<commit_after>/*
Package framed provides an implementations of io.Writer and io.Reader that write
and read whole frames only.

Frames are length-prefixed.  The first two bytes are an unsigned 16 bit int
stored in little-endian byte order indicating the length of the content.  The
remaining bytes are the actual content of the frame.

The use of a uint16 means that the maximum possible frame size (MaxFrameSize)
is 65535.
*/
package framed

import (
	"encoding/binary"
	"fmt"
	"io"
	"sync"
)

const (
	// FrameHeaderBits is the size of the frame header in bits
	FrameHeaderBits = 16

	// FrameHeaderLength is the size of the frame header in bytes
	FrameHeaderLength = FrameHeaderBits / 8

	// MaxFrameLength is the maximum possible size of a frame (not including the
	// length prefix)
	MaxFrameLength = 1<<FrameHeaderBits - 1

	tooLongError = "Attempted to write frame of length %d which is longer than maximum allowed length of %d"
)

var endianness = binary.LittleEndian

/*
A Reader enhances an io.ReadCloser to read data in contiguous frames. It
implements the io.Reader interface, but unlike typical io.Readers it only
returns whole frames.

A Reader also supports the ability to read frames using dynamically allocated
buffers via the ReadFrame method.
*/
type Reader struct {
	Stream io.Reader // the raw underlying connection
	mutex  sync.Mutex
}

/*
A Writer enhances an io.WriteCLoser to write data in contiguous frames. It
implements the io.Writer interface, but unlike typical io.Writers, it includes
information that allows a corresponding Reader to read whole frames without them
being fragmented.

A Writer also supports a method that writes multiple buffers to the underlying
stream as a single frame.
*/
type Writer struct {
	Stream io.Writer // the raw underlying connection
	mutex  sync.Mutex
}

func NewReader(r io.Reader) *Reader {
	return &Reader{Stream: r}
}

func NewWriter(w io.Writer) *Writer {
	return &Writer{Stream: w}
}

/*
Read implements the function from io.Reader.  Unlike io.Reader.Read,
frame.Read only returns full frames of data (assuming that the data was written
by a framed.Writer).
*/
func (framed *Reader) Read(buffer []byte) (n int, err error) {
	framed.mutex.Lock()
	defer framed.mutex.Unlock()

	var nb uint16
	err = binary.Read(framed.Stream, endianness, &nb)
	if err != nil {
		return
	}

	n = int(nb)

	bufferSize := len(buffer)
	if n > bufferSize {
		return 0, fmt.Errorf("Buffer of size %d is too small to hold frame of size %d", bufferSize, n)
	}

	// Read into buffer
	n, err = io.ReadFull(framed.Stream, buffer[:n])
	return
}

// ReadFrame reads the next frame, using a new buffer sized to hold the frame.
func (framed *Reader) ReadFrame() (frame []byte, err error) {
	framed.mutex.Lock()
	defer framed.mutex.Unlock()

	var nb uint16
	err = binary.Read(framed.Stream, endianness, &nb)
	if err != nil {
		return
	}

	l := int(nb)
	frame = make([]byte, l)

	// Read into buffer
	_, err = io.ReadFull(framed.Stream, frame)
	return
}

/*
Write implements the Write method from io.Writer.  It prepends a frame length
header that allows the framed.Reader on the other end to read the whole frame.
*/
func (framed *Writer) Write(frame []byte) (n int, err error) {
	framed.mutex.Lock()
	defer framed.mutex.Unlock()

	n = len(frame)
	if n > MaxFrameLength {
		return 0, fmt.Errorf(tooLongError, n, MaxFrameLength)
	}

	// Write the length header
	if err = binary.Write(framed.Stream, endianness, uint16(n)); err != nil {
		return
	}

	// Write the data
	var written int
	if written, err = framed.Stream.Write(frame); err != nil {
		return
	}
	if written != n {
		err = fmt.Errorf("%d bytes written, expected to write %d", written, n)
	}
	return
}

func (framed *Writer) WritePieces(pieces ...[]byte) (n int, err error) {
	framed.mutex.Lock()
	defer framed.mutex.Unlock()

	for _, piece := range pieces {
		n = n + len(piece)
	}

	if n > MaxFrameLength {
		return 0, fmt.Errorf(tooLongError, n, MaxFrameLength)
	}

	// Write the length header
	if err = binary.Write(framed.Stream, endianness, uint16(n)); err != nil {
		return
	}

	// Write the data
	var written int
	for _, piece := range pieces {
		var nw int
		if nw, err = framed.Stream.Write(piece); err != nil {
			return
		}
		written = written + nw
	}
	if written != n {
		err = fmt.Errorf("%d bytes written, expected to write %d", written, n)
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package fs

import (
	"container/list"
	"errors"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
)

const dirSeparator = '/'

// Implementation detail, do not touch.
//
// How long we cache the most recent listing for a particular directory from
// GCS before regarding it as stale.
//
// Intended to paper over performance issues caused by quick follow-up calls;
// for example when the fuse VFS performs a readdir followed quickly by a
// lookup for each child. The drawback is that this increases the time before a
// write by a foreign machine within a recently-listed directory will be seen
// locally.
//
// TODO(jacobsa): Set this according to real-world performance issues when the
// kernel does e.g. ReadDir followed by Lookup. Can probably be set quite
// small.
//
// TODO(jacobsa): Can this be moved to a decorator implementation of gcs.Bucket
// instead of living here?
var ListingCacheTTL = 10 * time.Second

// Implementation detail, do not touch.
//
// How long we remember that we took some action on the contents of a directory
// (linking or unlinking), and pretend the action is reflected in the listing
// even if it is not.
//
// Intended to paper over the fact that GCS doesn't offer list-your-own-writes
// consistency: it may be an arbitrarily long time before you see the creation
// or deletion of an object in a subsequent listing, and even if you see it in
// one listing you may not in the next. The drawback is that modifications to
// recently-modified directories by foreign machines will not be reflected
// locally for awhile.
//
// TODO(jacobsa): Set this according to information about listing staleness
// distributions from the GCS team.
//
// TODO(jacobsa): Can this be moved to a decorator implementation of gcs.Bucket
// instead of living here?
var ChildActionMemoryTTL = 5 * time.Minute

// See the childModifications field of dir.
type childModification struct {
	time time.Time
	name string

	// INVARIANT: nil or of type *file or *dir.
	node fusefs.Node
}

// A "directory" in GCS, defined by an object name prefix.
//
// For example, if the bucket contains objects "foo/bar" and "foo/baz", this
// implicitly defines the directory "foo/". No matter what the contents of the
// bucket, there is an implicit root directory "".
type dir struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	logger *log.Logger
	clock  timeutil.Clock
	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// INVARIANT: objectPrefix is "" (representing the root directory) or ends
	// with dirSeparator.
	objectPrefix string

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// Our current best understanding of the contents of the directory in GCS,
	// formed by listing the bucket and then patching according to child addition
	// and removal records at the time, and patched since then by subsequent
	// additions and removals.
	//
	// The time after which this should be generated anew from a new listing is
	// also stored. This is set to the time at which the listing completed plus
	// ListingCacheTTL.
	//
	// INVARIANT: contents != nil
	// INVARIANT: All nodes are of type *dir or *file.
	// INVARIANT: All nodes are indexed by names that agree with node contents.
	contents           map[string]fusefs.Node // GUARDED_BY(mu)
	contentsExpiration time.Time              // GUARDED_BY(mu)

	// A collection of children that have recently been added or removed locally
	// and the time at which it happened, ordered by the sequence in which it
	// happened. Elements M with M.node == nil are removals; all others are
	// additions.
	//
	// For a record M in this list with M's age less than ChildActionMemoryTTL,
	// any listing from the bucket should be augmented by pretending M just
	// happened.
	//
	// TODO(jacobsa): Make sure to test link followed by unlink, and unlink
	// followed by link.
	//
	// TODO(jacobsa): Make sure to test that these expire eventually, i.e. that
	// foreign overwrites, deletes, and recreates are reflected eventually.
	//
	// INVARIANT: All elements are of type childModification.
	// INVARIANT: Contains no duplicate names.
	// INVARIANT: For each M with M.node == nil, contents does not contain M.name.
	// INVARIANT: For each M with M.node != nil, contents[M.name] == M.node.
	childModifications list.List // GUARDED_BY(mu)

	// An index of childModifications by name.
	//
	// INVARIANT: childModificationsIndex != nil
	// INVARIANT: For all names N in the map, the indexed modification has name N.
	// INVARIANT: Contains exactly the set of names in childModifications.
	childModificationsIndex map[string]*list.Element // GUARDED_BY(mu)
}

// Make sure dir implements the interfaces we think it does.
var (
	// TODO(jacobsa): I think we want to embed fusefs.NdoeRef in all of our
	// fusefs.Node types, so that we better benefit from fusefs.Server node
	// caching.
	_ fusefs.Node               = &dir{}
	_ fusefs.NodeCreater        = &dir{}
	_ fusefs.NodeMknoder        = &dir{}
	_ fusefs.NodeStringLookuper = &dir{}

	_ fusefs.Handle             = &dir{}
	_ fusefs.HandleReadDirAller = &dir{}
)

func newDir(
	logger *log.Logger,
	clock timeutil.Clock,
	bucket gcs.Bucket,
	objectPrefix string) *dir {
	// Check the object prefix.
	if objectPrefix != "" && objectPrefix[len(objectPrefix)-1] != dirSeparator {
		panic("Unexpected object prefix: " + objectPrefix)
	}

	// Create the dir struct.
	d := &dir{
		logger:                  logger,
		clock:                   clock,
		bucket:                  bucket,
		objectPrefix:            objectPrefix,
		contents:                make(map[string]fusefs.Node),
		childModificationsIndex: make(map[string]*list.Element),
	}

	d.mu = syncutil.NewInvariantMutex(func() { d.checkInvariants() })

	return d
}

func (d *dir) checkInvariants()

// Ensure that d.contents is fresh and usable. Must be called before using
// d.contents.
//
// TODO(jacobsa): If contents hasn't expired, return immediately. Otherwise
// list parasitically while holding the lock (why not, we can make this more
// subtle later if we must) and modify the result appropriately.
//
// EXCLUSIVE_LOCKS_REQUIRED(d.mu)
func (d *dir) ensureContents(ctx context.Context) error

func (d *dir) Attr() fuse.Attr {
	return fuse.Attr{
		// TODO(jacobsa): Reflect that we allow writes now. Make sure to test.
		// TODO(jacobsa): Expose ACLs from GCS?
		Mode: os.ModeDir | 0500,
	}
}

// LOCKS_EXCLUDED(d.mu)
func (d *dir) ReadDirAll(ctx context.Context) (ents []fuse.Dirent, err error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	d.logger.Printf("ReadDirAll: [%s]/%s", d.bucket.Name(), d.objectPrefix)

	// Ensure that we can use d.contents.
	if err = d.ensureContents(ctx); err != nil {
		err = fmt.Errorf("d.ensureContents: %v", err)
		return
	}

	// Read out entries from the contents map.
	for name, node := range d.contents {
		ent := fuse.Dirent{
			Name: name,
		}

		if _, ok := node.(*dir); ok {
			ent.Type = fuse.DT_Dir
		} else {
			ent.Type = fuse.DT_File
		}

		ents = append(ents, ent)
	}

	return
}

// LOCKS_EXCLUDED(d.mu)
func (d *dir) Lookup(
	ctx context.Context,
	name string) (n fusefs.Node, err error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	d.logger.Printf("Lookup: ([%s]/%s) %s", d.bucket.Name(), d.objectPrefix, name)

	// Ensure that we can use d.contents.
	if err = d.ensureContents(ctx); err != nil {
		err = fmt.Errorf("d.ensureContents: %v", err)
		return
	}

	// Find the object within the map.
	var ok bool
	if n, ok = d.contents[name]; ok {
		return
	}

	err = fuse.ENOENT
	return
}

func (d *dir) Create(
	ctx context.Context,
	req *fuse.CreateRequest,
	resp *fuse.CreateResponse) (
	fusefs.Node,
	fusefs.Handle,
	error) {
	// Tell fuse to use Mknod followed by Open, rather than re-implementing much
	// of Open here.
	return nil, nil, fuse.ENOSYS
}

func (d *dir) Mknod(
	ctx context.Context,
	req *fuse.MknodRequest) (fusefs.Node, error) {
	return nil, errors.New("TODO(jacobsa): Support Mknod.")
}
<commit_msg>Partially implemented dir.checkInvariants.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package fs

import (
	"container/list"
	"errors"
	"fmt"
	"log"
	"os"
	"path"
	"reflect"
	"time"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
)

const dirSeparator = '/'

// Implementation detail, do not touch.
//
// How long we cache the most recent listing for a particular directory from
// GCS before regarding it as stale.
//
// Intended to paper over performance issues caused by quick follow-up calls;
// for example when the fuse VFS performs a readdir followed quickly by a
// lookup for each child. The drawback is that this increases the time before a
// write by a foreign machine within a recently-listed directory will be seen
// locally.
//
// TODO(jacobsa): Set this according to real-world performance issues when the
// kernel does e.g. ReadDir followed by Lookup. Can probably be set quite
// small.
//
// TODO(jacobsa): Can this be moved to a decorator implementation of gcs.Bucket
// instead of living here?
var ListingCacheTTL = 10 * time.Second

// Implementation detail, do not touch.
//
// How long we remember that we took some action on the contents of a directory
// (linking or unlinking), and pretend the action is reflected in the listing
// even if it is not.
//
// Intended to paper over the fact that GCS doesn't offer list-your-own-writes
// consistency: it may be an arbitrarily long time before you see the creation
// or deletion of an object in a subsequent listing, and even if you see it in
// one listing you may not in the next. The drawback is that modifications to
// recently-modified directories by foreign machines will not be reflected
// locally for awhile.
//
// TODO(jacobsa): Set this according to information about listing staleness
// distributions from the GCS team.
//
// TODO(jacobsa): Can this be moved to a decorator implementation of gcs.Bucket
// instead of living here?
var ChildActionMemoryTTL = 5 * time.Minute

// See the childModifications field of dir.
type childModification struct {
	time time.Time
	name string

	// INVARIANT: nil or of type *file or *dir.
	node fusefs.Node
}

// A "directory" in GCS, defined by an object name prefix.
//
// For example, if the bucket contains objects "foo/bar" and "foo/baz", this
// implicitly defines the directory "foo/". No matter what the contents of the
// bucket, there is an implicit root directory "".
type dir struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	logger *log.Logger
	clock  timeutil.Clock
	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// INVARIANT: objectPrefix is "" (representing the root directory) or ends
	// with dirSeparator.
	objectPrefix string

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// Our current best understanding of the contents of the directory in GCS,
	// formed by listing the bucket and then patching according to child addition
	// and removal records at the time, and patched since then by subsequent
	// additions and removals.
	//
	// The time after which this should be generated anew from a new listing is
	// also stored. This is set to the time at which the listing completed plus
	// ListingCacheTTL.
	//
	// INVARIANT: contents != nil
	// INVARIANT: All nodes are of type *dir or *file.
	// INVARIANT: All nodes are indexed by names that agree with node contents.
	contents           map[string]fusefs.Node // GUARDED_BY(mu)
	contentsExpiration time.Time              // GUARDED_BY(mu)

	// A collection of children that have recently been added or removed locally
	// and the time at which it happened, ordered by the sequence in which it
	// happened. Elements M with M.node == nil are removals; all others are
	// additions.
	//
	// For a record M in this list with M's age less than ChildActionMemoryTTL,
	// any listing from the bucket should be augmented by pretending M just
	// happened.
	//
	// TODO(jacobsa): Make sure to test link followed by unlink, and unlink
	// followed by link.
	//
	// TODO(jacobsa): Make sure to test that these expire eventually, i.e. that
	// foreign overwrites, deletes, and recreates are reflected eventually.
	//
	// INVARIANT: All elements are of type childModification.
	// INVARIANT: Contains no duplicate names.
	// INVARIANT: For each M with M.node == nil, contents does not contain M.name.
	// INVARIANT: For each M with M.node != nil, contents[M.name] == M.node.
	childModifications list.List // GUARDED_BY(mu)

	// An index of childModifications by name.
	//
	// INVARIANT: childModificationsIndex != nil
	// INVARIANT: For all names N in the map, the indexed modification has name N.
	// INVARIANT: Contains exactly the set of names in childModifications.
	childModificationsIndex map[string]*list.Element // GUARDED_BY(mu)
}

// Make sure dir implements the interfaces we think it does.
var (
	// TODO(jacobsa): I think we want to embed fusefs.NdoeRef in all of our
	// fusefs.Node types, so that we better benefit from fusefs.Server node
	// caching.
	_ fusefs.Node               = &dir{}
	_ fusefs.NodeCreater        = &dir{}
	_ fusefs.NodeMknoder        = &dir{}
	_ fusefs.NodeStringLookuper = &dir{}

	_ fusefs.Handle             = &dir{}
	_ fusefs.HandleReadDirAller = &dir{}
)

func newDir(
	logger *log.Logger,
	clock timeutil.Clock,
	bucket gcs.Bucket,
	objectPrefix string) *dir {
	// Check the object prefix.
	if objectPrefix != "" && objectPrefix[len(objectPrefix)-1] != dirSeparator {
		panic("Unexpected object prefix: " + objectPrefix)
	}

	// Create the dir struct.
	d := &dir{
		logger:                  logger,
		clock:                   clock,
		bucket:                  bucket,
		objectPrefix:            objectPrefix,
		contents:                make(map[string]fusefs.Node),
		childModificationsIndex: make(map[string]*list.Element),
	}

	d.mu = syncutil.NewInvariantMutex(func() { d.checkInvariants() })

	return d
}

func (d *dir) checkInvariants() {
	// Check that maps are non-nil.
	if d.contents == nil || d.childModificationsIndex == nil {
		panic("Expected contents and childModificationsIndex to be non-nil.")
	}

	// Check each element of the contents map.
	for name, node := range d.contents {
		if node == nil {
			panic("nil node for name: " + name)
		}

		d, okDir := node.(*dir)
		f, okFile := node.(*file)
		if !okDir && !okFile {
			panic(fmt.Sprintf("Unexpected node type: %v", reflect.TypeOf(node)))
		}

		if okDir && name != path.Base(d.objectPrefix) {
			panic(fmt.Sprintf("Name mismatch: %s vs. %s", name, d.objectPrefix))
		}

		if okFile && name != path.Base(f.objectName) {
			panic(fmt.Sprintf("Name mismatch: %s vs. %s", name, f.objectName))
		}
	}

	panic("TODO")
}

// Ensure that d.contents is fresh and usable. Must be called before using
// d.contents.
//
// TODO(jacobsa): If contents hasn't expired, return immediately. Otherwise
// list parasitically while holding the lock (why not, we can make this more
// subtle later if we must) and modify the result appropriately.
//
// EXCLUSIVE_LOCKS_REQUIRED(d.mu)
func (d *dir) ensureContents(ctx context.Context) error

func (d *dir) Attr() fuse.Attr {
	return fuse.Attr{
		// TODO(jacobsa): Reflect that we allow writes now. Make sure to test.
		// TODO(jacobsa): Expose ACLs from GCS?
		Mode: os.ModeDir | 0500,
	}
}

// LOCKS_EXCLUDED(d.mu)
func (d *dir) ReadDirAll(ctx context.Context) (ents []fuse.Dirent, err error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	d.logger.Printf("ReadDirAll: [%s]/%s", d.bucket.Name(), d.objectPrefix)

	// Ensure that we can use d.contents.
	if err = d.ensureContents(ctx); err != nil {
		err = fmt.Errorf("d.ensureContents: %v", err)
		return
	}

	// Read out entries from the contents map.
	for name, node := range d.contents {
		ent := fuse.Dirent{
			Name: name,
		}

		if _, ok := node.(*dir); ok {
			ent.Type = fuse.DT_Dir
		} else {
			ent.Type = fuse.DT_File
		}

		ents = append(ents, ent)
	}

	return
}

// LOCKS_EXCLUDED(d.mu)
func (d *dir) Lookup(
	ctx context.Context,
	name string) (n fusefs.Node, err error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	d.logger.Printf("Lookup: ([%s]/%s) %s", d.bucket.Name(), d.objectPrefix, name)

	// Ensure that we can use d.contents.
	if err = d.ensureContents(ctx); err != nil {
		err = fmt.Errorf("d.ensureContents: %v", err)
		return
	}

	// Find the object within the map.
	var ok bool
	if n, ok = d.contents[name]; ok {
		return
	}

	err = fuse.ENOENT
	return
}

func (d *dir) Create(
	ctx context.Context,
	req *fuse.CreateRequest,
	resp *fuse.CreateResponse) (
	fusefs.Node,
	fusefs.Handle,
	error) {
	// Tell fuse to use Mknod followed by Open, rather than re-implementing much
	// of Open here.
	return nil, nil, fuse.ENOSYS
}

func (d *dir) Mknod(
	ctx context.Context,
	req *fuse.MknodRequest) (fusefs.Node, error) {
	return nil, errors.New("TODO(jacobsa): Support Mknod.")
}
<|endoftext|>


<commit_before>// LICENSE: Anyone may do anything with this.

package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/http/fcgi"
	"strconv"
)

const (
	errorMsg = "Path parameter must be a number from 1 to 100."
)

func handler(w http.ResponseWriter, r *http.Request) {
	if s, ok := cache[r.URL.Path[1:]]; ok {
		fmt.Fprint(w, s)
		return
	}
	http.Error(w, errorMsg, http.StatusNotFound)
}

var cache map[string]string

func main() {
	args := struct{ port string }{}
	flag.StringVar(&args.port, "port", "", "The port to serve on.")
	flag.Parse()
	cache = make(map[string]string, 100)
	for i := 0; i < 101; i++ {
		s := strconv.Itoa(i)
		t := s
		f, b := i%3 == 0, i%5 == 0
		if f || b {
			t = ""
		}
		if f {
			t = "Fizz"
		}
		if b {
			t += "Buzz"
		}
		cache[s] = t
	}
	http.HandleFunc("/", handler)
	switch {
	case args.port != "":
		if err := http.ListenAndServe(args.port, nil); err != nil {
			log.Fatal(err)
		}
	default:
		if err := fcgi.Serve(nil, nil); err != nil {
			log.Fatal(err)
		}
	}
}
<commit_msg>Add newline<commit_after>// LICENSE: Anyone may do anything with this.

package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/http/fcgi"
	"strconv"
)

const (
	errorMsg = "Path parameter must be a number from 1 to 100."
)

func handler(w http.ResponseWriter, r *http.Request) {
	if s, ok := cache[r.URL.Path[1:]]; ok {
		fmt.Fprint(w, s)
		return
	}
	http.Error(w, errorMsg, http.StatusNotFound)
}

var cache map[string]string

func main() {
	args := struct{ port string }{}
	flag.StringVar(&args.port, "port", "", "The port to serve on.")
	flag.Parse()
	cache = make(map[string]string, 100)
	for i := 0; i < 101; i++ {
		s := strconv.Itoa(i)
		t := s
		f, b := i%3 == 0, i%5 == 0
		if f || b {
			t = ""
		}
		if f {
			t = "Fizz"
		}
		if b {
			t += "Buzz"
		}
		cache[s] = t + "\n"
	}
	http.HandleFunc("/", handler)
	switch {
	case args.port != "":
		if err := http.ListenAndServe(args.port, nil); err != nil {
			log.Fatal(err)
		}
	default:
		if err := fcgi.Serve(nil, nil); err != nil {
			log.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>/*

Package cos provides the core logic to tie together storage
and validation for a Chain Open Standard blockchain.
This comprises all behavior
that's common to every full node, as well as other functions that
need to operate on the blockchain state.

Here are a few examples of typical full node types.

Generator

An generator has two basic jobs: collecting transactions from
other nodes and putting them into blocks.

To collect pending transactions, call InsertPendingTx for each one.

To add a new block to the blockchain, call GenerateBlock, sign
the block (possibly collecting signatures from other parties),
and call AddBlock.

Signer

An signer has one basic job: sign exactly one valid block at each height.

To sign an unsigned block obtained from a generator node, first
validate against the current blockchain state, then call
ComputeBlockSignature, and finally send the signature back to the
generator node.

Manager

A manager's job is to select outputs for spending and to compose
transactions.

To publish a new transaction, prepare your transaction (select
outputs, and compose and sign the tx), call InsertPendingTx and
send the transaction to a generator node.
(TODO(kr): Then call WaitTx; when it returns (with no error),
the transaction has been confirmed.)

To ingest a block, call AddBlock.

*/
package cos

import (
	"sync"

	"golang.org/x/net/context"

	"github.com/btcsuite/btcd/btcec"

	"chain/cos/bc"
	"chain/cos/patricia"
	"chain/cos/state"
	"chain/errors"
)

var (
	// ErrTheDistantFuture (https://youtu.be/2IPAOxrH7Ro) is returned when
	// waiting for a blockheight too far in excess of the tip of the
	// blockchain.
	ErrTheDistantFuture = errors.New("the block height is too damn high")

	// ErrNoBlocks is returned when LatestBlock is called and the store
	// contains no blocks.
	ErrNoBlocks = errors.New("no blocks in the store")

	// ErrBadStateHeight is returned from Store.StateTree when the
	// height parameter does not match the latest block height.
	ErrBadStateHeight = errors.New("requested block height does not match current state")
)

type BlockCallback func(ctx context.Context, block *bc.Block, conflicts []*bc.Tx)
type TxCallback func(context.Context, *bc.Tx)

// Store provides storage for blockchain data: blocks, asset
// definition pointers, and pending transactions.
//
// Note, this is different from state.View. A View provides
// access to the state at a given point in time -- outputs and
// ADPs. It doesn't distinguish between blockchain outputs and
// pool outputs; a View instance might provide either, or both.
// Store, by contrast, provides lower-level access to the
// concrete blockchain. It can insert raw block data, add and
// remove pending txs from the pool, and update the UTXO state
// directly. An FC uses Store to create a set of Views,
// uses those views to validate a tx or block, then uses Store
// to commit the validated data.
type Store interface {
	// tx pool
	GetTxs(context.Context, ...bc.Hash) (poolTxs, bcTxs map[bc.Hash]*bc.Tx, err error)
	ApplyTx(context.Context, *bc.Tx, map[bc.AssetID]*state.AssetState) error
	CleanPool(ctx context.Context, confirmed, conflicting []*bc.Tx, assets map[bc.AssetID]*state.AssetState) error
	PoolTxs(context.Context) ([]*bc.Tx, error)
	NewPoolViewForPrevouts(context.Context, []*bc.Tx) (state.ViewReader, error)

	// blocks
	ApplyBlock(context.Context, *bc.Block, []*state.Output, map[bc.AssetID]*state.AssetState, *patricia.Tree) ([]*bc.Tx, error)
	FinalizeBlock(context.Context, uint64) error
	LatestBlock(context.Context) (*bc.Block, error)
	NewViewForPrevouts(context.Context, []*bc.Tx) (state.ViewReader, error)
	StateTree(context.Context, uint64) (*patricia.Tree, error)
}

// FC provides a complete, minimal blockchain database. It
// delegates the underlying storage to other objects, and uses
// validation logic from package validation to decide what
// objects can be safely stored.
type FC struct {
	blockCallbacks []BlockCallback
	txCallbacks    []TxCallback
	trustedKeys    []*btcec.PublicKey
	height         struct {
		cond sync.Cond // protects n
		n    uint64
	}
	store Store
}

// NewFC returns a new FC using store as the underlying storage.
//
// AddBlock will skip validation for any block signed by a key
// in trustedKeys. Typically, trustedKeys contains the public key
// for the local block-signer process; the presence of its
// signature indicates the block was already validated locally.
func NewFC(ctx context.Context, store Store, trustedKeys []*btcec.PublicKey, heights <-chan uint64) (*FC, error) {
	fc := &FC{store: store, trustedKeys: trustedKeys}

	latestBlock, err := fc.LatestBlock(ctx)
	if err != nil && errors.Root(err) != ErrNoBlocks {
		return nil, errors.Wrap(err, "looking up latest block")
	}
	if latestBlock != nil {
		fc.height.n = latestBlock.Height
	}
	// Now fc.height.n may still be zero because of ErrNoBlocks.

	fc.height.cond.L = new(sync.Mutex)

	if heights != nil {
		go func() {
			for h := range heights {
				fc.setHeight(h)
			}
		}()
	}

	return fc, nil
}

func (fc *FC) AddBlockCallback(f BlockCallback) {
	fc.blockCallbacks = append(fc.blockCallbacks, f)
}

func (fc *FC) AddTxCallback(f TxCallback) {
	fc.txCallbacks = append(fc.txCallbacks, f)
}

func (fc *FC) LatestBlock(ctx context.Context) (*bc.Block, error) {
	b, err := fc.store.LatestBlock(ctx)
	if err != nil {
		return nil, err
	}
	if b == nil {
		return nil, ErrNoBlocks
	}
	return b, nil
}

func (fc *FC) WaitForBlock(ctx context.Context, height uint64) error {
	const slop = 3

	fc.height.cond.L.Lock()
	defer fc.height.cond.L.Unlock()

	if height > fc.height.n+slop {
		return ErrTheDistantFuture
	}

	for fc.height.n < height {
		fc.height.cond.Wait()
	}

	return nil
}

func (fc *FC) GetTxs(ctx context.Context, hashes ...bc.Hash) (poolTxs, bcTxs map[bc.Hash]*bc.Tx, err error) {
	return fc.store.GetTxs(ctx, hashes...)
}
<commit_msg>cos: update stale package comment<commit_after>/*

Package cos provides the core logic to tie together storage
and validation for a Chain Open Standard blockchain.
This comprises all behavior
that's common to every full node, as well as other functions that
need to operate on the blockchain state.

Here are a few examples of typical full node types.

Generator

A generator has two basic jobs: collecting transactions from
other nodes and putting them into blocks.

To collect pending transactions, call AddTx for each one.

To add a new block to the blockchain, call GenerateBlock, sign
the block (possibly collecting signatures from other parties),
and call AddBlock.

Signer

An signer has one basic job: sign exactly one valid block at each height.

To sign an unsigned block obtained from a generator node, first
validate against the current blockchain state, then call
ComputeBlockSignature, and finally send the signature back to the
generator node.

Manager

A manager's job is to select outputs for spending and to compose
transactions.

To publish a new transaction, prepare your transaction (select
outputs, and compose and sign the tx), call AddTx, and
send the transaction to a generator node. To wait for confirmation,
call WaitForBlock on successive block heights and inspect the
blockchain state (using GetTxs or GetBlock) until you find that
the transaction has been either confirmed or rejected.

To ingest a block, call AddBlock.

*/
package cos

import (
	"sync"

	"golang.org/x/net/context"

	"github.com/btcsuite/btcd/btcec"

	"chain/cos/bc"
	"chain/cos/patricia"
	"chain/cos/state"
	"chain/errors"
)

var (
	// ErrTheDistantFuture (https://youtu.be/2IPAOxrH7Ro) is returned when
	// waiting for a blockheight too far in excess of the tip of the
	// blockchain.
	ErrTheDistantFuture = errors.New("the block height is too damn high")

	// ErrNoBlocks is returned when LatestBlock is called and the store
	// contains no blocks.
	ErrNoBlocks = errors.New("no blocks in the store")

	// ErrBadStateHeight is returned from Store.StateTree when the
	// height parameter does not match the latest block height.
	ErrBadStateHeight = errors.New("requested block height does not match current state")
)

type BlockCallback func(ctx context.Context, block *bc.Block, conflicts []*bc.Tx)
type TxCallback func(context.Context, *bc.Tx)

// Store provides storage for blockchain data: blocks, asset
// definition pointers, and pending transactions.
//
// Note, this is different from state.View. A View provides
// access to the state at a given point in time -- outputs and
// ADPs. It doesn't distinguish between blockchain outputs and
// pool outputs; a View instance might provide either, or both.
// Store, by contrast, provides lower-level access to the
// concrete blockchain. It can insert raw block data, add and
// remove pending txs from the pool, and update the UTXO state
// directly. An FC uses Store to create a set of Views,
// uses those views to validate a tx or block, then uses Store
// to commit the validated data.
type Store interface {
	// tx pool
	GetTxs(context.Context, ...bc.Hash) (poolTxs, bcTxs map[bc.Hash]*bc.Tx, err error)
	ApplyTx(context.Context, *bc.Tx, map[bc.AssetID]*state.AssetState) error
	CleanPool(ctx context.Context, confirmed, conflicting []*bc.Tx, assets map[bc.AssetID]*state.AssetState) error
	PoolTxs(context.Context) ([]*bc.Tx, error)
	NewPoolViewForPrevouts(context.Context, []*bc.Tx) (state.ViewReader, error)

	// blocks
	ApplyBlock(context.Context, *bc.Block, []*state.Output, map[bc.AssetID]*state.AssetState, *patricia.Tree) ([]*bc.Tx, error)
	FinalizeBlock(context.Context, uint64) error
	LatestBlock(context.Context) (*bc.Block, error)
	NewViewForPrevouts(context.Context, []*bc.Tx) (state.ViewReader, error)
	StateTree(context.Context, uint64) (*patricia.Tree, error)
}

// FC provides a complete, minimal blockchain database. It
// delegates the underlying storage to other objects, and uses
// validation logic from package validation to decide what
// objects can be safely stored.
type FC struct {
	blockCallbacks []BlockCallback
	txCallbacks    []TxCallback
	trustedKeys    []*btcec.PublicKey
	height         struct {
		cond sync.Cond // protects n
		n    uint64
	}
	store Store
}

// NewFC returns a new FC using store as the underlying storage.
//
// AddBlock will skip validation for any block signed by a key
// in trustedKeys. Typically, trustedKeys contains the public key
// for the local block-signer process; the presence of its
// signature indicates the block was already validated locally.
func NewFC(ctx context.Context, store Store, trustedKeys []*btcec.PublicKey, heights <-chan uint64) (*FC, error) {
	fc := &FC{store: store, trustedKeys: trustedKeys}

	latestBlock, err := fc.LatestBlock(ctx)
	if err != nil && errors.Root(err) != ErrNoBlocks {
		return nil, errors.Wrap(err, "looking up latest block")
	}
	if latestBlock != nil {
		fc.height.n = latestBlock.Height
	}
	// Now fc.height.n may still be zero because of ErrNoBlocks.

	fc.height.cond.L = new(sync.Mutex)

	if heights != nil {
		go func() {
			for h := range heights {
				fc.setHeight(h)
			}
		}()
	}

	return fc, nil
}

func (fc *FC) AddBlockCallback(f BlockCallback) {
	fc.blockCallbacks = append(fc.blockCallbacks, f)
}

func (fc *FC) AddTxCallback(f TxCallback) {
	fc.txCallbacks = append(fc.txCallbacks, f)
}

func (fc *FC) LatestBlock(ctx context.Context) (*bc.Block, error) {
	b, err := fc.store.LatestBlock(ctx)
	if err != nil {
		return nil, err
	}
	if b == nil {
		return nil, ErrNoBlocks
	}
	return b, nil
}

func (fc *FC) WaitForBlock(ctx context.Context, height uint64) error {
	const slop = 3

	fc.height.cond.L.Lock()
	defer fc.height.cond.L.Unlock()

	if height > fc.height.n+slop {
		return ErrTheDistantFuture
	}

	for fc.height.n < height {
		fc.height.cond.Wait()
	}

	return nil
}

func (fc *FC) GetTxs(ctx context.Context, hashes ...bc.Hash) (poolTxs, bcTxs map[bc.Hash]*bc.Tx, err error) {
	return fc.store.GetTxs(ctx, hashes...)
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2015 Ryan Fowler
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package ratelim

import (
	"sync/atomic"
)

type Counter struct {
	cnt *int64
}

func NewCounter() *Counter {
	var cnt int64
	return &Counter{cnt: &cnt}
}

func (c *Counter) Inc() int64 {
	return atomic.AddInt64(c.cnt, 1)
}

func (c *Counter) IncBy(i int64) int64 {
	return atomic.AddInt64(c.cnt, i)
}

func (c *Counter) Dec() int64 {
	return atomic.AddInt64(c.cnt, -1)
}

func (c *Counter) DecBy(i int64) int64 {
	return atomic.AddInt64(c.cnt, -i)
}

func (c *Counter) Zero() int64 {
	return atomic.SwapInt64(c.cnt, 0)
}

func (c *Counter) SetTo(i int64) int64 {
	return atomic.SwapInt64(c.cnt, i)
}
<commit_msg>Update Counter type<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2015 Ryan Fowler
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package ratelim

import (
	"sync/atomic"
)

type Counter struct {
	cnt int64
}

func NewCounter() *Counter {
	return &Counter{}
}

func (c *Counter) Inc() int64 {
	return atomic.AddInt64(&c.cnt, 1)
}

func (c *Counter) IncBy(i int64) int64 {
	return atomic.AddInt64(&c.cnt, i)
}

func (c *Counter) Dec() int64 {
	return atomic.AddInt64(&c.cnt, -1)
}

func (c *Counter) DecBy(i int64) int64 {
	return atomic.AddInt64(&c.cnt, -i)
}

func (c *Counter) Zero() int64 {
	return atomic.SwapInt64(&c.cnt, 0)
}

func (c *Counter) SetTo(i int64) int64 {
	return atomic.SwapInt64(&c.cnt, i)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"

	"github.com/gitpods/gitpods/cmd"
	"github.com/mattes/migrate"
	_ "github.com/mattes/migrate/database/postgres"
	_ "github.com/mattes/migrate/source/file"
	"github.com/urfave/cli"
)

func main() {
	app := cli.NewApp()
	app.Name = "gitpods-api"
	app.Usage = "git in the cloud!"

	app.Action = migrateAction
	app.Flags = migrateFlags

	if err := app.Run(os.Args); err != nil {
		log.Fatal(err)
	}
}

type migrateConf struct {
	DatabaseDriver string
	DatabaseDSN    string
	MigrationsPath string
}

var (
	migrateConfig = migrateConf{}

	migrateFlags = []cli.Flag{
		cli.StringFlag{
			Name:        cmd.FlagDatabaseDriver,
			EnvVar:      cmd.EnvDatabaseDriver,
			Usage:       "The database driver to use: memory & postgres",
			Value:       "postgres",
			Destination: &migrateConfig.DatabaseDriver,
		},
		cli.StringFlag{
			Name:        cmd.FlagDatabaseDSN,
			EnvVar:      cmd.EnvDatabaseDSN,
			Usage:       "The database connection data",
			Destination: &migrateConfig.DatabaseDSN,
		},
		cli.StringFlag{
			Name:        cmd.FlagMigrationsPath,
			EnvVar:      cmd.EnvMigrationsPath,
			Usage:       "The path to the folder containing all migrations",
			Destination: &migrateConfig.MigrationsPath,
		},
	}
)

func migrateAction(c *cli.Context) error {
	m, err := migrate.New("file://"+migrateConfig.MigrationsPath, migrateConfig.DatabaseDSN)
	if err != nil {
		return err
	}

	return m.Up()
}
<commit_msg>Print something when starting and finishing migrations successfully<commit_after>package main

import (
	"log"
	"os"

	"github.com/gitpods/gitpods/cmd"
	"github.com/mattes/migrate"
	_ "github.com/mattes/migrate/database/postgres"
	_ "github.com/mattes/migrate/source/file"
	"github.com/urfave/cli"
)

func main() {
	app := cli.NewApp()
	app.Name = "gitpods-api"
	app.Usage = "git in the cloud!"

	app.Action = migrateAction
	app.Flags = migrateFlags

	if err := app.Run(os.Args); err != nil {
		log.Fatal(err)
	}
}

type migrateConf struct {
	DatabaseDriver string
	DatabaseDSN    string
	MigrationsPath string
}

var (
	migrateConfig = migrateConf{}

	migrateFlags = []cli.Flag{
		cli.StringFlag{
			Name:        cmd.FlagDatabaseDriver,
			EnvVar:      cmd.EnvDatabaseDriver,
			Usage:       "The database driver to use: memory & postgres",
			Value:       "postgres",
			Destination: &migrateConfig.DatabaseDriver,
		},
		cli.StringFlag{
			Name:        cmd.FlagDatabaseDSN,
			EnvVar:      cmd.EnvDatabaseDSN,
			Usage:       "The database connection data",
			Destination: &migrateConfig.DatabaseDSN,
		},
		cli.StringFlag{
			Name:        cmd.FlagMigrationsPath,
			EnvVar:      cmd.EnvMigrationsPath,
			Usage:       "The path to the folder containing all migrations",
			Destination: &migrateConfig.MigrationsPath,
		},
	}
)

func migrateAction(c *cli.Context) error {
	log.Println("start migration...")

	m, err := migrate.New("file://"+migrateConfig.MigrationsPath, migrateConfig.DatabaseDSN)
	if err != nil {
		return err
	}

	if err := m.Up(); err != nil {
		return err
	}

	log.Println("migration successful")
	return nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"time"

	"context"
	"github.com/mitchellh/mapstructure"
	"github.com/pkg/errors"
	"github.com/zrepl/zrepl/rpc"
	"github.com/zrepl/zrepl/zfs"
	"sync"
)

type LocalJob struct {
	Name              string
	Mapping           *DatasetMapFilter
	SnapshotPrefix    string
	Interval          time.Duration
	InitialReplPolicy InitialReplPolicy
	PruneLHS          PrunePolicy
	PruneRHS          PrunePolicy
	Debug             JobDebugSettings
	snapperTask       *Task
	replTask          *Task
	pruneRHSTask      *Task
	pruneLHSTask      *Task
}

func parseLocalJob(c JobParsingContext, name string, i map[string]interface{}) (j *LocalJob, err error) {

	var asMap struct {
		Mapping           map[string]string
		SnapshotPrefix    string `mapstructure:"snapshot_prefix"`
		Interval          string
		InitialReplPolicy string                 `mapstructure:"initial_repl_policy"`
		PruneLHS          map[string]interface{} `mapstructure:"prune_lhs"`
		PruneRHS          map[string]interface{} `mapstructure:"prune_rhs"`
		Debug             map[string]interface{}
	}

	if err = mapstructure.Decode(i, &asMap); err != nil {
		err = errors.Wrap(err, "mapstructure error")
		return nil, err
	}

	j = &LocalJob{Name: name}

	if j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false); err != nil {
		return
	}

	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
		return
	}

	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
		err = errors.Wrap(err, "cannot parse interval")
		return
	}

	if j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY); err != nil {
		return
	}

	if j.PruneLHS, err = parsePrunePolicy(asMap.PruneLHS); err != nil {
		err = errors.Wrap(err, "cannot parse 'prune_lhs'")
		return
	}
	if j.PruneRHS, err = parsePrunePolicy(asMap.PruneRHS); err != nil {
		err = errors.Wrap(err, "cannot parse 'prune_rhs'")
		return
	}

	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
		err = errors.Wrap(err, "cannot parse 'debug'")
		return
	}

	return
}

func (j *LocalJob) JobName() string {
	return j.Name
}

func (j *LocalJob) JobStart(ctx context.Context) {

	log := ctx.Value(contextKeyLog).(Logger)
	defer log.Info("exiting")

	j.snapperTask = NewTask("snapshot", log)
	j.replTask = NewTask("repl", log)
	j.pruneRHSTask = NewTask("prune_rhs", log)
	j.pruneLHSTask = NewTask("prune_lhs", log)

	local := rpc.NewLocalRPC()
	// Allow access to any dataset since we control what mapping
	// is passed to the pull routine.
	// All local datasets will be passed to its Map() function,
	// but only those for which a mapping exists will actually be pulled.
	// We can pay this small performance penalty for now.
	handler := NewHandler(log.WithField(logTaskField, "handler"), localPullACL{}, NewPrefixFilter(j.SnapshotPrefix))

	registerEndpoints(local, handler)

	snapper := IntervalAutosnap{
		task:             j.snapperTask,
		DatasetFilter:    j.Mapping.AsFilter(),
		Prefix:           j.SnapshotPrefix,
		SnapshotInterval: j.Interval,
	}

	plhs, err := j.Pruner(j.pruneLHSTask, PrunePolicySideLeft, false)
	if err != nil {
		log.WithError(err).Error("error creating lhs pruner")
		return
	}
	prhs, err := j.Pruner(j.pruneRHSTask, PrunePolicySideRight, false)
	if err != nil {
		log.WithError(err).Error("error creating rhs pruner")
		return
	}

	makeCtx := func(parent context.Context, taskName string) (ctx context.Context) {
		return context.WithValue(parent, contextKeyLog, log.WithField(logTaskField, taskName))
	}
	var snapCtx, plCtx, prCtx, pullCtx context.Context
	snapCtx = makeCtx(ctx, "autosnap")
	plCtx = makeCtx(ctx, "prune_lhs")
	prCtx = makeCtx(ctx, "prune_rhs")
	pullCtx = makeCtx(ctx, "repl")

	didSnaps := make(chan struct{})
	go snapper.Run(snapCtx, didSnaps)

outer:
	for {

		select {
		case <-ctx.Done():
			break outer
		case <-didSnaps:
			log.Debug("finished taking snapshots")
			log.Info("starting replication procedure")
		}

		{
			log := pullCtx.Value(contextKeyLog).(Logger)
			log.Debug("replicating from lhs to rhs")
			puller := Puller{j.replTask, local, j.Mapping, j.InitialReplPolicy}
			puller.Pull()

			// use a ctx as soon as Pull gains ctx support
			select {
			case <-ctx.Done():
				break outer
			default:
			}
		}

		var wg sync.WaitGroup

		log.Info("pruning lhs")
		wg.Add(1)
		go func() {
			plhs.Run(plCtx)
			wg.Done()
		}()

		log.Info("pruning rhs")
		wg.Add(1)
		go func() {
			prhs.Run(prCtx)
			wg.Done()
		}()

		wg.Wait()

	}

	log.WithError(ctx.Err()).Info("context")

}

func (j *LocalJob) JobStatus(ctxt context.Context) (*JobStatus, error) {
	return &JobStatus{Tasks: []*TaskStatus{
		j.snapperTask.Status(),
		j.pruneLHSTask.Status(),
		j.pruneRHSTask.Status(),
		j.replTask.Status(),
	}}, nil
}

func (j *LocalJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) {

	var dsfilter zfs.DatasetFilter
	var pp PrunePolicy
	switch side {
	case PrunePolicySideLeft:
		pp = j.PruneLHS
		dsfilter = j.Mapping.AsFilter()
	case PrunePolicySideRight:
		pp = j.PruneRHS
		dsfilter, err = j.Mapping.InvertedFilter()
		if err != nil {
			err = errors.Wrap(err, "cannot invert mapping for prune_rhs")
			return
		}
	default:
		err = errors.Errorf("must be either left or right side")
		return
	}

	p = Pruner{
		task,
		time.Now(),
		dryRun,
		dsfilter,
		j.SnapshotPrefix,
		pp,
	}

	return
}
<commit_msg>job local: refactor + use Task API<commit_after>package cmd

import (
	"time"

	"context"
	"github.com/mitchellh/mapstructure"
	"github.com/pkg/errors"
	"github.com/zrepl/zrepl/rpc"
	"github.com/zrepl/zrepl/zfs"
	"sync"
)

type LocalJob struct {
	Name              string
	Mapping           *DatasetMapFilter
	SnapshotPrefix    string
	Interval          time.Duration
	InitialReplPolicy InitialReplPolicy
	PruneLHS          PrunePolicy
	PruneRHS          PrunePolicy
	Debug             JobDebugSettings
	snapperTask       *Task
	mainTask          *Task
	handlerTask       *Task
	pruneRHSTask      *Task
	pruneLHSTask      *Task
}

func parseLocalJob(c JobParsingContext, name string, i map[string]interface{}) (j *LocalJob, err error) {

	var asMap struct {
		Mapping           map[string]string
		SnapshotPrefix    string `mapstructure:"snapshot_prefix"`
		Interval          string
		InitialReplPolicy string                 `mapstructure:"initial_repl_policy"`
		PruneLHS          map[string]interface{} `mapstructure:"prune_lhs"`
		PruneRHS          map[string]interface{} `mapstructure:"prune_rhs"`
		Debug             map[string]interface{}
	}

	if err = mapstructure.Decode(i, &asMap); err != nil {
		err = errors.Wrap(err, "mapstructure error")
		return nil, err
	}

	j = &LocalJob{Name: name}

	if j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false); err != nil {
		return
	}

	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
		return
	}

	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
		err = errors.Wrap(err, "cannot parse interval")
		return
	}

	if j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY); err != nil {
		return
	}

	if j.PruneLHS, err = parsePrunePolicy(asMap.PruneLHS); err != nil {
		err = errors.Wrap(err, "cannot parse 'prune_lhs'")
		return
	}
	if j.PruneRHS, err = parsePrunePolicy(asMap.PruneRHS); err != nil {
		err = errors.Wrap(err, "cannot parse 'prune_rhs'")
		return
	}

	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
		err = errors.Wrap(err, "cannot parse 'debug'")
		return
	}

	return
}

func (j *LocalJob) JobName() string {
	return j.Name
}

func (j *LocalJob) JobStart(ctx context.Context) {

	rootLog := ctx.Value(contextKeyLog).(Logger)

	j.snapperTask = NewTask("snapshot", rootLog)
	j.mainTask = NewTask("main", rootLog)
	j.handlerTask = NewTask("handler", rootLog)
	j.pruneRHSTask = NewTask("prune_rhs", rootLog)
	j.pruneLHSTask = NewTask("prune_lhs", rootLog)

	local := rpc.NewLocalRPC()
	// Allow access to any dataset since we control what mapping
	// is passed to the pull routine.
	// All local datasets will be passed to its Map() function,
	// but only those for which a mapping exists will actually be pulled.
	// We can pay this small performance penalty for now.
	handler := NewHandler(j.handlerTask.Log(), localPullACL{}, NewPrefixFilter(j.SnapshotPrefix))

	registerEndpoints(local, handler)

	snapper := IntervalAutosnap{
		task:             j.snapperTask,
		DatasetFilter:    j.Mapping.AsFilter(),
		Prefix:           j.SnapshotPrefix,
		SnapshotInterval: j.Interval,
	}

	plhs, err := j.Pruner(j.pruneLHSTask, PrunePolicySideLeft, false)
	if err != nil {
		rootLog.WithError(err).Error("error creating lhs pruner")
		return
	}
	prhs, err := j.Pruner(j.pruneRHSTask, PrunePolicySideRight, false)
	if err != nil {
		rootLog.WithError(err).Error("error creating rhs pruner")
		return
	}

	didSnaps := make(chan struct{})
	go snapper.Run(ctx, didSnaps)

outer:
	for {

		select {
		case <-ctx.Done():
			j.mainTask.Log().WithError(ctx.Err()).Info("context")
			break outer
		case <-didSnaps:
			j.mainTask.Log().Debug("finished taking snapshots")
			j.mainTask.Log().Info("starting replication procedure")
		}

		j.mainTask.Log().Debug("replicating from lhs to rhs")
		j.mainTask.Enter("replicate")
		puller := Puller{j.mainTask, local, j.Mapping, j.InitialReplPolicy}
		puller.Pull()
		j.mainTask.Finish()

		// use a ctx as soon as Pull gains ctx support
		select {
		case <-ctx.Done():
			break outer
		default:
		}

		var wg sync.WaitGroup

		j.mainTask.Log().Info("pruning lhs")
		wg.Add(1)
		go func() {
			plhs.Run(ctx)
			wg.Done()
		}()

		j.mainTask.Log().Info("pruning rhs")
		wg.Add(1)
		go func() {
			prhs.Run(ctx)
			wg.Done()
		}()

		wg.Wait()

	}

}

func (j *LocalJob) JobStatus(ctxt context.Context) (*JobStatus, error) {
	return &JobStatus{Tasks: []*TaskStatus{
		j.snapperTask.Status(),
		j.pruneLHSTask.Status(),
		j.pruneRHSTask.Status(),
		j.mainTask.Status(),
	}}, nil
}

func (j *LocalJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) {

	var dsfilter zfs.DatasetFilter
	var pp PrunePolicy
	switch side {
	case PrunePolicySideLeft:
		pp = j.PruneLHS
		dsfilter = j.Mapping.AsFilter()
	case PrunePolicySideRight:
		pp = j.PruneRHS
		dsfilter, err = j.Mapping.InvertedFilter()
		if err != nil {
			err = errors.Wrap(err, "cannot invert mapping for prune_rhs")
			return
		}
	default:
		err = errors.Errorf("must be either left or right side")
		return
	}

	p = Pruner{
		task,
		time.Now(),
		dryRun,
		dsfilter,
		j.SnapshotPrefix,
		pp,
	}

	return
}
<|endoftext|>


<commit_before>package notmain

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"

	"github.com/letsencrypt/boulder/cmd"
	"github.com/letsencrypt/boulder/crl/crl_x509"
	"github.com/letsencrypt/boulder/issuance"
	"github.com/letsencrypt/boulder/linter"
	crlint "github.com/letsencrypt/boulder/linter/lints/crl"
)

func validateShard(url string, issuer *issuance.Certificate) (*crl_x509.RevocationList, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, fmt.Errorf("downloading crl: %w", err)
	}
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("downloading crl: http status %d", resp.StatusCode)
	}

	crlBytes, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("reading CRL bytes: %w", err)
	}

	crl, err := crl_x509.ParseRevocationList(crlBytes)
	if err != nil {
		return nil, fmt.Errorf("parsing CRL: %w", err)
	}

	err = linter.ProcessResultSet(crlint.LintCRL(crl))
	if err != nil {
		return nil, fmt.Errorf("linting CRL: %w", err)
	}

	err = crl.CheckSignatureFrom(issuer.Certificate)
	if err != nil {
		return nil, fmt.Errorf("checking CRL signature: %w", err)
	}

	return crl, nil
}

func main() {
	urlFile := flag.String("crls", "", "path to a file containing a JSON Array of CRL URLs")
	issuerFile := flag.String("issuer", "", "path to an issuer certificate on disk")
	emitRevoked := flag.Bool("emitRevoked", false, "emit revoked serial numbers on stdout, one per line, hex-encoded")
	flag.Parse()

	logger := cmd.NewLogger(cmd.SyslogConfig{StdoutLevel: 6, SyslogLevel: -1})

	urlFileContents, err := os.ReadFile(*urlFile)
	cmd.FailOnError(err, "Reading CRL URLs file")

	var urls []string
	err = json.Unmarshal(urlFileContents, &urls)
	cmd.FailOnError(err, "Parsing JSON Array of CRL URLs")

	issuer, err := issuance.LoadCertificate(*issuerFile)
	cmd.FailOnError(err, "Loading issuer certificate")

	errCount := 0
	for _, url := range urls {
		crl, err := validateShard(url, issuer)
		if err != nil {
			errCount += 1
			logger.Errf("CRL %q failed: %s", url, err)
			continue
		}
		if *emitRevoked {
			for _, c := range crl.RevokedCertificates {
				fmt.Printf("%x\n", c.SerialNumber)
			}
		}
	}

	if errCount != 0 {
		cmd.Fail(fmt.Sprintf("Encountered %d errors", errCount))
	}
	logger.AuditInfo("All CRLs validated")
}

func init() {
	cmd.RegisterCommand("crl-checker", main)
}
<commit_msg>crl-checker: add check for too-old CRLs (#6407)<commit_after>package notmain

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"time"

	"github.com/letsencrypt/boulder/cmd"
	"github.com/letsencrypt/boulder/crl/crl_x509"
	"github.com/letsencrypt/boulder/issuance"
	"github.com/letsencrypt/boulder/linter"
	crlint "github.com/letsencrypt/boulder/linter/lints/crl"
)

func downloadShard(url string) (*crl_x509.RevocationList, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, fmt.Errorf("downloading crl: %w", err)
	}
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("downloading crl: http status %d", resp.StatusCode)
	}

	crlBytes, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("reading CRL bytes: %w", err)
	}

	crl, err := crl_x509.ParseRevocationList(crlBytes)
	if err != nil {
		return nil, fmt.Errorf("parsing CRL: %w", err)
	}

	return crl, nil
}

func validateShard(crl *crl_x509.RevocationList, issuer *issuance.Certificate, ageLimit time.Duration) error {
	err := linter.ProcessResultSet(crlint.LintCRL(crl))
	if err != nil {
		return fmt.Errorf("linting CRL: %w", err)
	}

	err = crl.CheckSignatureFrom(issuer.Certificate)
	if err != nil {
		return fmt.Errorf("checking CRL signature: %w", err)
	}

	if time.Since(crl.ThisUpdate) >= ageLimit {
		return fmt.Errorf("thisUpdate more than %s in the past: %v", ageLimit, crl.ThisUpdate)
	}

	return nil
}

func main() {
	urlFile := flag.String("crls", "", "path to a file containing a JSON Array of CRL URLs")
	issuerFile := flag.String("issuer", "", "path to an issuer certificate on disk")
	ageLimitStr := flag.String("ageLimit", "168h", "maximum allowable age of a CRL shard")
	emitRevoked := flag.Bool("emitRevoked", false, "emit revoked serial numbers on stdout, one per line, hex-encoded")
	flag.Parse()

	logger := cmd.NewLogger(cmd.SyslogConfig{StdoutLevel: 6, SyslogLevel: -1})

	urlFileContents, err := os.ReadFile(*urlFile)
	cmd.FailOnError(err, "Reading CRL URLs file")

	var urls []string
	err = json.Unmarshal(urlFileContents, &urls)
	cmd.FailOnError(err, "Parsing JSON Array of CRL URLs")

	issuer, err := issuance.LoadCertificate(*issuerFile)
	cmd.FailOnError(err, "Loading issuer certificate")

	ageLimit, err := time.ParseDuration(*ageLimitStr)
	cmd.FailOnError(err, "Parsing age limit")

	errCount := 0
	for _, url := range urls {
		crl, err := downloadShard(url)
		if err != nil {
			errCount += 1
			logger.Errf("fetching CRL %q failed: %s", url, err)
			continue
		}

		err = validateShard(crl, issuer, ageLimit)
		if err != nil {
			errCount += 1
			logger.Errf("checking CRL %q failed: %s", url, err)
			continue
		}

		if *emitRevoked {
			for _, c := range crl.RevokedCertificates {
				fmt.Printf("%x\n", c.SerialNumber)
			}
		}
	}

	if errCount != 0 {
		cmd.Fail(fmt.Sprintf("Encountered %d errors", errCount))
	}
	logger.AuditInfo("All CRLs validated")
}

func init() {
	cmd.RegisterCommand("crl-checker", main)
}
<|endoftext|>


<commit_before>package cpu

import (
	"errors"
	"fmt"
	"log"
	"os"
	"strconv"
	"time"

	"github.com/Soulou/acadock-monitoring/config"
	"github.com/Soulou/acadock-monitoring/docker"
)

const (
	LXC_CPUACCT_USAGE_FILE = "cpuacct.usage"
)

var RefreshTime int

func init() {
	var err error
	RefreshTime, err = strconv.Atoi(config.ENV["REFRESH_TIME"])
	if err != nil {
		panic(err)
	}
}

var (
	previousCpuUsages = make(map[string]int64)
	cpuUsages         = make(map[string]int64)
)

func cpuacctUsage(container string) (int64, error) {
	file := config.CgroupPath("cpuacct", container) + "/" + LXC_CPUACCT_USAGE_FILE
	f, err := os.Open(file)
	if err != nil {
		log.Println(err)
		return 0, err
	}

	buffer := make([]byte, 16)
	n, err := f.Read(buffer)
	buffer = buffer[:n]

	bufferStr := string(buffer)
	bufferStr = bufferStr[:len(bufferStr)-1]

	res, err := strconv.ParseInt(bufferStr, 10, strconv.IntSize)
	if err != nil {
		log.Println("Failed to parse : ", err)
		return 0, err
	}
	return res, nil
}

func Monitor() {
	containers := make(chan string)
	go docker.ListenNewContainers(containers)
	go docker.ListRunningContainers(containers)
	for c := range containers {
		go monitorContainer(c)
	}
}

func monitorContainer(id string) {
	fmt.Println("monitor cpu", id)
	tick := time.NewTicker(time.Duration(RefreshTime) * time.Second)
	for {
		<-tick.C
		usage, err := cpuacctUsage(id)
		if err != nil {
			if _, ok := cpuUsages[id]; ok {
				delete(cpuUsages, id)
			}
			if _, ok := previousCpuUsages[id]; ok {
				delete(previousCpuUsages, id)
			}
			log.Println("stop monitoring", id)
			return
		}

		if prevUsage, ok := cpuUsages[id]; ok {
			previousCpuUsages[id] = prevUsage
		}
		cpuUsages[id] = usage
	}
}

func GetUsage(id string) (int64, error) {
	id, err := docker.ExpandId(id)
	if err != nil {
		log.Println("Error when expanding id:", err)
		return -1, err
	}
	if _, ok := previousCpuUsages[id]; !ok {
		return -1, errors.New("not ready")
	}
	return int64((float64((cpuUsages[id] - previousCpuUsages[id])) / float64(1e9) / float64(RefreshTime)) * 100), nil
}
<commit_msg>Parse 64bits integer for cpu monitoring<commit_after>package cpu

import (
	"errors"
	"fmt"
	"log"
	"os"
	"strconv"
	"time"

	"github.com/Soulou/acadock-monitoring/config"
	"github.com/Soulou/acadock-monitoring/docker"
)

const (
	LXC_CPUACCT_USAGE_FILE = "cpuacct.usage"
)

var RefreshTime int

func init() {
	var err error
	RefreshTime, err = strconv.Atoi(config.ENV["REFRESH_TIME"])
	if err != nil {
		panic(err)
	}
}

var (
	previousCpuUsages = make(map[string]int64)
	cpuUsages         = make(map[string]int64)
)

func cpuacctUsage(container string) (int64, error) {
	file := config.CgroupPath("cpuacct", container) + "/" + LXC_CPUACCT_USAGE_FILE
	f, err := os.Open(file)
	if err != nil {
		log.Println(err)
		return 0, err
	}

	buffer := make([]byte, 16)
	n, err := f.Read(buffer)
	buffer = buffer[:n]

	bufferStr := string(buffer)
	bufferStr = bufferStr[:len(bufferStr)-1]

	res, err := strconv.ParseInt(bufferStr, 10, 64)
	if err != nil {
		log.Println("Failed to parse : ", err)
		return 0, err
	}
	return res, nil
}

func Monitor() {
	containers := make(chan string)
	go docker.ListenNewContainers(containers)
	go docker.ListRunningContainers(containers)
	for c := range containers {
		go monitorContainer(c)
	}
}

func monitorContainer(id string) {
	fmt.Println("monitor cpu", id)
	tick := time.NewTicker(time.Duration(RefreshTime) * time.Second)
	for {
		<-tick.C
		usage, err := cpuacctUsage(id)
		if err != nil {
			if _, ok := cpuUsages[id]; ok {
				delete(cpuUsages, id)
			}
			if _, ok := previousCpuUsages[id]; ok {
				delete(previousCpuUsages, id)
			}
			log.Println("stop monitoring", id)
			return
		}

		if prevUsage, ok := cpuUsages[id]; ok {
			previousCpuUsages[id] = prevUsage
		}
		cpuUsages[id] = usage
	}
}

func GetUsage(id string) (int64, error) {
	id, err := docker.ExpandId(id)
	if err != nil {
		log.Println("Error when expanding id:", err)
		return -1, err
	}
	if _, ok := previousCpuUsages[id]; !ok {
		return -1, errors.New("not ready")
	}
	return int64((float64((cpuUsages[id] - previousCpuUsages[id])) / float64(1e9) / float64(RefreshTime)) * 100), nil
}
<|endoftext|>


<commit_before>package main

import (
	cryptorand "crypto/rand"
	"crypto/sha512"
	"flag"
	"fmt"
	//	"math/rand"
	"os"
	"runtime"
	"time"
)

var (
	phrase = ""
)

// randomSeed produces a int64 seed based on crypto/rand and time.
func randomSeed() int64 {
	var seed int64

	urandom := make([]byte, 8)
	cryptorand.Reader.Read(urandom)

	for key, value := range urandom {
		seed ^= (int64(value) ^ time.Now().UTC().UnixNano()) << (uint(key) * 8)
	}

	return seed
}

func init() {
	flag.StringVar(&phrase, "phrase", "", "the Garble phrase, by default random")
	flag.Parse()

	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(1)
	}

	if phrase == "" {
		phrase = fmt.Sprintf("%016x", uint64(randomSeed()))
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	args := flag.Args()

	for i := range args {
		fmt.Println(args[i], "with", phrase)
	}

	hash := sha512.New()
	seed := []byte(phrase)
	b := make([]byte, hash.Size())
	for i := 0; i < 2; i++ {
		hash.Write(seed)
		hash.Sum(b[:0])
		fmt.Println(b)
	}
}
<commit_msg>open files<commit_after>package main

import (
	cryptorand "crypto/rand"
	"crypto/sha512"
	"flag"
	"fmt"
	//	"math/rand"
	"os"
	"runtime"
	"time"
)

var (
	phrase = ""
)

// randomSeed produces a int64 seed based on crypto/rand and time.
func randomSeed() int64 {
	var seed int64

	urandom := make([]byte, 8)
	cryptorand.Reader.Read(urandom)

	for key, value := range urandom {
		seed ^= (int64(value) ^ time.Now().UTC().UnixNano()) << (uint(key) * 8)
	}

	return seed
}

func init() {
	flag.StringVar(&phrase, "phrase", "", "the Garble phrase, by default random")
	flag.Parse()

	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(1)
	}

	if phrase == "" {
		phrase = fmt.Sprintf("%016x", uint64(randomSeed()))
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	files := make([]*os.File, flag.NArg())

	for i, arg := range flag.Args() {
		f, err := os.OpenFile(arg, os.O_RDWR, 0666)
		if err != nil {
			panic(err)
		}
		defer f.Close()
		files[i] = f
		fmt.Println(arg, "with", phrase)
	}

	hash := sha512.New()
	seed := []byte(phrase)
	b := make([]byte, hash.Size())
	for i := 0; i < 2; i++ {
		hash.Write(seed)
		hash.Sum(b[:0])
		fmt.Println(b)
	}
}
<|endoftext|>