<commit_before>// Copyright (C) 2011, Kyle Lemons
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice,
//   this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright notice,
//   this list of conditions and the following disclaimer in the documentation
//   and/or other materials provided with the distribution.
// * Neither the name of the author nor the names of its contributors may be
//   used to endorse or promote products derived from this software without
//   specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
//
// Contributors:
//   andybalholm

// Gypsy is a simplified YAML parser written in Go.  It is intended to be used as
// a simple configuration file, and as such does not support a lot of the more
// nuanced syntaxes allowed in full-fledged YAML.  YAML does not allow indent with
// tabs, and GYPSY does not ever consider a tab to be a space character.  It is
// recommended that your editor be configured to convert tabs to spaces when
// editing Gypsy config files.
//
// Gypsy understands the following to be a list:
//
//     - one
//     - two
//     - three
//
// This is parsed as a `yaml.List`, and can be retrieved from the
// `yaml.Node.List()` method.  In this case, each element of the `yaml.List` would
// be a `yaml.Scalar` whose value can be retrieved with the `yaml.Scalar.String()`
// method.
//
// Gypsy understands the following to be a mapping:
//
//     key:     value
//     foo:     bar
//     running: away
//
// A mapping is an unordered list of `key:value` pairs.  All whitespace after the
// colon is stripped from the value and is used for alignment purposes during
// export.  If the value is not a list or a map, everything after the first
// non-space character until the end of the line is used as the `yaml.Scalar`
// value.
//
// Gypsy allows arbitrary nesting of maps inside lists, lists inside of maps, and
// maps and/or lists nested inside of themselves.
//
// A map inside of a list:
//
//     - name: John Smith
//       age:  42
//     - name: Jane Smith
//       age:  45
//
// A list inside of a map:
//
//     schools:
//       - Meadow Glen
//       - Forest Creek
//       - Shady Grove
//     libraries:
//       - Joseph Hollingsworth Memorial
//       - Andrew Keriman Memorial
//
// A list of lists:
//
//     - - one
//       - two
//       - three
//     - - un
//       - deux
//       - trois
//     - - ichi
//       - ni
//       - san
//
// A map of maps:
//
//     google:
//       company: Google, Inc.
//       ticker:  GOOG
//       url:     http://google.com/
//     yahoo:
//       company: Yahoo, Inc.
//       ticker:  YHOO
//       url:     http://yahoo.com/
//
// In the case of a map of maps, all sub-keys must be on subsequent lines and
// indented equally.  It is allowable for the first key/value to be on the same
// line if there is more than one key/value pair, but this is not recommended.
//
// Values can also be expressed in long form (leading whitespace of the first line
// is removed from it and all subsequent lines).  In the normal (baz) case,
// newlines are treated as spaces, all indentation is removed.  In the folded case
// (bar), newlines are treated as spaces, except pairs of newlines (e.g. a blank
// line) are treated as a single newline, only the indentation level of the first
// line is removed, and newlines at the end of indented lines are preserved.  In
// the verbatim (foo) case, only the indent at the level of the first line is
// stripped.  The example:
//
//     foo: |
//       lorem ipsum dolor
//       sit amet
//     bar: >
//       lorem ipsum
//
//         dolor
//
//       sit amet
//     baz:
//       lorem ipsum
//        dolor sit amet
//
// The YAML subset understood by Gypsy can be expressed (loosely) in the following
// grammar (not including comments):
//
//               OBJECT = MAPPING | SEQUENCE | SCALAR .
//         SHORT-OBJECT = SHORT-MAPPING | SHORT-SEQUENCE | SHORT-SCALAR .
//                  EOL = '\n'
//
//              MAPPING = { LONG-MAPPING | SHORT-MAPPING } .
//             SEQUENCE = { LONG-SEQUENCE | SHORT-SEQUENCE } .
//               SCALAR = { LONG-SCALAR | SHORT-SCALAR } .
//
//         LONG-MAPPING = { INDENT KEY ':' OBJECT EOL } .
//        SHORT-MAPPING = '{' KEY ':' SHORT-OBJECT { ',' KEY ':' SHORT-OBJECT } '}' EOL .
//
//        LONG-SEQUENCE = { INDENT '-' OBJECT EOL } EOL .
//       SHORT-SEQUENCE = '[' SHORT-OBJECT { ',' SHORT-OBJECT } ']' EOL .
//
//          LONG-SCALAR = ( '|' | '>' | ) EOL { INDENT SHORT-SCALAR EOL }
//         SHORT-SCALAR = { alpha | digit | punct | ' ' | '\t' } .
//
//                  KEY = { alpha | digit }
//               INDENT = { ' ' }
//
// Any line where the first non-space character is a sharp sign (#) is a comment.
// It will be ignored.
// Only full-line comments are allowed.
package yaml

// BUG(kevlar): Multi-line strings are currently not supported.
<commit_msg>Moved license to LICENSE<commit_after>// Gypsy is a simplified YAML parser written in Go.  It is intended to be used as
// a simple configuration file, and as such does not support a lot of the more
// nuanced syntaxes allowed in full-fledged YAML.  YAML does not allow indent with
// tabs, and GYPSY does not ever consider a tab to be a space character.  It is
// recommended that your editor be configured to convert tabs to spaces when
// editing Gypsy config files.
//
// Gypsy understands the following to be a list:
//
//     - one
//     - two
//     - three
//
// This is parsed as a `yaml.List`, and can be retrieved from the
// `yaml.Node.List()` method.  In this case, each element of the `yaml.List` would
// be a `yaml.Scalar` whose value can be retrieved with the `yaml.Scalar.String()`
// method.
//
// Gypsy understands the following to be a mapping:
//
//     key:     value
//     foo:     bar
//     running: away
//
// A mapping is an unordered list of `key:value` pairs.  All whitespace after the
// colon is stripped from the value and is used for alignment purposes during
// export.  If the value is not a list or a map, everything after the first
// non-space character until the end of the line is used as the `yaml.Scalar`
// value.
//
// Gypsy allows arbitrary nesting of maps inside lists, lists inside of maps, and
// maps and/or lists nested inside of themselves.
//
// A map inside of a list:
//
//     - name: John Smith
//       age:  42
//     - name: Jane Smith
//       age:  45
//
// A list inside of a map:
//
//     schools:
//       - Meadow Glen
//       - Forest Creek
//       - Shady Grove
//     libraries:
//       - Joseph Hollingsworth Memorial
//       - Andrew Keriman Memorial
//
// A list of lists:
//
//     - - one
//       - two
//       - three
//     - - un
//       - deux
//       - trois
//     - - ichi
//       - ni
//       - san
//
// A map of maps:
//
//     google:
//       company: Google, Inc.
//       ticker:  GOOG
//       url:     http://google.com/
//     yahoo:
//       company: Yahoo, Inc.
//       ticker:  YHOO
//       url:     http://yahoo.com/
//
// In the case of a map of maps, all sub-keys must be on subsequent lines and
// indented equally.  It is allowable for the first key/value to be on the same
// line if there is more than one key/value pair, but this is not recommended.
//
// Values can also be expressed in long form (leading whitespace of the first line
// is removed from it and all subsequent lines).  In the normal (baz) case,
// newlines are treated as spaces, all indentation is removed.  In the folded case
// (bar), newlines are treated as spaces, except pairs of newlines (e.g. a blank
// line) are treated as a single newline, only the indentation level of the first
// line is removed, and newlines at the end of indented lines are preserved.  In
// the verbatim (foo) case, only the indent at the level of the first line is
// stripped.  The example:
//
//     foo: |
//       lorem ipsum dolor
//       sit amet
//     bar: >
//       lorem ipsum
//
//         dolor
//
//       sit amet
//     baz:
//       lorem ipsum
//        dolor sit amet
//
// The YAML subset understood by Gypsy can be expressed (loosely) in the following
// grammar (not including comments):
//
//               OBJECT = MAPPING | SEQUENCE | SCALAR .
//         SHORT-OBJECT = SHORT-MAPPING | SHORT-SEQUENCE | SHORT-SCALAR .
//                  EOL = '\n'
//
//              MAPPING = { LONG-MAPPING | SHORT-MAPPING } .
//             SEQUENCE = { LONG-SEQUENCE | SHORT-SEQUENCE } .
//               SCALAR = { LONG-SCALAR | SHORT-SCALAR } .
//
//         LONG-MAPPING = { INDENT KEY ':' OBJECT EOL } .
//        SHORT-MAPPING = '{' KEY ':' SHORT-OBJECT { ',' KEY ':' SHORT-OBJECT } '}' EOL .
//
//        LONG-SEQUENCE = { INDENT '-' OBJECT EOL } EOL .
//       SHORT-SEQUENCE = '[' SHORT-OBJECT { ',' SHORT-OBJECT } ']' EOL .
//
//          LONG-SCALAR = ( '|' | '>' | ) EOL { INDENT SHORT-SCALAR EOL }
//         SHORT-SCALAR = { alpha | digit | punct | ' ' | '\t' } .
//
//                  KEY = { alpha | digit }
//               INDENT = { ' ' }
//
// Any line where the first non-space character is a sharp sign (#) is a comment.
// It will be ignored.
// Only full-line comments are allowed.
package yaml

// BUG(kevlar): Multi-line strings are currently not supported.
<|endoftext|>


<commit_before>// 29 december 2012
package main

import (
	"fmt"
	"os"
	"io"
	"path/filepath"
	"archive/zip"
	"strconv"
	"crypto/sha1"
	"encoding/hex"
	"bytes"
	"strings"
	"log"
)

var sha1hash = sha1.New()

func crc32match(zipcrc uint32, gamecrc string) bool {
	if gamecrc == "" {	// assume lack of CRC32 means do not check
		return true
	}
	n, err := strconv.ParseUint(gamecrc, 16, 32)
	if err != nil {
		log.Fatalf("string convert error reading crc32 (%q): %v", gamecrc, err)
	}
	return uint32(n) == zipcrc
}

func sha1check(zf *zip.File, expectstring string) (bool, error) {
	expected, err := hex.DecodeString(expectstring)
	if err != nil {
		log.Fatalf("hex decode error reading sha1 (%q): %v", expectstring, err)
	}

	f, err := zf.Open()
	if err != nil {
		return false, fmt.Errorf("could not open given zip file entry: %v", err)
	}
	defer f.Close()

	sha1hash.Reset()
	n, err := io.Copy(sha1hash, f)
	if err != nil {
		return false, fmt.Errorf("could not read given zip file entry: %v", err)
	}
	// TODO could we have integer size/signedness conversion failure here? zf.UncompressedSize is not an int64
	if n != int64(zf.UncompressedSize) {
		return false, fmt.Errorf("short read from zip file or write to hash but no error returned (expected %d bytes; got %d)", int64(zf.UncompressedSize), n)
	}

	return bytes.Equal(expected, sha1hash.Sum(nil)), nil
}

func (g *Game) filename_ROM(rompath string) string {
	return filepath.Join(rompath, g.Name + ".zip")
}

func (g *Game) checkIn(rompath string, roms map[string]*ROM) (bool, error) {
	zipname := g.filename_ROM(rompath)
	f, err := zip.OpenReader(zipname)
	if os.IsNotExist(err) {		// if the file does not exist, try the next rompath
		return false, nil
	}
	if err != nil {			// something different happened
		return false, fmt.Errorf("could not open zip file %s: %v", zipname, err)
	}
	defer f.Close()

	// true values will be written to this as we find valid ROMs
	// if the length of this does not equal the length of roms when we're done; we missed something and therefore something else is wrong
	var found = map[string]bool{}

	for _, file := range f.File {
		rom, ok := roms[file.Name]
		if !ok {				// not in archive
			return false, nil
		}
		if file.UncompressedSize != rom.Size {
			return false, nil
		}
		if !crc32match(file.CRC32, rom.CRC32) {
			return false, nil
		}
		good, err := sha1check(file, rom.SHA1)
		if err != nil {
			return false, fmt.Errorf("could not calculate SHA-1 sum of %s in %s: %v", g.Name, zipname, err)
		}
		if !good {
			return false, nil
		}
		found[file.Name] = true		// mark as done
	}

	// if we reached here everything we know about checked out, so if there are any leftover files in the game, that means something is wrong
	return len(roms) == len(found), nil
}

// remove all ROMs belonging to this set and its parents from the list
func (g *Game) strikeROMs(roms map[string]*ROM) {
	for _, rom := range g.ROMs {
		delete(roms, rom.Name)
	}
	for _, parent := range g.Parents {
		games[parent].strikeROMs(roms)
	}
}

func (g *Game) Find() (found bool, err error) {
	// did we find this already?
	if g.Found {
		return true, nil
	}

	// populate list of ROMs
	var roms = make(map[string]*ROM)
	for i := range g.ROMs {
		if g.ROMs[i].Status != nodump {	// otherwise games with known undumped ROMs will return "not found" because the map never depletes
			// some ROM sets (scregg, for instance) have trailing spaces in the filenames given in he XML file (dc0.c6, in this example)
			// TODO this will also remove leading spaces; is that correct?
			roms[strings.TrimSpace(g.ROMs[i].Name)] = &(g.ROMs[i])
		}
	}

	// find the parents and remove their ROMs rom the list
	for _, parent := range g.Parents {
		found, err := games[parent].Find()
		if err != nil {
			return false, fmt.Errorf("error finding parent %s: %v", parent, err)
		}
		// TODO really bail out?
		if !found {
			return false, err
		}
		games[parent].strikeROMs(roms)
	}

	if len(roms) == 0 {		// no ROMs left to check (either has no ROMs or is just a CHD after BIOSes)
		// TODO this will be changed when we start looking for CHDs
		g.Found = true
		return true, nil
	}

	// go through the directories, finding the right file
	for _, d := range dirs {
		found, err := g.checkIn(d, roms)
		if err != nil {
			return false, err
		}
		if found {
			g.Found = true
			g.ROMLoc = g.filename_ROM(d)
			return true, nil
		}
	}

	// nope
	return false, nil
}
<commit_msg>Leaving some notes to myself<commit_after>// 29 december 2012
package main

import (
	"fmt"
	"os"
	"io"
	"path/filepath"
	"archive/zip"
	"strconv"
	"crypto/sha1"
	"encoding/hex"
	"bytes"
	"strings"
	"log"
)

// TODO:
// - select one constructor syntax for the maps?
// - condense map[string]*ROM into a named type?

var sha1hash = sha1.New()

func crc32match(zipcrc uint32, gamecrc string) bool {
	if gamecrc == "" {	// assume lack of CRC32 means do not check
		return true
	}
	n, err := strconv.ParseUint(gamecrc, 16, 32)
	if err != nil {
		log.Fatalf("string convert error reading crc32 (%q): %v", gamecrc, err)
	}
	return uint32(n) == zipcrc
}

func sha1check(zf *zip.File, expectstring string) (bool, error) {
	expected, err := hex.DecodeString(expectstring)
	if err != nil {
		log.Fatalf("hex decode error reading sha1 (%q): %v", expectstring, err)
	}

	f, err := zf.Open()
	if err != nil {
		return false, fmt.Errorf("could not open given zip file entry: %v", err)
	}
	defer f.Close()

	sha1hash.Reset()
	n, err := io.Copy(sha1hash, f)
	if err != nil {
		return false, fmt.Errorf("could not read given zip file entry: %v", err)
	}
	// TODO could we have integer size/signedness conversion failure here? zf.UncompressedSize is not an int64
	if n != int64(zf.UncompressedSize) {
		return false, fmt.Errorf("short read from zip file or write to hash but no error returned (expected %d bytes; got %d)", int64(zf.UncompressedSize), n)
	}

	return bytes.Equal(expected, sha1hash.Sum(nil)), nil
}

func (g *Game) filename_ROM(rompath string) string {
	return filepath.Join(rompath, g.Name + ".zip")
}

func (g *Game) checkIn(rompath string, roms map[string]*ROM) (bool, error) {
	zipname := g.filename_ROM(rompath)
	f, err := zip.OpenReader(zipname)
	if os.IsNotExist(err) {		// if the file does not exist, try the next rompath
		return false, nil
	}
	if err != nil {			// something different happened
		return false, fmt.Errorf("could not open zip file %s: %v", zipname, err)
	}
	defer f.Close()

	// true values will be written to this as we find valid ROMs
	// if the length of this does not equal the length of roms when we're done; we missed something and therefore something else is wrong
	var found = map[string]bool{}

	for _, file := range f.File {
		rom, ok := roms[file.Name]
		if !ok {				// not in archive
			return false, nil
		}
		if file.UncompressedSize != rom.Size {
			return false, nil
		}
		if !crc32match(file.CRC32, rom.CRC32) {
			return false, nil
		}
		good, err := sha1check(file, rom.SHA1)
		if err != nil {
			return false, fmt.Errorf("could not calculate SHA-1 sum of %s in %s: %v", g.Name, zipname, err)
		}
		if !good {
			return false, nil
		}
		found[file.Name] = true		// mark as done
	}

	// if we reached here everything we know about checked out, so if there are any leftover files in the game, that means something is wrong
	return len(roms) == len(found), nil
}

// remove all ROMs belonging to this set and its parents from the list
func (g *Game) strikeROMs(roms map[string]*ROM) {
	for _, rom := range g.ROMs {
		delete(roms, rom.Name)
	}
	for _, parent := range g.Parents {
		games[parent].strikeROMs(roms)
	}
}

func (g *Game) Find() (found bool, err error) {
	// did we find this already?
	if g.Found {
		return true, nil
	}

	// populate list of ROMs
	var roms = make(map[string]*ROM)
	for i := range g.ROMs {
		if g.ROMs[i].Status != nodump {	// otherwise games with known undumped ROMs will return "not found" because the map never depletes
			// some ROM sets (scregg, for instance) have trailing spaces in the filenames given in he XML file (dc0.c6, in this example)
			// TODO this will also remove leading spaces; is that correct?
			roms[strings.TrimSpace(g.ROMs[i].Name)] = &(g.ROMs[i])
		}
	}

	// find the parents and remove their ROMs rom the list
	for _, parent := range g.Parents {
		found, err := games[parent].Find()
		if err != nil {
			return false, fmt.Errorf("error finding parent %s: %v", parent, err)
		}
		// TODO really bail out?
		if !found {
			return false, err
		}
		games[parent].strikeROMs(roms)
	}

	if len(roms) == 0 {		// no ROMs left to check (either has no ROMs or is just a CHD after BIOSes)
		// TODO this will be changed when we start looking for CHDs
		g.Found = true
		return true, nil
	}

	// go through the directories, finding the right file
	for _, d := range dirs {
		found, err := g.checkIn(d, roms)
		if err != nil {
			return false, err
		}
		if found {
			g.Found = true
			g.ROMLoc = g.filename_ROM(d)
			return true, nil
		}
	}

	// nope
	return false, nil
}
<|endoftext|>


<commit_before>//  Copyright (c) 2015 Rackspace
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
//  implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

/*
Package objectserver provides a Object Server implementation.
*/
package objectserver
<commit_msg>go: document replication<commit_after>//  Copyright (c) 2015 Rackspace
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
//  implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

/*
Package objectserver provides a Object Server implementation.

Hummingbird Replication

The hummingbird object server is backwards-compatible with the python-swift
replicator, but the hummingbird replicator has a few improvements that only
allow it to work with the hummingbird object server.

A replication ID is sent with all requests to associate the calls for a single
partition replication operation.  This helps eliminate a common problem where we
serve an expensive REPLICATE call, only to reject the subsequent rsync due to
concurrency limits.  The server can inexpensively reject the initial REPLICATE
call if it's already at the incoming replication concurrency limit.

The SYNC method replaces rsync for transmitting data.  It functions like a
simple PUT operation to the real destination file path relative to the root,
and can accept .data, .meta, or .ts files.  It looks something like:

	SYNC /sda/objects/123/fff/ffffffffffffffffffffffffffffffff/12345.6789.data HTTP/1.1
	X-Attrs: [hex-encoded xattr data]
	X-Replication-Id: [unique ID per partition replication]
	Content-Length: [size]
	Expect: 100-Continue

SYNC request responses:
	201: file was successfully uploaded
	400: there was a problem with the request
	409: this file or a newer one already exists
	500: mostly caused by filesystem errors
	507: disk unmounted or full

The X-Attrs header contains the hex-encoded raw pickled metadata from the
file's xattrs.

If the replicator receives a 409 response with the "Newer-File-Exists: true"
header, the file is stale and can be removed.  It's hoped that this will reduce
the incidence of orphaned object data that can occur with long-lived handoffs.

Since the SYNC method invalidates hashes, the followup REPLICATE call to force
the server to recalculate suffixes is no longer required.  However, an "end"
request is sent to inform the server that the replication pass has finished,
and it can remove that replication ID from its concurrency accounting.
*/
package objectserver
<|endoftext|>


<commit_before>package manifest_test

import (
	"bufio"
	"fmt"
	"strings"

	"github.com/docker-library/go-dockerlibrary/manifest"
)

func Example() {
	man, err := manifest.Parse(bufio.NewReader(strings.NewReader(`# RFC 2822

	# I LOVE CAKE

Maintainers: InfoSiftr <github@infosiftr.com> (@infosiftr),
             Johan Euphrosine <proppy@google.com> (@proppy)
GitRepo: https://github.com/docker-library/golang.git
GitFetch: refs/heads/master
SharedTags: latest

 # hi

 	 # blasphemer

# Go 1.6
Tags: 1.6.1, 1.6, 1
GitCommit: 0ce80411b9f41e9c3a21fc0a1bffba6ae761825a
Directory: 1.6

# Go 1.5
Tags: 1.5.3
SharedTags: 1.5.3-debian, 1.5-debian
GitCommit: d7e2a8d90a9b8f5dfd5bcd428e0c33b68c40cc19
Directory: 1.5

Tags: 1.5
SharedTags: 1.5-debian
GitCommit: d7e2a8d90a9b8f5dfd5bcd428e0c33b68c40cc19
Directory: 1.5

`)))
	if err != nil {
		panic(err)
	}
	fmt.Printf("-------------\n2822:\n%s\n", man)

	fmt.Printf("\nShared Tag Groups:\n")
	for _, group := range man.GetSharedTagGroups() {
		fmt.Printf("\n  - %s\n", strings.Join(group.SharedTags, ", "))
		for _, entry := range group.Entries {
			fmt.Printf("    - %s\n", entry.TagsString())
		}
	}
	fmt.Printf("\n")

	man, err = manifest.Parse(bufio.NewReader(strings.NewReader(`
# maintainer: InfoSiftr <github@infosiftr.com> (@infosiftr)
# maintainer: John Smith <jsmith@example.com> (@example-jsmith)

# first set
a: b@c d
e: b@c d

 # second set
f: g@h
i: g@h j
`)))
	if err != nil {
		panic(err)
	}
	fmt.Printf("-------------\nline-based:\n%v\n", man)

	// Output:
	// -------------
	// 2822:
	// Maintainers: InfoSiftr <github@infosiftr.com> (@infosiftr), Johan Euphrosine <proppy@google.com> (@proppy)
	// SharedTags: latest
	// GitRepo: https://github.com/docker-library/golang.git
	//
	// Tags: 1.6.1, 1.6, 1
	// GitCommit: 0ce80411b9f41e9c3a21fc0a1bffba6ae761825a
	// Directory: 1.6
	//
	// Tags: 1.5.3, 1.5
	// SharedTags: latest, 1.5.3-debian, 1.5-debian
	// GitCommit: d7e2a8d90a9b8f5dfd5bcd428e0c33b68c40cc19
	// Directory: 1.5
	//
	// Shared Tag Groups:
	//
	//   - latest
	//     - 1.6.1, 1.6, 1
	//     - 1.5.3, 1.5
	//
	//   - 1.5.3-debian, 1.5-debian
	//     - 1.5.3, 1.5
	//
	// -------------
	// line-based:
	// Maintainers: InfoSiftr <github@infosiftr.com> (@infosiftr), John Smith <jsmith@example.com> (@example-jsmith)
	// GitFetch: refs/heads/*
	//
	// Tags: a, e
	// GitRepo: b
	// GitCommit: c
	// Directory: d
	//
	// Tags: f
	// GitRepo: g
	// GitFetch: refs/tags/h
	// GitCommit: FETCH_HEAD
	//
	// Tags: i
	// GitRepo: g
	// GitFetch: refs/tags/h
	// GitCommit: FETCH_HEAD
	// Directory: j
}
<commit_msg>Add example Constraints for another 0.4% test coverage increase<commit_after>package manifest_test

import (
	"bufio"
	"fmt"
	"strings"

	"github.com/docker-library/go-dockerlibrary/manifest"
)

func Example() {
	man, err := manifest.Parse(bufio.NewReader(strings.NewReader(`# RFC 2822

	# I LOVE CAKE

Maintainers: InfoSiftr <github@infosiftr.com> (@infosiftr),
             Johan Euphrosine <proppy@google.com> (@proppy)
GitRepo: https://github.com/docker-library/golang.git
GitFetch: refs/heads/master
SharedTags: latest

 # hi

 	 # blasphemer

# Go 1.6
Tags: 1.6.1, 1.6, 1
GitCommit: 0ce80411b9f41e9c3a21fc0a1bffba6ae761825a
Directory: 1.6
Constraints: some-random-build-server

# Go 1.5
Tags: 1.5.3
SharedTags: 1.5.3-debian, 1.5-debian
GitCommit: d7e2a8d90a9b8f5dfd5bcd428e0c33b68c40cc19
Directory: 1.5

Tags: 1.5
SharedTags: 1.5-debian
GitCommit: d7e2a8d90a9b8f5dfd5bcd428e0c33b68c40cc19
Directory: 1.5

`)))
	if err != nil {
		panic(err)
	}
	fmt.Printf("-------------\n2822:\n%s\n", man)

	fmt.Printf("\nShared Tag Groups:\n")
	for _, group := range man.GetSharedTagGroups() {
		fmt.Printf("\n  - %s\n", strings.Join(group.SharedTags, ", "))
		for _, entry := range group.Entries {
			fmt.Printf("    - %s\n", entry.TagsString())
		}
	}
	fmt.Printf("\n")

	man, err = manifest.Parse(bufio.NewReader(strings.NewReader(`
# maintainer: InfoSiftr <github@infosiftr.com> (@infosiftr)
# maintainer: John Smith <jsmith@example.com> (@example-jsmith)

# first set
a: b@c d
e: b@c d

 # second set
f: g@h
i: g@h j
`)))
	if err != nil {
		panic(err)
	}
	fmt.Printf("-------------\nline-based:\n%v\n", man)

	// Output:
	// -------------
	// 2822:
	// Maintainers: InfoSiftr <github@infosiftr.com> (@infosiftr), Johan Euphrosine <proppy@google.com> (@proppy)
	// SharedTags: latest
	// GitRepo: https://github.com/docker-library/golang.git
	//
	// Tags: 1.6.1, 1.6, 1
	// GitCommit: 0ce80411b9f41e9c3a21fc0a1bffba6ae761825a
	// Directory: 1.6
	// Constraints: some-random-build-server
	//
	// Tags: 1.5.3, 1.5
	// SharedTags: latest, 1.5.3-debian, 1.5-debian
	// GitCommit: d7e2a8d90a9b8f5dfd5bcd428e0c33b68c40cc19
	// Directory: 1.5
	//
	// Shared Tag Groups:
	//
	//   - latest
	//     - 1.6.1, 1.6, 1
	//     - 1.5.3, 1.5
	//
	//   - 1.5.3-debian, 1.5-debian
	//     - 1.5.3, 1.5
	//
	// -------------
	// line-based:
	// Maintainers: InfoSiftr <github@infosiftr.com> (@infosiftr), John Smith <jsmith@example.com> (@example-jsmith)
	// GitFetch: refs/heads/*
	//
	// Tags: a, e
	// GitRepo: b
	// GitCommit: c
	// Directory: d
	//
	// Tags: f
	// GitRepo: g
	// GitFetch: refs/tags/h
	// GitCommit: FETCH_HEAD
	//
	// Tags: i
	// GitRepo: g
	// GitFetch: refs/tags/h
	// GitCommit: FETCH_HEAD
	// Directory: j
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package service

import (
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
	"github.com/keybase/go-framed-msgpack-rpc/rpc"
	"golang.org/x/net/context"
)

type SimpleFSHandler struct {
	*BaseHandler
	libkb.Contextified
}

func NewSimpleFSHandler(xp rpc.Transporter, g *libkb.GlobalContext) *SimpleFSHandler {
	return &SimpleFSHandler{
		BaseHandler:  NewBaseHandler(g, xp),
		Contextified: libkb.NewContextified(g),
	}
}

var _ keybase1.SimpleFSInterface = (*SimpleFSHandler)(nil)

func (s *SimpleFSHandler) client() (*keybase1.SimpleFSClient, error) {
	xp := s.G().ConnectionManager.LookupByClientType(keybase1.ClientType_KBFS)
	if xp == nil {
		return nil, libkb.KBFSNotRunningError{}
	}
	return &keybase1.SimpleFSClient{
		Cli: rpc.NewClient(xp, libkb.NewContextifiedErrorUnwrapper(s.G()), nil),
	}, nil
}

// SimpleFSList - Begin list of items in directory at path
// Retrieve results with readList()
// Can be a single file to get flags/status
func (s *SimpleFSHandler) SimpleFSList(ctx context.Context, arg keybase1.SimpleFSListArg) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSList(ctx, arg)
}

// SimpleFSListRecursive - Begin recursive list of items in directory at path
func (s *SimpleFSHandler) SimpleFSListRecursive(ctx context.Context, arg keybase1.SimpleFSListRecursiveArg) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSListRecursive(ctx, arg)
}

// SimpleFSListRecursiveToDepth - Begin recursive list of items in directory at
// path to a given depth.
func (s *SimpleFSHandler) SimpleFSListRecursiveToDepth(ctx context.Context, arg keybase1.SimpleFSListRecursiveToDepthArg) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSListRecursiveToDepth(ctx, arg)
}

// SimpleFSReadList - Get list of Paths in progress. Can indicate status of pending
// to get more entries.
func (s *SimpleFSHandler) SimpleFSReadList(ctx context.Context, arg keybase1.OpID) (keybase1.SimpleFSListResult, error) {
	cli, err := s.client()
	if err != nil {
		return keybase1.SimpleFSListResult{}, err
	}
	return cli.SimpleFSReadList(ctx, arg)
}

// SimpleFSCopy - Begin copy of file or directory
func (s *SimpleFSHandler) SimpleFSCopy(ctx context.Context, arg keybase1.SimpleFSCopyArg) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSCopy(ctx, arg)
}

// SimpleFSCopyRecursive - Begin recursive copy of directory
func (s *SimpleFSHandler) SimpleFSCopyRecursive(ctx context.Context, arg keybase1.SimpleFSCopyRecursiveArg) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSCopyRecursive(ctx, arg)
}

// SimpleFSMove - Begin move of file or directory, from/to KBFS only
func (s *SimpleFSHandler) SimpleFSMove(ctx context.Context, arg keybase1.SimpleFSMoveArg) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSMove(ctx, arg)
}

// SimpleFSRename - Rename file or directory, KBFS side only
func (s *SimpleFSHandler) SimpleFSRename(ctx context.Context, arg keybase1.SimpleFSRenameArg) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSRename(ctx, arg)
}

// SimpleFSSymlink - Make a symlink from KBFS to either elsewhere in KBFS or in the regular filesystem
func (s *SimpleFSHandler) SimpleFSSymlink(ctx context.Context, arg keybase1.SimpleFSSymlinkArg) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSSymlink(ctx, arg)
}

// SimpleFSOpen - Create/open a file and leave it open
// or create a directory
// Files must be closed afterwards.
func (s *SimpleFSHandler) SimpleFSOpen(ctx context.Context, arg keybase1.SimpleFSOpenArg) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSOpen(ctx, arg)
}

// SimpleFSSetStat - Set/clear file bits - only executable for now
func (s *SimpleFSHandler) SimpleFSSetStat(ctx context.Context, arg keybase1.SimpleFSSetStatArg) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSSetStat(ctx, arg)
}

// SimpleFSRead - Read (possibly partial) contents of open file,
// up to the amount specified by size.
// Repeat until zero bytes are returned or error.
// If size is zero, read an arbitrary amount.
func (s *SimpleFSHandler) SimpleFSRead(ctx context.Context, arg keybase1.SimpleFSReadArg) (keybase1.FileContent, error) {
	cli, err := s.client()
	if err != nil {
		return keybase1.FileContent{}, err
	}
	return cli.SimpleFSRead(ctx, arg)
}

// SimpleFSWrite - Append content to opened file.
// May be repeated until OpID is closed.
func (s *SimpleFSHandler) SimpleFSWrite(ctx context.Context, arg keybase1.SimpleFSWriteArg) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSWrite(ctx, arg)
}

// SimpleFSRemove - Remove file or directory from filesystem
func (s *SimpleFSHandler) SimpleFSRemove(ctx context.Context, arg keybase1.SimpleFSRemoveArg) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSRemove(ctx, arg)
}

// SimpleFSStat - Get info about file
func (s *SimpleFSHandler) SimpleFSStat(ctx context.Context, arg keybase1.SimpleFSStatArg) (keybase1.Dirent, error) {
	cli, err := s.client()
	if err != nil {
		return keybase1.Dirent{}, err
	}
	return cli.SimpleFSStat(ctx, arg)
}

// SimpleFSGetRevisions - Get revision info for a directory entry
func (s *SimpleFSHandler) SimpleFSGetRevisions(
	ctx context.Context, arg keybase1.SimpleFSGetRevisionsArg) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSGetRevisions(ctx, arg)
}

// SimpleFSReadRevisions - Get list of revisions in progress. Can
// indicate status of pending to get more entries.
func (s *SimpleFSHandler) SimpleFSReadRevisions(
	ctx context.Context, opID keybase1.OpID) (
	keybase1.GetRevisionsResult, error) {
	cli, err := s.client()
	if err != nil {
		return keybase1.GetRevisionsResult{}, err
	}
	return cli.SimpleFSReadRevisions(ctx, opID)
}

// SimpleFSMakeOpid - Convenience helper for generating new random value
func (s *SimpleFSHandler) SimpleFSMakeOpid(ctx context.Context) (keybase1.OpID, error) {
	cli, err := s.client()
	if err != nil {
		return keybase1.OpID{}, err
	}
	return cli.SimpleFSMakeOpid(ctx)
}

// SimpleFSClose - Close OpID, cancels any pending operation.
// Must be called after list/copy/remove
func (s *SimpleFSHandler) SimpleFSClose(ctx context.Context, arg keybase1.OpID) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSClose(ctx, arg)
}

// SimpleFSCancel - Cancels a running operation, like copy.
func (s *SimpleFSHandler) SimpleFSCancel(ctx context.Context, arg keybase1.OpID) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSCancel(ctx, arg)
}

// SimpleFSCheck - Check progress of pending operation
func (s *SimpleFSHandler) SimpleFSCheck(ctx context.Context, arg keybase1.OpID) (keybase1.OpProgress, error) {
	cli, err := s.client()
	if err != nil {
		return keybase1.OpProgress{}, err
	}
	return cli.SimpleFSCheck(ctx, arg)
}

// SimpleFSGetOps - Get all the outstanding operations
func (s *SimpleFSHandler) SimpleFSGetOps(ctx context.Context) ([]keybase1.OpDescription, error) {
	cli, err := s.client()
	if err != nil {
		return []keybase1.OpDescription{}, err
	}
	return cli.SimpleFSGetOps(ctx)
}

// SimpleFSWait - Blocking wait for the pending operation to finish
func (s *SimpleFSHandler) SimpleFSWait(ctx context.Context, arg keybase1.OpID) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSWait(ctx, arg)
}

// SimpleFSDumpDebuggingInfo - Instructs KBFS to dump debugging info
// into its logs.
func (s *SimpleFSHandler) SimpleFSDumpDebuggingInfo(ctx context.Context) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSDumpDebuggingInfo(ctx)
}

// SimpleFSSyncStatus - Get sync status.
func (s *SimpleFSHandler) SimpleFSSyncStatus(ctx context.Context, filter keybase1.ListFilter) (keybase1.FSSyncStatus, error) {
	cli, err := s.client()
	if err != nil {
		return keybase1.FSSyncStatus{}, err
	}
	return cli.SimpleFSSyncStatus(ctx, filter)
}

// SimpleFSGetHTTPAddressAndToken implements the SimpleFSInterface.
func (s *SimpleFSHandler) SimpleFSGetHTTPAddressAndToken(ctx context.Context) (keybase1.SimpleFSGetHTTPAddressAndTokenResponse, error) {
	cli, err := s.client()
	if err != nil {
		return keybase1.SimpleFSGetHTTPAddressAndTokenResponse{}, err
	}
	return cli.SimpleFSGetHTTPAddressAndToken(ctx)
}

// SimpleFSUserEditHistory implements the SimpleFSInterface.
func (s *SimpleFSHandler) SimpleFSUserEditHistory(ctx context.Context) (
	res []keybase1.FSFolderEditHistory, err error) {
	cli, err := s.client()
	if err != nil {
		return nil, err
	}
	return cli.SimpleFSUserEditHistory(ctx)
}

// SimpleFSFolderEditHistory implements the SimpleFSInterface.
func (s *SimpleFSHandler) SimpleFSFolderEditHistory(
	ctx context.Context, path keybase1.Path) (
	res keybase1.FSFolderEditHistory, err error) {
	cli, err := s.client()
	if err != nil {
		return keybase1.FSFolderEditHistory{}, err
	}
	return cli.SimpleFSFolderEditHistory(ctx, path)
}

// SimpleFSGetUserQuotaUsage implements the SimpleFSInterface.
func (s *SimpleFSHandler) SimpleFSGetUserQuotaUsage(ctx context.Context) (
	keybase1.SimpleFSQuotaUsage, error) {
	cli, err := s.client()
	if err != nil {
		return keybase1.SimpleFSQuotaUsage{}, err
	}
	return cli.SimpleFSGetUserQuotaUsage(ctx)
}
<commit_msg>add timeout on all SimpleFS RPCs except for SimpleFSWait (#14302)<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package service

import (
	"time"

	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
	"github.com/keybase/go-framed-msgpack-rpc/rpc"
	"golang.org/x/net/context"
)

// This is actually a very generous timeout. Fast operations like Stat should
// usually finish fairly quickly. Slow operations in SimpleFS are handled in an
// async manner, where the RPC starts the operation but does not wait for it to
// finish (there's SimpleFSWait). So all RPCs except for SimpleFSWait should
// return fairly fast, and if not, there might have been an unexpected hang due
// to bugs or networking issues. Either way we should just timeout to avoid
// piling up RPC sessions.
const simpleFSTimeout = 8 * time.Second

type SimpleFSHandler struct {
	*BaseHandler
	libkb.Contextified
}

func NewSimpleFSHandler(xp rpc.Transporter, g *libkb.GlobalContext) *SimpleFSHandler {
	return &SimpleFSHandler{
		BaseHandler:  NewBaseHandler(g, xp),
		Contextified: libkb.NewContextified(g),
	}
}

var _ keybase1.SimpleFSInterface = (*SimpleFSHandler)(nil)

func (s *SimpleFSHandler) wrapContextWithTimeout(ctx context.Context) (newCtx context.Context, cancel func()) {
	return context.WithTimeout(ctx, simpleFSTimeout)
}

func (s *SimpleFSHandler) client() (*keybase1.SimpleFSClient, error) {
	xp := s.G().ConnectionManager.LookupByClientType(keybase1.ClientType_KBFS)
	if xp == nil {
		return nil, libkb.KBFSNotRunningError{}
	}
	return &keybase1.SimpleFSClient{
		Cli: rpc.NewClient(xp, libkb.NewContextifiedErrorUnwrapper(s.G()), nil),
	}, nil
}

// SimpleFSList - Begin list of items in directory at path
// Retrieve results with readList()
// Can be a single file to get flags/status
func (s *SimpleFSHandler) SimpleFSList(ctx context.Context, arg keybase1.SimpleFSListArg) error {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSList(ctx, arg)
}

// SimpleFSListRecursive - Begin recursive list of items in directory at path
func (s *SimpleFSHandler) SimpleFSListRecursive(ctx context.Context, arg keybase1.SimpleFSListRecursiveArg) error {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSListRecursive(ctx, arg)
}

// SimpleFSListRecursiveToDepth - Begin recursive list of items in directory at
// path to a given depth.
func (s *SimpleFSHandler) SimpleFSListRecursiveToDepth(ctx context.Context, arg keybase1.SimpleFSListRecursiveToDepthArg) error {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSListRecursiveToDepth(ctx, arg)
}

// SimpleFSReadList - Get list of Paths in progress. Can indicate status of pending
// to get more entries.
func (s *SimpleFSHandler) SimpleFSReadList(ctx context.Context, arg keybase1.OpID) (keybase1.SimpleFSListResult, error) {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return keybase1.SimpleFSListResult{}, err
	}
	return cli.SimpleFSReadList(ctx, arg)
}

// SimpleFSCopy - Begin copy of file or directory
func (s *SimpleFSHandler) SimpleFSCopy(ctx context.Context, arg keybase1.SimpleFSCopyArg) error {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSCopy(ctx, arg)
}

// SimpleFSCopyRecursive - Begin recursive copy of directory
func (s *SimpleFSHandler) SimpleFSCopyRecursive(ctx context.Context, arg keybase1.SimpleFSCopyRecursiveArg) error {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSCopyRecursive(ctx, arg)
}

// SimpleFSMove - Begin move of file or directory, from/to KBFS only
func (s *SimpleFSHandler) SimpleFSMove(ctx context.Context, arg keybase1.SimpleFSMoveArg) error {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSMove(ctx, arg)
}

// SimpleFSRename - Rename file or directory, KBFS side only
func (s *SimpleFSHandler) SimpleFSRename(ctx context.Context, arg keybase1.SimpleFSRenameArg) error {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSRename(ctx, arg)
}

// SimpleFSSymlink - Make a symlink from KBFS to either elsewhere in KBFS or in the regular filesystem
func (s *SimpleFSHandler) SimpleFSSymlink(ctx context.Context, arg keybase1.SimpleFSSymlinkArg) error {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSSymlink(ctx, arg)
}

// SimpleFSOpen - Create/open a file and leave it open
// or create a directory
// Files must be closed afterwards.
func (s *SimpleFSHandler) SimpleFSOpen(ctx context.Context, arg keybase1.SimpleFSOpenArg) error {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSOpen(ctx, arg)
}

// SimpleFSSetStat - Set/clear file bits - only executable for now
func (s *SimpleFSHandler) SimpleFSSetStat(ctx context.Context, arg keybase1.SimpleFSSetStatArg) error {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSSetStat(ctx, arg)
}

// SimpleFSRead - Read (possibly partial) contents of open file,
// up to the amount specified by size.
// Repeat until zero bytes are returned or error.
// If size is zero, read an arbitrary amount.
func (s *SimpleFSHandler) SimpleFSRead(ctx context.Context, arg keybase1.SimpleFSReadArg) (keybase1.FileContent, error) {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return keybase1.FileContent{}, err
	}
	return cli.SimpleFSRead(ctx, arg)
}

// SimpleFSWrite - Append content to opened file.
// May be repeated until OpID is closed.
func (s *SimpleFSHandler) SimpleFSWrite(ctx context.Context, arg keybase1.SimpleFSWriteArg) error {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSWrite(ctx, arg)
}

// SimpleFSRemove - Remove file or directory from filesystem
func (s *SimpleFSHandler) SimpleFSRemove(ctx context.Context, arg keybase1.SimpleFSRemoveArg) error {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSRemove(ctx, arg)
}

// SimpleFSStat - Get info about file
func (s *SimpleFSHandler) SimpleFSStat(ctx context.Context, arg keybase1.SimpleFSStatArg) (keybase1.Dirent, error) {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return keybase1.Dirent{}, err
	}
	return cli.SimpleFSStat(ctx, arg)
}

// SimpleFSGetRevisions - Get revision info for a directory entry
func (s *SimpleFSHandler) SimpleFSGetRevisions(
	ctx context.Context, arg keybase1.SimpleFSGetRevisionsArg) error {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSGetRevisions(ctx, arg)
}

// SimpleFSReadRevisions - Get list of revisions in progress. Can
// indicate status of pending to get more entries.
func (s *SimpleFSHandler) SimpleFSReadRevisions(
	ctx context.Context, opID keybase1.OpID) (
	keybase1.GetRevisionsResult, error) {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return keybase1.GetRevisionsResult{}, err
	}
	return cli.SimpleFSReadRevisions(ctx, opID)
}

// SimpleFSMakeOpid - Convenience helper for generating new random value
func (s *SimpleFSHandler) SimpleFSMakeOpid(ctx context.Context) (keybase1.OpID, error) {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return keybase1.OpID{}, err
	}
	return cli.SimpleFSMakeOpid(ctx)
}

// SimpleFSClose - Close OpID, cancels any pending operation.
// Must be called after list/copy/remove
func (s *SimpleFSHandler) SimpleFSClose(ctx context.Context, arg keybase1.OpID) error {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSClose(ctx, arg)
}

// SimpleFSCancel - Cancels a running operation, like copy.
func (s *SimpleFSHandler) SimpleFSCancel(ctx context.Context, arg keybase1.OpID) error {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSCancel(ctx, arg)
}

// SimpleFSCheck - Check progress of pending operation
func (s *SimpleFSHandler) SimpleFSCheck(ctx context.Context, arg keybase1.OpID) (keybase1.OpProgress, error) {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return keybase1.OpProgress{}, err
	}
	return cli.SimpleFSCheck(ctx, arg)
}

// SimpleFSGetOps - Get all the outstanding operations
func (s *SimpleFSHandler) SimpleFSGetOps(ctx context.Context) ([]keybase1.OpDescription, error) {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return []keybase1.OpDescription{}, err
	}
	return cli.SimpleFSGetOps(ctx)
}

// SimpleFSWait - Blocking wait for the pending operation to finish
func (s *SimpleFSHandler) SimpleFSWait(ctx context.Context, arg keybase1.OpID) error {
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSWait(ctx, arg)
}

// SimpleFSDumpDebuggingInfo - Instructs KBFS to dump debugging info
// into its logs.
func (s *SimpleFSHandler) SimpleFSDumpDebuggingInfo(ctx context.Context) error {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return err
	}
	return cli.SimpleFSDumpDebuggingInfo(ctx)
}

// SimpleFSSyncStatus - Get sync status.
func (s *SimpleFSHandler) SimpleFSSyncStatus(ctx context.Context, filter keybase1.ListFilter) (keybase1.FSSyncStatus, error) {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return keybase1.FSSyncStatus{}, err
	}
	return cli.SimpleFSSyncStatus(ctx, filter)
}

// SimpleFSGetHTTPAddressAndToken implements the SimpleFSInterface.
func (s *SimpleFSHandler) SimpleFSGetHTTPAddressAndToken(ctx context.Context) (keybase1.SimpleFSGetHTTPAddressAndTokenResponse, error) {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return keybase1.SimpleFSGetHTTPAddressAndTokenResponse{}, err
	}
	return cli.SimpleFSGetHTTPAddressAndToken(ctx)
}

// SimpleFSUserEditHistory implements the SimpleFSInterface.
func (s *SimpleFSHandler) SimpleFSUserEditHistory(ctx context.Context) (
	res []keybase1.FSFolderEditHistory, err error) {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return nil, err
	}
	return cli.SimpleFSUserEditHistory(ctx)
}

// SimpleFSFolderEditHistory implements the SimpleFSInterface.
func (s *SimpleFSHandler) SimpleFSFolderEditHistory(
	ctx context.Context, path keybase1.Path) (
	res keybase1.FSFolderEditHistory, err error) {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return keybase1.FSFolderEditHistory{}, err
	}
	return cli.SimpleFSFolderEditHistory(ctx, path)
}

// SimpleFSGetUserQuotaUsage implements the SimpleFSInterface.
func (s *SimpleFSHandler) SimpleFSGetUserQuotaUsage(ctx context.Context) (
	keybase1.SimpleFSQuotaUsage, error) {
	ctx, cancel := s.wrapContextWithTimeout(ctx)
	defer cancel()
	cli, err := s.client()
	if err != nil {
		return keybase1.SimpleFSQuotaUsage{}, err
	}
	return cli.SimpleFSGetUserQuotaUsage(ctx)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"github.com/go-martini/martini"
	"github.com/martini-contrib/cors"
	"github.com/soveran/redisurl"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"text/template"
	"time"
)

// Expire caches in an hour
const CACHE_TTL = 60 * 60

type Server struct {
	Redis   *redis.Conn
	Martini *martini.ClassicMartini
}

type Score struct {
	TotalScore     float32              `json:"total_score"`
	Breakdown      map[string]float32   `json:"breakdown"`
	HumanBreakdown map[string][]float32 `json:"human_breakdown"`
}

type ScoreResponse struct {
	Score     float32            `json:"score"`
	URL       string             `json:"url"`
	Breakdown map[string]float32 `json:"breakdown"`
}

type HumanScoreResponse struct {
	Score     float32              `json:"score"`
	URL       string               `json:"url"`
	Breakdown map[string][]float32 `json:"breakdown"`
}

type ScoreSVG struct {
	ThreeDigitLayout  bool
	SingleDigitLayout bool
	Value             string
	Color             string
}

type ErrorResponse struct {
	Error string `json:"error"`
}

func MarshalToJsonBytes(res interface{}) []byte {
	resAsJson, _ := json.Marshal(res)
	return ([]byte(resAsJson))
}

func GetScoreResponseAsJson(score Score, url_or_slug string, human_breakdown bool) []byte {
	var res interface{}

	if human_breakdown {
		res = &HumanScoreResponse{
			Score:     score.TotalScore,
			Breakdown: score.HumanBreakdown,
			URL:       url_or_slug}
	} else {
		res = &ScoreResponse{
			Score:     score.TotalScore,
			Breakdown: score.Breakdown,
			URL:       url_or_slug}
	}

	return MarshalToJsonBytes(res)
}

func (score Score) AsColor() string {
	if score.TotalScore < 25 {
		return "#E74C3C"
	}
	if score.TotalScore < 80 {
		return "#F39C12"
	}
	return "#2ECC71"
}

func (score Score) AsScoreTemplate() ScoreSVG {
	return ScoreSVG{
		ThreeDigitLayout:  score.TotalScore >= 100,
		SingleDigitLayout: score.TotalScore < 10,
		Value:             strconv.Itoa(int(score.TotalScore)),
		Color:             score.AsColor(),
	}
}

func GetScoreResponseAsSVG(score_svg ScoreSVG) []byte {
	var score_template_string = ""
	var score_template = template.New("score template")

	var doc bytes.Buffer
	var err error

	if score_template_string == "" {
		var score_template_bytes []byte
		if score_template_bytes, err = ioutil.ReadFile("./templates/score.svg"); err == nil {
			score_template_string = string(score_template_bytes)
		}
		score_template, err = score_template.Parse(score_template_string)
	}

	if err == nil {
		err = score_template.Execute(&doc, score_svg)
	}
	HandleError(err)

	return doc.Bytes()
}

func GetScoreErrorAsSVG() []byte {
	return GetScoreResponseAsSVG(ScoreSVG{
		Value: "Err",
		Color: "#838383",
	})
}

func GetScoreErrorAsJson(url_or_slug string) []byte {
	res := &ErrorResponse{
		Error: "Could not determine score for " + url_or_slug}
	return MarshalToJsonBytes(res)
}

func CacheKeyForUrlOrSlug(url_or_slug string) string {
	return "url_or_slug_v4:" + url_or_slug
}

func WriteSVGWithETag(res http.ResponseWriter, body []byte) {
	hash := md5.New()
	io.WriteString(hash, string(body))
	etag := fmt.Sprintf("\"%x\"", hash.Sum(nil))
	res.Header().Set("ETag", etag)
	res.Write(body)
}

func (server *Server) GetScore(res http.ResponseWriter, req *http.Request, params martini.Params) {
	query_params := req.URL.Query()
	url_or_slug := ""
	ok := false
	human_breakdown := false
	hit_cache := true
	format := params["format"]
	if format == "svg" {
		res.Header().Set("Content-Type", "image/svg+xml")
		res.Header().Set("Cache-Control", "no-cache, private")
	} else if format == "txt" {
		res.Header().Set("Content-Type", "text/plain")
	} else {
		res.Header().Set("Content-Type", "application/json")
	}
	var param_matches []string
	var score *Score
	var err error

	if param_matches, ok = query_params["url"]; !ok {
		param_matches = query_params["github"]
	}
	if len(param_matches) == 0 {
		err = errors.New("No value for :url or :github query parameter")
	}

	if err == nil {
		url_or_slug = param_matches[0]

		if param_matches, ok = query_params["human_breakdown"]; ok {
			human_breakdown = param_matches[0] == "true"
		}

		if param_matches, ok = query_params["force"]; ok {
			hit_cache = false
		}

		score, err = server.GetScoreForUrlOrSlug(url_or_slug, hit_cache)

	}
	HandleError(err)

	if score == nil {
		if format == "svg" {
			WriteSVGWithETag(res, GetScoreErrorAsSVG())
		} else if format == "txt" {
			res.Write([]byte("error"))
		} else {
			res.Write(GetScoreErrorAsJson(url_or_slug))
		}
	} else {
		if format == "svg" {
			WriteSVGWithETag(res, GetScoreResponseAsSVG(score.AsScoreTemplate()))
		} else if format == "txt" {
			res.Write([]byte(strconv.Itoa(int(score.TotalScore))))
		} else {
			res.Write(GetScoreResponseAsJson(*score, url_or_slug, human_breakdown))
		}
	}

}

func (server *Server) GetCachedScoreForUrlOrSlug(url_or_slug string) (*Score, error) {
	var score *Score
	scoreJson, err := redis.String((*server.Redis).Do("GET", CacheKeyForUrlOrSlug(url_or_slug)))
	if scoreJson != "" {
		score = &Score{}
		if err = json.Unmarshal([]byte(scoreJson), &score); err != nil {
			score = nil
		}
	}

	return score, err
}

func (server *Server) CacheScoreForUrlOrSlug(scoreJson string, url_or_slug string) {
	(*server.Redis).Do("SET", CacheKeyForUrlOrSlug(url_or_slug), scoreJson)
	(*server.Redis).Do("EXPIRE", CacheKeyForUrlOrSlug(url_or_slug), CACHE_TTL)
}

func (server *Server) GetScoreForUrlOrSlug(url_or_slug string, hit_cache bool) (*Score, error) {
	var score *Score
	var err error
	if score, err = server.GetCachedScoreForUrlOrSlug(url_or_slug); err != nil || !hit_cache {
		rubyCmd := exec.Command("./get_score.rb", url_or_slug)
		var scoreOut []byte
		if scoreOut, err = rubyCmd.Output(); err == nil {
			lines := strings.Split(string(scoreOut), "\n")
			scoreJson := lines[len(lines)-2]
			server.CacheScoreForUrlOrSlug(scoreJson, url_or_slug)
			score = &Score{}
			if err = json.Unmarshal([]byte(scoreJson), &score); err != nil {
				score = nil
			}
		}
	}

	return score, err
}

func ConnectRedis(redisChannel chan redis.Conn) {
	redisAddress := os.Getenv("REDIS_URL")
	if redisAddress == "" {
		redisAddress = os.Getenv("REDISCLOUD_URL")
		if redisAddress == "" {
			redisAddress = "redis://localhost:6379"
		}
	}

	connection, redisError := redisurl.ConnectToURL(redisAddress)
	if connection != nil {
		redisChannel <- connection
	} else if redisError != nil {
		fmt.Println(redisError)
	} else {
		fmt.Println("Everyting was nil?")
	}
}

func CreateServer(server *Server) {
	m := martini.Classic()
	m.Use(cors.Allow(&cors.Options{
		AllowOrigins:     []string{"*"},
		AllowMethods:     []string{"GET"},
		ExposeHeaders:    []string{"Content-Type, Cache-Control, Expires, Etag, Last-Modified"},
		AllowCredentials: true,
	}))
	m.Get("/score(\\.(?P<format>json|html|svg|txt))?", server.GetScore)
	server.Martini = m
	m.Run()
}

func Start(redisChannel chan redis.Conn, retryCount int, server *Server) {
	go ConnectRedis(redisChannel)
	select {
	case redisConnection := <-redisChannel:
		server.Redis = &redisConnection
		fmt.Println("Redis connected")
		defer redisConnection.Close()
		CreateServer(server)
	case <-time.After(time.Second * 1):
		retryCount += 1
		if retryCount < 5 {
			fmt.Println("Retrying Redis connection")
			Start(redisChannel, retryCount, server)
		} else {
			panic("Something is going wrong")
		}
	}

}

func main() {
	redisChannel := make(chan redis.Conn, 1)
	retryCount := 0
	server := Server{}
	Start(redisChannel, retryCount, &server)
}
<commit_msg>don't load the template every request doh<commit_after>package main

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"github.com/go-martini/martini"
	"github.com/martini-contrib/cors"
	"github.com/soveran/redisurl"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"text/template"
	"time"
)

// Expire caches in an hour
const CACHE_TTL = 60 * 60

type Server struct {
	Redis   *redis.Conn
	Martini *martini.ClassicMartini
}

type Score struct {
	TotalScore     float32              `json:"total_score"`
	Breakdown      map[string]float32   `json:"breakdown"`
	HumanBreakdown map[string][]float32 `json:"human_breakdown"`
}

type ScoreResponse struct {
	Score     float32            `json:"score"`
	URL       string             `json:"url"`
	Breakdown map[string]float32 `json:"breakdown"`
}

type HumanScoreResponse struct {
	Score     float32              `json:"score"`
	URL       string               `json:"url"`
	Breakdown map[string][]float32 `json:"breakdown"`
}

type ScoreSVG struct {
	ThreeDigitLayout  bool
	SingleDigitLayout bool
	Value             string
	Color             string
}

type ErrorResponse struct {
	Error string `json:"error"`
}

func MarshalToJsonBytes(res interface{}) []byte {
	resAsJson, _ := json.Marshal(res)
	return ([]byte(resAsJson))
}

func GetScoreResponseAsJson(score Score, url_or_slug string, human_breakdown bool) []byte {
	var res interface{}

	if human_breakdown {
		res = &HumanScoreResponse{
			Score:     score.TotalScore,
			Breakdown: score.HumanBreakdown,
			URL:       url_or_slug}
	} else {
		res = &ScoreResponse{
			Score:     score.TotalScore,
			Breakdown: score.Breakdown,
			URL:       url_or_slug}
	}

	return MarshalToJsonBytes(res)
}

func (score Score) AsColor() string {
	if score.TotalScore < 25 {
		return "#E74C3C"
	}
	if score.TotalScore < 80 {
		return "#F39C12"
	}
	return "#2ECC71"
}

func (score Score) AsScoreTemplate() ScoreSVG {
	return ScoreSVG{
		ThreeDigitLayout:  score.TotalScore >= 100,
		SingleDigitLayout: score.TotalScore < 10,
		Value:             strconv.Itoa(int(score.TotalScore)),
		Color:             score.AsColor(),
	}
}

var score_template_string = ""
var score_template = template.New("score template")

func GetScoreResponseAsSVG(score_svg ScoreSVG) []byte {
	var doc bytes.Buffer
	var err error

	if score_template_string == "" {
		var score_template_bytes []byte
		if score_template_bytes, err = ioutil.ReadFile("./templates/score.svg"); err == nil {
			score_template_string = string(score_template_bytes)
		}
		score_template, err = score_template.Parse(score_template_string)
	}

	if err == nil {
		err = score_template.Execute(&doc, score_svg)
	}
	HandleError(err)

	return doc.Bytes()
}

func GetScoreErrorAsSVG() []byte {
	return GetScoreResponseAsSVG(ScoreSVG{
		Value: "Err",
		Color: "#838383",
	})
}

func GetScoreErrorAsJson(url_or_slug string) []byte {
	res := &ErrorResponse{
		Error: "Could not determine score for " + url_or_slug}
	return MarshalToJsonBytes(res)
}

func CacheKeyForUrlOrSlug(url_or_slug string) string {
	return "url_or_slug_v4:" + url_or_slug
}

func WriteSVGWithETag(res http.ResponseWriter, body []byte) {
	hash := md5.New()
	io.WriteString(hash, string(body))
	etag := fmt.Sprintf("\"%x\"", hash.Sum(nil))
	res.Header().Set("ETag", etag)
	res.Write(body)
}

func (server *Server) GetScore(res http.ResponseWriter, req *http.Request, params martini.Params) {
	query_params := req.URL.Query()
	url_or_slug := ""
	ok := false
	human_breakdown := false
	hit_cache := true
	format := params["format"]
	if format == "svg" {
		res.Header().Set("Content-Type", "image/svg+xml")
		res.Header().Set("Cache-Control", "no-cache, private")
	} else if format == "txt" {
		res.Header().Set("Content-Type", "text/plain")
	} else {
		res.Header().Set("Content-Type", "application/json")
	}
	var param_matches []string
	var score *Score
	var err error

	if param_matches, ok = query_params["url"]; !ok {
		param_matches = query_params["github"]
	}
	if len(param_matches) == 0 {
		err = errors.New("No value for :url or :github query parameter")
	}

	if err == nil {
		url_or_slug = param_matches[0]

		if param_matches, ok = query_params["human_breakdown"]; ok {
			human_breakdown = param_matches[0] == "true"
		}

		if param_matches, ok = query_params["force"]; ok {
			hit_cache = false
		}

		score, err = server.GetScoreForUrlOrSlug(url_or_slug, hit_cache)

	}
	HandleError(err)

	if score == nil {
		if format == "svg" {
			WriteSVGWithETag(res, GetScoreErrorAsSVG())
		} else if format == "txt" {
			res.Write([]byte("error"))
		} else {
			res.Write(GetScoreErrorAsJson(url_or_slug))
		}
	} else {
		if format == "svg" {
			WriteSVGWithETag(res, GetScoreResponseAsSVG(score.AsScoreTemplate()))
		} else if format == "txt" {
			res.Write([]byte(strconv.Itoa(int(score.TotalScore))))
		} else {
			res.Write(GetScoreResponseAsJson(*score, url_or_slug, human_breakdown))
		}
	}

}

func (server *Server) GetCachedScoreForUrlOrSlug(url_or_slug string) (*Score, error) {
	var score *Score
	scoreJson, err := redis.String((*server.Redis).Do("GET", CacheKeyForUrlOrSlug(url_or_slug)))
	if scoreJson != "" {
		score = &Score{}
		if err = json.Unmarshal([]byte(scoreJson), &score); err != nil {
			score = nil
		}
	}

	return score, err
}

func (server *Server) CacheScoreForUrlOrSlug(scoreJson string, url_or_slug string) {
	(*server.Redis).Do("SET", CacheKeyForUrlOrSlug(url_or_slug), scoreJson)
	(*server.Redis).Do("EXPIRE", CacheKeyForUrlOrSlug(url_or_slug), CACHE_TTL)
}

func (server *Server) GetScoreForUrlOrSlug(url_or_slug string, hit_cache bool) (*Score, error) {
	var score *Score
	var err error
	if score, err = server.GetCachedScoreForUrlOrSlug(url_or_slug); err != nil || !hit_cache {
		log.Printf("Cache miss for %s", url_or_slug)
		rubyCmd := exec.Command("./get_score.rb", url_or_slug)
		var scoreOut []byte
		if scoreOut, err = rubyCmd.Output(); err == nil {
			lines := strings.Split(string(scoreOut), "\n")
			scoreJson := lines[len(lines)-2]
			server.CacheScoreForUrlOrSlug(scoreJson, url_or_slug)
			score = &Score{}
			if err = json.Unmarshal([]byte(scoreJson), &score); err != nil {
				score = nil
			}
		}
	}

	return score, err
}

func ConnectRedis(redisChannel chan redis.Conn) {
	redisAddress := os.Getenv("REDIS_URL")
	if redisAddress == "" {
		redisAddress = os.Getenv("REDISCLOUD_URL")
		if redisAddress == "" {
			redisAddress = "redis://localhost:6379"
		}
	}

	connection, redisError := redisurl.ConnectToURL(redisAddress)
	if connection != nil {
		redisChannel <- connection
	} else if redisError != nil {
		fmt.Println(redisError)
	} else {
		fmt.Println("Everyting was nil?")
	}
}

func CreateServer(server *Server) {
	m := martini.Classic()
	m.Use(cors.Allow(&cors.Options{
		AllowOrigins:     []string{"*"},
		AllowMethods:     []string{"GET"},
		ExposeHeaders:    []string{"Content-Type, Cache-Control, Expires, Etag, Last-Modified"},
		AllowCredentials: true,
	}))
	m.Get("/score(\\.(?P<format>json|html|svg|txt))?", server.GetScore)
	server.Martini = m
	m.Run()
}

func Start(redisChannel chan redis.Conn, retryCount int, server *Server) {
	go ConnectRedis(redisChannel)
	select {
	case redisConnection := <-redisChannel:
		server.Redis = &redisConnection
		fmt.Println("Redis connected")
		defer redisConnection.Close()
		CreateServer(server)
	case <-time.After(time.Second * 1):
		retryCount += 1
		if retryCount < 5 {
			fmt.Println("Retrying Redis connection")
			Start(redisChannel, retryCount, server)
		} else {
			panic("Something is going wrong")
		}
	}

}

func main() {
	redisChannel := make(chan redis.Conn, 1)
	retryCount := 0
	server := Server{}
	Start(redisChannel, retryCount, &server)
}
<|endoftext|>


<commit_before>package cliwriter

import "github.com/fatih/color"
import "os"
import "fmt"

var errWriter *color.Color
var tipWriter *color.Color
var successWriter *color.Color

func init() {
	errWriter = color.New(color.FgRed)
	tipWriter = color.New(color.FgBlue)
	successWriter = color.New(color.FgGreen)
}

func Errorf(format string, args ...interface{}) {
	errWriter.Fprintf(os.Stderr, format, args)
}
func Error(content string) {
	errWriter.Fprint(os.Stderr, content)
}
func Errorln(content string) {
	errWriter.Fprintln(os.Stderr, content)
}
func Infof(format string, args ...interface{}) {
	fmt.Printf(format, args)
}
func Info(content string) {
	fmt.Print(content)
}
func Infoln(content string) {
	fmt.Println(content)
}

func Tipf(format string, args ...interface{}) {
	tipWriter.Fprintf(os.Stdout, format)
}
func Tip(content string) {
	tipWriter.Fprint(os.Stdout, content)
}
func Tipln(content string) {
	tipWriter.Fprintln(os.Stdout, content)
}

func Successf(format string, args ...interface{}) {
	successWriter.Fprintf(os.Stdout, format, args)
}

func Success(content string) {
	successWriter.Fprint(os.Stdout, content)
}

func Successln(content string) {
	successWriter.Fprintln(os.Stdout, content)
}
<commit_msg>cliwriter包新增内容前缀支持，新增颜色重置<commit_after>package cliwriter

import "github.com/fatih/color"
import "os"
import "fmt"

var (
	errWriter   *color.Color
	ErrorColor  = color.FgRed
	ErrorPrefix = ""

	tipWriter *color.Color
	TipColor  = color.FgBlue
	TipPrefix = "==> "

	successWriter *color.Color
	SuccessPrefix = ""
	SuccessColor  = color.FgGreen
)

func init() {
	errWriter = color.New(color.FgRed)
	tipWriter = color.New(color.FgBlue)
	successWriter = color.New(color.FgGreen)
}

func ResetColor() {
	errWriter.Add(ErrorColor)

	tipWriter.Add(TipColor)

	successWriter.Add(SuccessColor)
}

func Errorf(format string, args ...interface{}) {
	errWriter.Fprintf(os.Stderr, ErrorPrefix+format, args)
}
func Error(content string) {
	errWriter.Fprint(os.Stderr, ErrorPrefix+content)
}
func Errorln(content string) {
	errWriter.Fprintln(os.Stderr, ErrorPrefix+content)
}
func Infof(format string, args ...interface{}) {
	fmt.Printf(format, args)
}
func Info(content string) {
	fmt.Print(content)
}
func Infoln(content string) {
	fmt.Println(content)
}

func Tipf(format string, args ...interface{}) {
	tipWriter.Fprintf(os.Stdout, TipPrefix+format, args)
}
func Tip(content string) {
	tipWriter.Fprint(os.Stdout, TipPrefix+content)
}
func Tipln(content string) {
	tipWriter.Fprintln(os.Stdout, TipPrefix+content)
}

func Successf(format string, args ...interface{}) {
	successWriter.Fprintf(os.Stdout, SuccessPrefix+format, args)
}

func Success(content string) {
	successWriter.Fprint(os.Stdout, SuccessPrefix+content)
}

func Successln(content string) {
	successWriter.Fprintln(os.Stdout, SuccessPrefix+content)
}
<|endoftext|>


<commit_before>// clock counts down to or up from a target time.
package main

import (
	"fmt"
	"os"
	"time"
)

func main() {
	target := time.Date(2017, 6, 21, 0, 24, 7, 0, time.Local)
	motto := "Just Go"
	printTargetTime(target, motto)
	exitOnEnterKey()

	var previous time.Time
	for {
		now := time.Now().Truncate(time.Second)
		if now != previous {
			previous = now
			countdown := now.Sub(target) // Negative times are before the target
			printCountdown(now, countdown)
		}
		time.Sleep(100 * time.Millisecond)
	}
}

func exitOnEnterKey() {
	go func() {
		buf := make([]byte, 1)
		_, _ = os.Stdin.Read(buf)
		os.Exit(0)
	}()
}

const (
	indent         = "\t"
	highlightStart = "\x1b[1;35m"
	highlightEnd   = "\x1b[0m"
)

func printTargetTime(target time.Time, motto string) {
	fmt.Print(indent, highlightStart, motto, highlightEnd, "\n")
	fmt.Print(indent, target.Format(time.UnixDate), "\n")
}

func printCountdown(now time.Time, countdown time.Duration) {
	var sign string
	if countdown >= 0 {
		sign = "+"
	} else {
		sign = "-"
		countdown = -countdown
	}

	days := int(countdown / (24 * time.Hour))
	countdown = countdown % (24 * time.Hour)

	fmt.Print(indent, now.Format(time.UnixDate), "  ", sign)
	if days > 0 {
		fmt.Print(days, "d")
	}
	fmt.Print(countdown, "          \r")
	os.Stdout.Sync()
}
<commit_msg>7/1/17<commit_after>// clock counts down to or up from a target time.
package main

import (
	"fmt"
	"os"
	"time"
)

func main() {
	target := time.Date(2017, 7, 1, 0, 0, 0, 0, time.Local)
	motto := "Just Go"
	printTargetTime(target, motto)
	exitOnEnterKey()

	var previous time.Time
	for {
		now := time.Now().Truncate(time.Second)
		if now != previous {
			previous = now
			countdown := now.Sub(target) // Negative times are before the target
			printCountdown(now, countdown)
		}
		time.Sleep(100 * time.Millisecond)
	}
}

func exitOnEnterKey() {
	go func() {
		buf := make([]byte, 1)
		_, _ = os.Stdin.Read(buf)
		os.Exit(0)
	}()
}

const (
	indent         = "\t"
	highlightStart = "\x1b[1;35m"
	highlightEnd   = "\x1b[0m"
)

func printTargetTime(target time.Time, motto string) {
	fmt.Print(indent, highlightStart, motto, highlightEnd, "\n")
	fmt.Print(indent, target.Format(time.UnixDate), "\n")
}

func printCountdown(now time.Time, countdown time.Duration) {
	var sign string
	if countdown >= 0 {
		sign = "+"
	} else {
		sign = "-"
		countdown = -countdown
	}

	days := int(countdown / (24 * time.Hour))
	countdown = countdown % (24 * time.Hour)

	fmt.Print(indent, now.Format(time.UnixDate), "  ", sign)
	if days > 0 {
		fmt.Print(days, "d")
	}
	fmt.Print(countdown, "          \r")
	os.Stdout.Sync()
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package vtctld contains all the code to expose a vtctld server
// based on the provided topo.Server.
package vtctld

import (
	"context"
	"flag"
	"net/http"
	"strings"

	"vitess.io/vitess/go/acl"
	"vitess.io/vitess/go/vt/discovery"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/topo"
	"vitess.io/vitess/go/vt/vtctl"
	"vitess.io/vitess/go/vt/wrangler"
	"vitess.io/vitess/web/vtctld2"

	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
	vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata"
)

var (
	enableRealtimeStats = flag.Bool("enable_realtime_stats", false, "Required for the Realtime Stats view. If set, vtctld will maintain a streaming RPC to each tablet (in all cells) to gather the realtime health stats.")
	enableUI            = flag.Bool("enable_vtctld_ui", true, "If true, the vtctld web interface will be enabled. Default is true.")
	_                   = flag.String("durability_policy", "none", "type of durability to enforce. Default is none. Other values are dictated by registered plugins")
	sanitizeLogMessages = flag.Bool("vtctld_sanitize_log_messages", false, "When true, vtctld sanitizes logging.")

	_ = flag.String("web_dir", "", "NOT USED, here for backward compatibility")
	_ = flag.String("web_dir2", "", "NOT USED, here for backward compatibility")

	// deprecated, only here for backwards compatibility:
	deprecatedOnlineDDLCheckInterval = flag.Duration("online_ddl_check_interval", 0, "deprecated. Will be removed in next Vitess version")
)

const (
	appPrefix = "/app/"
)

// InitVtctld initializes all the vtctld functionality.
func InitVtctld(ts *topo.Server) error {
	if *deprecatedOnlineDDLCheckInterval != 0 {
		log.Warningf("the flag '--online_ddl_check_interval' is deprecated and will be removed in future versions. It is currently unused.")
	}

	actionRepo := NewActionRepository(ts)

	// keyspace actions
	actionRepo.RegisterKeyspaceAction("ValidateKeyspace",
		func(ctx context.Context, wr *wrangler.Wrangler, keyspace string) (string, error) {
			return "", wr.ValidateKeyspace(ctx, keyspace, false)
		})

	actionRepo.RegisterKeyspaceAction("ValidateSchemaKeyspace",
		func(ctx context.Context, wr *wrangler.Wrangler, keyspace string) (string, error) {
			return "", wr.ValidateSchemaKeyspace(ctx, keyspace, nil /*excludeTables*/, false /*includeViews*/, false /*skipNoPrimary*/, false /*includeVSchema*/)
		})

	actionRepo.RegisterKeyspaceAction("ValidateVersionKeyspace",
		func(ctx context.Context, wr *wrangler.Wrangler, keyspace string) (string, error) {
			return "", wr.ValidateVersionKeyspace(ctx, keyspace)
		})

	actionRepo.RegisterKeyspaceAction("ValidatePermissionsKeyspace",
		func(ctx context.Context, wr *wrangler.Wrangler, keyspace string) (string, error) {
			return "", wr.ValidatePermissionsKeyspace(ctx, keyspace)
		})

	// shard actions
	actionRepo.RegisterShardAction("ValidateShard",
		func(ctx context.Context, wr *wrangler.Wrangler, keyspace, shard string) (string, error) {
			return "", wr.ValidateShard(ctx, keyspace, shard, false)
		})

	actionRepo.RegisterShardAction("ValidateSchemaShard",
		func(ctx context.Context, wr *wrangler.Wrangler, keyspace, shard string) (string, error) {
			return "", wr.ValidateSchemaShard(ctx, keyspace, shard, nil, false, false /*includeVSchema*/)
		})

	actionRepo.RegisterShardAction("ValidateVersionShard",
		func(ctx context.Context, wr *wrangler.Wrangler, keyspace, shard string) (string, error) {
			return "", wr.ValidateVersionShard(ctx, keyspace, shard)
		})

	actionRepo.RegisterShardAction("ValidatePermissionsShard",
		func(ctx context.Context, wr *wrangler.Wrangler, keyspace, shard string) (string, error) {
			return "", wr.ValidatePermissionsShard(ctx, keyspace, shard)
		})

	// tablet actions
	actionRepo.RegisterTabletAction("Ping", "",
		func(ctx context.Context, wr *wrangler.Wrangler, tabletAlias *topodatapb.TabletAlias) (string, error) {
			ti, err := wr.TopoServer().GetTablet(ctx, tabletAlias)
			if err != nil {
				return "", err
			}
			return "", wr.TabletManagerClient().Ping(ctx, ti.Tablet)
		})

	actionRepo.RegisterTabletAction("RefreshState", acl.ADMIN,
		func(ctx context.Context, wr *wrangler.Wrangler, tabletAlias *topodatapb.TabletAlias) (string, error) {
			ti, err := wr.TopoServer().GetTablet(ctx, tabletAlias)
			if err != nil {
				return "", err
			}
			return "", wr.TabletManagerClient().RefreshState(ctx, ti.Tablet)
		})

	actionRepo.RegisterTabletAction("DeleteTablet", acl.ADMIN,
		func(ctx context.Context, wr *wrangler.Wrangler, tabletAlias *topodatapb.TabletAlias) (string, error) {
			return "", wr.DeleteTablet(ctx, tabletAlias, false)
		})

	actionRepo.RegisterTabletAction("ReloadSchema", acl.ADMIN,
		func(ctx context.Context, wr *wrangler.Wrangler, tabletAlias *topodatapb.TabletAlias) (string, error) {
			_, err := wr.VtctldServer().ReloadSchema(ctx, &vtctldatapb.ReloadSchemaRequest{
				TabletAlias: tabletAlias,
			})
			return "", err
		})

	// Anything unrecognized gets redirected to the main app page.
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, appPrefix, http.StatusFound)
	})

	http.Handle(appPrefix, staticContentHandler(*enableUI))

	var healthCheck *discovery.HealthCheckImpl
	if *enableRealtimeStats {
		ctx := context.Background()
		cells, err := ts.GetKnownCells(ctx)
		if err != nil {
			log.Errorf("Failed to get the list of known cells, failed to instantiate the healthcheck at startup: %v", err)
		} else {
			healthCheck = discovery.NewHealthCheck(ctx, *vtctl.HealthcheckRetryDelay, *vtctl.HealthCheckTimeout, ts, *localCell, strings.Join(cells, ","))
		}
	}

	// Serve the REST API for the vtctld web app.
	initAPI(context.Background(), ts, actionRepo, healthCheck)

	// Init redirects for explorers
	initExplorer(ts)

	// Init workflow manager.
	initWorkflowManager(ts)

	// Setup reverse proxy for all vttablets through /vttablet/.
	initVTTabletRedirection(ts)

	return nil
}

func staticContentHandler(enabled bool) http.Handler {
	if enabled {
		return http.FileServer(http.FS(vtctld2.Content))
	}

	fn := func(w http.ResponseWriter, r *http.Request) {
		http.NotFound(w, r)
	}
	return http.HandlerFunc(fn)
}
<commit_msg>[vtctld] Fix nil-ness in healthcheck (#11067)<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package vtctld contains all the code to expose a vtctld server
// based on the provided topo.Server.
package vtctld

import (
	"context"
	"flag"
	"net/http"
	"strings"

	"vitess.io/vitess/go/acl"
	"vitess.io/vitess/go/vt/discovery"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/topo"
	"vitess.io/vitess/go/vt/vtctl"
	"vitess.io/vitess/go/vt/wrangler"
	"vitess.io/vitess/web/vtctld2"

	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
	vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata"
)

var (
	enableRealtimeStats = flag.Bool("enable_realtime_stats", false, "Required for the Realtime Stats view. If set, vtctld will maintain a streaming RPC to each tablet (in all cells) to gather the realtime health stats.")
	enableUI            = flag.Bool("enable_vtctld_ui", true, "If true, the vtctld web interface will be enabled. Default is true.")
	_                   = flag.String("durability_policy", "none", "type of durability to enforce. Default is none. Other values are dictated by registered plugins")
	sanitizeLogMessages = flag.Bool("vtctld_sanitize_log_messages", false, "When true, vtctld sanitizes logging.")

	_ = flag.String("web_dir", "", "NOT USED, here for backward compatibility")
	_ = flag.String("web_dir2", "", "NOT USED, here for backward compatibility")

	// deprecated, only here for backwards compatibility:
	deprecatedOnlineDDLCheckInterval = flag.Duration("online_ddl_check_interval", 0, "deprecated. Will be removed in next Vitess version")
)

const (
	appPrefix = "/app/"
)

// InitVtctld initializes all the vtctld functionality.
func InitVtctld(ts *topo.Server) error {
	if *deprecatedOnlineDDLCheckInterval != 0 {
		log.Warningf("the flag '--online_ddl_check_interval' is deprecated and will be removed in future versions. It is currently unused.")
	}

	actionRepo := NewActionRepository(ts)

	// keyspace actions
	actionRepo.RegisterKeyspaceAction("ValidateKeyspace",
		func(ctx context.Context, wr *wrangler.Wrangler, keyspace string) (string, error) {
			return "", wr.ValidateKeyspace(ctx, keyspace, false)
		})

	actionRepo.RegisterKeyspaceAction("ValidateSchemaKeyspace",
		func(ctx context.Context, wr *wrangler.Wrangler, keyspace string) (string, error) {
			return "", wr.ValidateSchemaKeyspace(ctx, keyspace, nil /*excludeTables*/, false /*includeViews*/, false /*skipNoPrimary*/, false /*includeVSchema*/)
		})

	actionRepo.RegisterKeyspaceAction("ValidateVersionKeyspace",
		func(ctx context.Context, wr *wrangler.Wrangler, keyspace string) (string, error) {
			return "", wr.ValidateVersionKeyspace(ctx, keyspace)
		})

	actionRepo.RegisterKeyspaceAction("ValidatePermissionsKeyspace",
		func(ctx context.Context, wr *wrangler.Wrangler, keyspace string) (string, error) {
			return "", wr.ValidatePermissionsKeyspace(ctx, keyspace)
		})

	// shard actions
	actionRepo.RegisterShardAction("ValidateShard",
		func(ctx context.Context, wr *wrangler.Wrangler, keyspace, shard string) (string, error) {
			return "", wr.ValidateShard(ctx, keyspace, shard, false)
		})

	actionRepo.RegisterShardAction("ValidateSchemaShard",
		func(ctx context.Context, wr *wrangler.Wrangler, keyspace, shard string) (string, error) {
			return "", wr.ValidateSchemaShard(ctx, keyspace, shard, nil, false, false /*includeVSchema*/)
		})

	actionRepo.RegisterShardAction("ValidateVersionShard",
		func(ctx context.Context, wr *wrangler.Wrangler, keyspace, shard string) (string, error) {
			return "", wr.ValidateVersionShard(ctx, keyspace, shard)
		})

	actionRepo.RegisterShardAction("ValidatePermissionsShard",
		func(ctx context.Context, wr *wrangler.Wrangler, keyspace, shard string) (string, error) {
			return "", wr.ValidatePermissionsShard(ctx, keyspace, shard)
		})

	// tablet actions
	actionRepo.RegisterTabletAction("Ping", "",
		func(ctx context.Context, wr *wrangler.Wrangler, tabletAlias *topodatapb.TabletAlias) (string, error) {
			ti, err := wr.TopoServer().GetTablet(ctx, tabletAlias)
			if err != nil {
				return "", err
			}
			return "", wr.TabletManagerClient().Ping(ctx, ti.Tablet)
		})

	actionRepo.RegisterTabletAction("RefreshState", acl.ADMIN,
		func(ctx context.Context, wr *wrangler.Wrangler, tabletAlias *topodatapb.TabletAlias) (string, error) {
			ti, err := wr.TopoServer().GetTablet(ctx, tabletAlias)
			if err != nil {
				return "", err
			}
			return "", wr.TabletManagerClient().RefreshState(ctx, ti.Tablet)
		})

	actionRepo.RegisterTabletAction("DeleteTablet", acl.ADMIN,
		func(ctx context.Context, wr *wrangler.Wrangler, tabletAlias *topodatapb.TabletAlias) (string, error) {
			return "", wr.DeleteTablet(ctx, tabletAlias, false)
		})

	actionRepo.RegisterTabletAction("ReloadSchema", acl.ADMIN,
		func(ctx context.Context, wr *wrangler.Wrangler, tabletAlias *topodatapb.TabletAlias) (string, error) {
			_, err := wr.VtctldServer().ReloadSchema(ctx, &vtctldatapb.ReloadSchemaRequest{
				TabletAlias: tabletAlias,
			})
			return "", err
		})

	// Anything unrecognized gets redirected to the main app page.
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, appPrefix, http.StatusFound)
	})

	http.Handle(appPrefix, staticContentHandler(*enableUI))

	var healthCheck discovery.HealthCheck
	if *enableRealtimeStats {
		ctx := context.Background()
		cells, err := ts.GetKnownCells(ctx)
		if err != nil {
			log.Errorf("Failed to get the list of known cells, failed to instantiate the healthcheck at startup: %v", err)
		} else {
			healthCheck = discovery.NewHealthCheck(ctx, *vtctl.HealthcheckRetryDelay, *vtctl.HealthCheckTimeout, ts, *localCell, strings.Join(cells, ","))
		}
	}

	// Serve the REST API for the vtctld web app.
	initAPI(context.Background(), ts, actionRepo, healthCheck)

	// Init redirects for explorers
	initExplorer(ts)

	// Init workflow manager.
	initWorkflowManager(ts)

	// Setup reverse proxy for all vttablets through /vttablet/.
	initVTTabletRedirection(ts)

	return nil
}

func staticContentHandler(enabled bool) http.Handler {
	if enabled {
		return http.FileServer(http.FS(vtctld2.Content))
	}

	fn := func(w http.ResponseWriter, r *http.Request) {
		http.NotFound(w, r)
	}
	return http.HandlerFunc(fn)
}
<|endoftext|>