<commit_before>/*
Copyright 2014 The Camlistore Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/mpl/scancabimport/third_party/github.com/golang/oauth2"
	"github.com/mpl/scancabimport/third_party/google.golang.org/cloud/datastore"
)

var (
	verbose = flag.Bool("v", false, "verbose")
)

const (
	projectId = "scancabcamli"

	// APIkey credentials. used to auth against both the app itself, and the datastore API.
	clientId     = "886924983567-hnd1dertfvi2g0lpjs72aae8hi35k364.apps.googleusercontent.com"
	clientSecret = "nope"

	tokenCacheFile = "tokencache.json"
	scansDir       = "scans"             // where the scanned files will be stored
	mediaObjects   = "mediaObjects.json" // scans metadata
	documents      = "documents.json"    // documents metadata
)

var (
	ds *datastore.Dataset
	cl *http.Client
)

// UserInfo represents the metadata associated with the Google User
// currently logged-in to the app
type UserInfo struct {
	// User stores the email address of the currently logged-in user
	// this is used as the primary key
	User string

	// MediaObjects is a count of the MediaObjects currently associated with this user
	MediaObjects int64

	// UploadPassword is a plain-text string that protects the scan upload API
	UploadPassword string
}

// MediaObject represents the metadata associated with each individual uploaded scan
type MediaObject struct {
	// Owner is the key of the UserInfo of the user that uploaded the file
	Owner *datastore.Key `json:"-"`

	// Id is the entity ID of the key associated with this MediaObject struct
	Id int64 `datastore:"-"`

	// Blob is the key of blobstore entry with this uploaded file
	Blob string `json:"-"`

	// Creation the time when this struct was originally created
	Creation time.Time

	// ContentType is the MIME-type of the uploaded file.
	// As the mime/multipart package does not detect Content-Type
	// before sending the file in the command line client, this is
	// detected in the webapp and so this field may differ from the
	// content-type for the associated blob in the blobstore
	ContentType string

	// Filename is the name of the file when it was uploaded
	Filename string

	// Size in bytes of the uploaded file
	Size int64

	// Document is the key of the associated Document struct.
	// A Document has many MediaObjects. When newly uploaded,
	// a MediaObject is not associated with a Document.
	Document *datastore.Key

	// LacksDocument is false when this MediaObject is associated with a Document.
	// When newly uploaded, a MediaObject is not associated with a Document.
	LacksDocument bool
}

// Document is a structure that groups scans into a logical unit.
// A letter (Stored as a document) could have several pages
// (each is a MediaObject), for example.
type Document struct {
	// Owner is the key of the UserInfo of the user that created the Document
	Owner *datastore.Key

	// Pages are the keys of each Media Object that contitute this Document
	Pages []*datastore.Key

	// Id is the entity ID of the key associated with this Document struct
	Id int64 `datastore:"-"`

	// DocDate is the user-nominated date associated with this document. It can
	// store any date the user likes but is intended to be when the document was
	// received, or, perhaps, written or sent
	DocDate time.Time

	// NoDate is false when DocDate has been set by the user
	NoDate bool

	// Creation is the date the Document struct was created
	Creation time.Time

	// Title is the user-nominated title of the document
	Title string

	// Description is the user-nominated description of the document
	Description string

	// Tags is the slice of zero or more tags associated with the document by the user
	Tags string

	// LowercaseTags is the content of Tags but stored lowercase as a
	// canonical version so searches on tags can be case-insensitive
	LowercaseTags string

	// NoTags is true when Tags is empty
	NoTags bool

	// PhysicalLocation is the user-nominated description of the location
	// of the physical document of which the MediaObjects associated with this
	// Document are scans
	PhysicalLocation string

	// DueDate is the user-nominated date that the document is "due". The
	// meaning of what "due" means in relation to each particular document
	// is up to the user
	DueDate time.Time
}

const (
	// TODO(mpl): figure out how high these can be cranked up.
	scansRequestLimit = 5
	docsRequestLimit  = 5
)

func getScans() ([]*MediaObject, error) {
	var scans []*MediaObject
	query := ds.NewQuery("MediaObject")
	query = query.Limit(scansRequestLimit)
	for {
		sc := make([]*MediaObject, scansRequestLimit)
		keys, next, err := ds.RunQuery(query, sc)
		if err != nil {
			return nil, err
		}
		// get the key id and store it in the media object because we'll need it
		// to fetch the corresponding file from the blobstore later.
		for i, obj := range sc {
			if obj == nil {
				break
			}
			obj.Id = keys[i].ID()
		}
		scans = append(scans, sc...)
		if next == nil {
			break
		}
		query = next
	}
	return scans, nil
}

func getDocuments() ([]*Document, error) {
	var docs []*Document
	query := ds.NewQuery("Document")
	query = query.Limit(scansRequestLimit)
	for {
		dc := make([]*Document, docsRequestLimit)
		//		keys, next, err := ds.RunQuery(query, dc)
		_, next, err := ds.RunQuery(query, dc)
		if err != nil {
			return nil, err
		}
		docs = append(docs, dc...)
		if next == nil {
			break
		}
		query = next
	}
	return docs, nil
}

func getScannedFile(resourceId, filename string) error {
	if resourceId == "" {
		log.Printf("WARNING: Not fetching scan because empty resourceId")
		return nil
	}
	if resourceId == "" {
		log.Printf("WARNING: Not fetching scan because empty filename")
		return nil
	}
	filePath := filepath.Join(scansDir, filename)
	if _, err := os.Stat(filePath); err == nil {
		log.Printf("%s already exists, skipping download.", filePath)
		return nil
	}
	resp, err := cl.Get("https://" + projectId + ".appspot.com/resource/" + resourceId + "/" + filename)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return fmt.Errorf("Status %v", resp.Status)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(filePath, body, 0700)
}

func cacheToken(tok *oauth2.Token) error {
	file, err := os.OpenFile(tokenCacheFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}
	defer func() {
		if err := file.Close(); err != nil {
			panic(err)
		}
	}()
	if err := json.NewEncoder(file).Encode(tok); err != nil {
		return err
	}
	return nil
}

func cachedToken() (*oauth2.Token, error) {
	file, err := os.Open(tokenCacheFile)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	tok := &oauth2.Token{}
	if err := json.NewDecoder(file).Decode(tok); err != nil {
		return nil, err
	}
	return tok, nil
}

func transportFromAPIKey() (*oauth2.Transport, error) {
	conf, err := oauth2.NewConfig(&oauth2.Options{
		Scopes: []string{"https://www.googleapis.com/auth/appengine.admin", // TODO(mpl): maybe not needed?
			"https://www.googleapis.com/auth/datastore",
			"https://www.googleapis.com/auth/userinfo.email"},
		ClientID:     clientId,
		ClientSecret: clientSecret,
		RedirectURL:  "urn:ietf:wg:oauth:2.0:oob",
	},
		"https://accounts.google.com/o/oauth2/auth",
		"https://accounts.google.com/o/oauth2/token")
	if err != nil {
		return nil, err
	}

	token, err := cachedToken()
	if err == nil {
		tr := conf.NewTransport()
		tr.SetToken(token)
		return tr, nil
	}

	// Redirect user to consent page to ask for permission
	// for the scopes specified above.
	url := conf.AuthCodeURL("state", "online", "auto")
	// url := conf.AuthCodeURL("state", "offline", "auto")
	fmt.Printf("Visit the URL for the auth dialog: %v\n", url)
	fmt.Println("And enter the authorization string displayed in your browser:")

	input := bufio.NewReader(os.Stdin)
	line, _, err := input.ReadLine()
	if err != nil {
		log.Fatalf("Failed to read line: %v", err)
	}
	authorizationCode := strings.TrimSpace(string(line))
	tr, err := conf.NewTransportWithCode(authorizationCode)
	if err != nil {
		return nil, err
	}
	if err := cacheToken(tr.Token()); err != nil {
		return nil, err
	}
	return tr, nil
}

func writeObjects(scans []*MediaObject, docs map[int64]*Document) error {
	f, err := os.OpenFile(mediaObjects, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}
	if err := json.NewEncoder(f).Encode(scans); err != nil {
		f.Close()
		return err
	}
	if err := f.Close(); err != nil {
		return err
	}
	docsArray := make([]*Document, len(docs))
	for _, v := range docs {
		docsArray = append(docsArray, v)
	}
	f, err = os.OpenFile(documents, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}
	if err := json.NewEncoder(f).Encode(docsArray); err != nil {
		f.Close()
		return err
	}
	if err := f.Close(); err != nil {
		return err
	}
	return nil
}

func main() {
	flag.Parse()

	if err := os.MkdirAll(scansDir, 0700); err != nil {
		log.Fatal(err)
	}

	tr, err := transportFromAPIKey()
	if err != nil {
		log.Fatal(err)
	}
	cl = &http.Client{Transport: tr}

	ds, err = datastore.NewDatasetWithTransport(projectId, tr)
	if err != nil {
		log.Fatal(err)
	}

	scans, err := getScans()
	if err != nil {
		log.Fatal(err)
	}

	docs := make(map[int64]*Document)
	users := make(map[int64]*UserInfo)
	for _, v := range scans {
		if v == nil {
			continue
		}
		if *verbose {
			fmt.Printf("%v\n", v)
		}
		if v.Owner != nil {
			userId := v.Owner.ID()
			if _, ok := users[userId]; !ok {
				userInfo := &UserInfo{}
				if err := ds.Get(v.Owner, userInfo); err != nil {
					log.Fatal(err)
				}
				users[userId] = userInfo
				if *verbose {
					fmt.Printf("Owner: %v\n", userInfo)
				}
			}
		}
		if err := getScannedFile(fmt.Sprintf("%d", v.Id), v.Filename); err != nil {
			log.Fatal(err)
		}
		if v != nil && !v.LacksDocument && v.Document != nil {
			docId := v.Document.ID()
			if _, ok := docs[docId]; ok {
				if *verbose {
					fmt.Printf("Document cache hit: %d\n", docId)
				}
				continue
			}
			document := &Document{}
			if err := ds.Get(v.Document, document); err != nil {
				log.Fatal(err)
			}
			document.Id = docId
			docs[docId] = document
			if *verbose {
				fmt.Printf("Document: %v\n", document)
			}
		}
	}

	if err := writeObjects(scans, docs); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>auth with env vars<commit_after>/*
Copyright 2014 The Camlistore Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/mpl/scancabimport/third_party/github.com/golang/oauth2"
	"github.com/mpl/scancabimport/third_party/google.golang.org/cloud/datastore"
)

var (
	verbose = flag.Bool("v", false, "verbose")
)

const (
	projectId = "scancabcamli"

	tokenCacheFile = "tokencache.json"
	scansDir       = "scans"             // where the scanned files will be stored
	mediaObjects   = "mediaObjects.json" // scans metadata
	documents      = "documents.json"    // documents metadata
)

var (
	ds *datastore.Dataset
	cl *http.Client
)

// UserInfo represents the metadata associated with the Google User
// currently logged-in to the app
type UserInfo struct {
	// User stores the email address of the currently logged-in user
	// this is used as the primary key
	User string

	// MediaObjects is a count of the MediaObjects currently associated with this user
	MediaObjects int64

	// UploadPassword is a plain-text string that protects the scan upload API
	UploadPassword string
}

// MediaObject represents the metadata associated with each individual uploaded scan
type MediaObject struct {
	// Owner is the key of the UserInfo of the user that uploaded the file
	Owner *datastore.Key `json:"-"`

	// Id is the entity ID of the key associated with this MediaObject struct
	Id int64 `datastore:"-"`

	// Blob is the key of blobstore entry with this uploaded file
	Blob string `json:"-"`

	// Creation the time when this struct was originally created
	Creation time.Time

	// ContentType is the MIME-type of the uploaded file.
	// As the mime/multipart package does not detect Content-Type
	// before sending the file in the command line client, this is
	// detected in the webapp and so this field may differ from the
	// content-type for the associated blob in the blobstore
	ContentType string

	// Filename is the name of the file when it was uploaded
	Filename string

	// Size in bytes of the uploaded file
	Size int64

	// Document is the key of the associated Document struct.
	// A Document has many MediaObjects. When newly uploaded,
	// a MediaObject is not associated with a Document.
	Document *datastore.Key

	// LacksDocument is false when this MediaObject is associated with a Document.
	// When newly uploaded, a MediaObject is not associated with a Document.
	LacksDocument bool
}

// Document is a structure that groups scans into a logical unit.
// A letter (Stored as a document) could have several pages
// (each is a MediaObject), for example.
type Document struct {
	// Owner is the key of the UserInfo of the user that created the Document
	Owner *datastore.Key

	// Pages are the keys of each Media Object that contitute this Document
	Pages []*datastore.Key

	// Id is the entity ID of the key associated with this Document struct
	Id int64 `datastore:"-"`

	// DocDate is the user-nominated date associated with this document. It can
	// store any date the user likes but is intended to be when the document was
	// received, or, perhaps, written or sent
	DocDate time.Time

	// NoDate is false when DocDate has been set by the user
	NoDate bool

	// Creation is the date the Document struct was created
	Creation time.Time

	// Title is the user-nominated title of the document
	Title string

	// Description is the user-nominated description of the document
	Description string

	// Tags is the slice of zero or more tags associated with the document by the user
	Tags string

	// LowercaseTags is the content of Tags but stored lowercase as a
	// canonical version so searches on tags can be case-insensitive
	LowercaseTags string

	// NoTags is true when Tags is empty
	NoTags bool

	// PhysicalLocation is the user-nominated description of the location
	// of the physical document of which the MediaObjects associated with this
	// Document are scans
	PhysicalLocation string

	// DueDate is the user-nominated date that the document is "due". The
	// meaning of what "due" means in relation to each particular document
	// is up to the user
	DueDate time.Time
}

const (
	// TODO(mpl): figure out how high these can be cranked up.
	scansRequestLimit = 5
	docsRequestLimit  = 5
)

func getScans() ([]*MediaObject, error) {
	var scans []*MediaObject
	query := ds.NewQuery("MediaObject")
	query = query.Limit(scansRequestLimit)
	for {
		sc := make([]*MediaObject, scansRequestLimit)
		keys, next, err := ds.RunQuery(query, sc)
		if err != nil {
			return nil, err
		}
		// get the key id and store it in the media object because we'll need it
		// to fetch the corresponding file from the blobstore later.
		for i, obj := range sc {
			if obj == nil {
				break
			}
			obj.Id = keys[i].ID()
		}
		scans = append(scans, sc...)
		if next == nil {
			break
		}
		query = next
	}
	return scans, nil
}

func getDocuments() ([]*Document, error) {
	var docs []*Document
	query := ds.NewQuery("Document")
	query = query.Limit(scansRequestLimit)
	for {
		dc := make([]*Document, docsRequestLimit)
		//		keys, next, err := ds.RunQuery(query, dc)
		_, next, err := ds.RunQuery(query, dc)
		if err != nil {
			return nil, err
		}
		docs = append(docs, dc...)
		if next == nil {
			break
		}
		query = next
	}
	return docs, nil
}

func getScannedFile(resourceId, filename string) error {
	if resourceId == "" {
		log.Printf("WARNING: Not fetching scan because empty resourceId")
		return nil
	}
	if resourceId == "" {
		log.Printf("WARNING: Not fetching scan because empty filename")
		return nil
	}
	filePath := filepath.Join(scansDir, filename)
	if _, err := os.Stat(filePath); err == nil {
		log.Printf("%s already exists, skipping download.", filePath)
		return nil
	}
	resp, err := cl.Get("https://" + projectId + ".appspot.com/resource/" + resourceId + "/" + filename)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return fmt.Errorf("Status %v", resp.Status)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(filePath, body, 0700)
}

func cacheToken(tok *oauth2.Token) error {
	file, err := os.OpenFile(tokenCacheFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}
	defer func() {
		if err := file.Close(); err != nil {
			panic(err)
		}
	}()
	if err := json.NewEncoder(file).Encode(tok); err != nil {
		return err
	}
	return nil
}

func cachedToken() (*oauth2.Token, error) {
	file, err := os.Open(tokenCacheFile)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	tok := &oauth2.Token{}
	if err := json.NewDecoder(file).Decode(tok); err != nil {
		return nil, err
	}
	return tok, nil
}

func transportFromAPIKey() (*oauth2.Transport, error) {
	// APIkey credentials. used to auth against both the app itself, and the datastore API.
	clientId := os.Getenv("CLIENTID")
	if clientId == "" {
		return nil, fmt.Errorf("CLIENTID not set")
	}
	clientSecret := os.Getenv("CLIENTSECRET")
	if clientSecret == "" {
		return nil, fmt.Errorf("CLIENTSECRET not set")
	}
	conf, err := oauth2.NewConfig(&oauth2.Options{
		Scopes: []string{"https://www.googleapis.com/auth/appengine.admin", // TODO(mpl): maybe not needed?
			"https://www.googleapis.com/auth/datastore",
			"https://www.googleapis.com/auth/userinfo.email"},
		ClientID:     clientId,
		ClientSecret: clientSecret,
		RedirectURL:  "urn:ietf:wg:oauth:2.0:oob",
	},
		"https://accounts.google.com/o/oauth2/auth",
		"https://accounts.google.com/o/oauth2/token")
	if err != nil {
		return nil, err
	}

	token, err := cachedToken()
	if err == nil {
		tr := conf.NewTransport()
		tr.SetToken(token)
		return tr, nil
	}

	// Redirect user to consent page to ask for permission
	// for the scopes specified above.
	url := conf.AuthCodeURL("state", "online", "auto")
	// url := conf.AuthCodeURL("state", "offline", "auto")
	fmt.Printf("Visit the URL for the auth dialog: %v\n", url)
	fmt.Println("And enter the authorization string displayed in your browser:")

	input := bufio.NewReader(os.Stdin)
	line, _, err := input.ReadLine()
	if err != nil {
		log.Fatalf("Failed to read line: %v", err)
	}
	authorizationCode := strings.TrimSpace(string(line))
	tr, err := conf.NewTransportWithCode(authorizationCode)
	if err != nil {
		return nil, err
	}
	if err := cacheToken(tr.Token()); err != nil {
		return nil, err
	}
	return tr, nil
}

func writeObjects(scans []*MediaObject, docs map[int64]*Document) error {
	f, err := os.OpenFile(mediaObjects, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}
	if err := json.NewEncoder(f).Encode(scans); err != nil {
		f.Close()
		return err
	}
	if err := f.Close(); err != nil {
		return err
	}
	docsArray := make([]*Document, len(docs))
	for _, v := range docs {
		docsArray = append(docsArray, v)
	}
	f, err = os.OpenFile(documents, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}
	if err := json.NewEncoder(f).Encode(docsArray); err != nil {
		f.Close()
		return err
	}
	if err := f.Close(); err != nil {
		return err
	}
	return nil
}

func main() {
	flag.Parse()

	if err := os.MkdirAll(scansDir, 0700); err != nil {
		log.Fatal(err)
	}

	tr, err := transportFromAPIKey()
	if err != nil {
		log.Fatal(err)
	}
	cl = &http.Client{Transport: tr}

	ds, err = datastore.NewDatasetWithTransport(projectId, tr)
	if err != nil {
		log.Fatal(err)
	}

	scans, err := getScans()
	if err != nil {
		log.Fatal(err)
	}

	docs := make(map[int64]*Document)
	users := make(map[int64]*UserInfo)
	for _, v := range scans {
		if v == nil {
			continue
		}
		if *verbose {
			fmt.Printf("%v\n", v)
		}
		if v.Owner != nil {
			userId := v.Owner.ID()
			if _, ok := users[userId]; !ok {
				userInfo := &UserInfo{}
				if err := ds.Get(v.Owner, userInfo); err != nil {
					log.Fatal(err)
				}
				users[userId] = userInfo
				if *verbose {
					fmt.Printf("Owner: %v\n", userInfo)
				}
			}
		}
		if err := getScannedFile(fmt.Sprintf("%d", v.Id), v.Filename); err != nil {
			log.Fatal(err)
		}
		if v != nil && !v.LacksDocument && v.Document != nil {
			docId := v.Document.ID()
			if _, ok := docs[docId]; ok {
				if *verbose {
					fmt.Printf("Document cache hit: %d\n", docId)
				}
				continue
			}
			document := &Document{}
			if err := ds.Get(v.Document, document); err != nil {
				log.Fatal(err)
			}
			document.Id = docId
			docs[docId] = document
			if *verbose {
				fmt.Printf("Document: %v\n", document)
			}
		}
	}

	if err := writeObjects(scans, docs); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/cespare/blackfriday"
	"github.com/howeyc/fsnotify"
)

// TODO: Fix paths when markdownd is installed via go get. Need to look at the proper place in $GOPATH to find
// vendor. Alternative approach: use go-bindata to pack pygments into the binary; the first time markdownd
// runs, expand pygments into /usr/local (or whatever).
// TODO: Allow for specifying the browser? (bcat has -b for this.)

const (
	tempfile   = "/tmp/markdownd_tempfile.html"
	pygmentPath = "vendor/pygments/pygmentize"
)

var (
	serve = flag.Bool("s", false, "Open the output in your browser.")
	watch = flag.Bool("w", false, "Open the output in a browser and watch the input file for changes to reload.")

	sseHeaders = [][2]string{
		{"Content-Type", "text/event-stream"},
		{"Cache-Control", "no-cache"},
		{"Connection", "keep-alive"},
	}

	pygmentize string
	validLanguages = make(map[string]struct{})

	mu       sync.RWMutex // protects rendered
	rendered []byte
)

func init() {
	flag.Parse()

	// Try to figure out where pygments is.
	exe, err := exec.LookPath(os.Args[0])
	if err != nil {
		fatal("Cannot locate markdownd executable.")
	}
	pygmentize = filepath.Join(filepath.Dir(exe), pygmentPath)

	if _, err := os.Stat(pygmentize); err != nil {
		fatal("Pygments not found:", err)
	}

	rawLexerList, err := exec.Command(pygmentize, "-L", "lexers").Output()
	if err != nil {
		fatal(err)
	}
	for _, line := range bytes.Split(rawLexerList, []byte("\n")) {
		if len(line) == 0 || line[0] != '*' {
			continue
		}
		for _, l := range bytes.Split(bytes.Trim(line, "* :"), []byte(",")) {
			lexer := string(bytes.TrimSpace(l))
			if len(lexer) != 0 {
				validLanguages[lexer] = struct{}{}
			}
		}
	}
}

func usage(status int) {
	fmt.Printf(`Usage:
  $ %s [OPTIONS] [MARKDOWN_FILE]
where OPTIONS are:
`, os.Args[0])
	flag.PrintDefaults()
	fmt.Println(`and MARKDOWN_FILE is some file containing markdown.
- If MARKDOWN_FILE is not given, markdownd will read markdown text from stdin.
- If you specify -w, you must specify MARKDOWN_FILE (stdin doesn't make sense).
- -w implies -s.
- If neither -w nor -s are given, the output is written to stdout.`)
	os.Exit(status)
}

func syntaxHighlight(out io.Writer, in io.Reader, language string) {
	_, ok := validLanguages[language]
	if !ok || language == "" {
		language = "text"
	}
	pygmentsCmd := exec.Command(pygmentize, "-l", language, "-f", "html", "-P", "encoding=utf-8")
	pygmentsCmd.Stdin = in
	pygmentsCmd.Stdout = out
	var stderr bytes.Buffer
	pygmentsCmd.Stderr = &stderr
	if err := pygmentsCmd.Run(); err != nil {
		fatal(err)
	}
}

// Render renders some markdown with syntax highlighting. It would be nicer if blackfriday.Markdown operated
// on io.Readers/Writers, but it uses []bytes so we need to fully buffer everything.
func render(input []byte) []byte {
	flags := 0
	flags |= blackfriday.HTML_GITHUB_BLOCKCODE
	renderer := blackfriday.HtmlRenderer(flags, "", "")
	renderer.SetBlockCodeProcessor(syntaxHighlight)

	extensions := 0
	extensions |= blackfriday.EXTENSION_FENCED_CODE
	extensions |= blackfriday.EXTENSION_TABLES
	extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS
	extensions |= blackfriday.EXTENSION_AUTOLINK

	return blackfriday.Markdown(input, renderer, extensions)
}

func renderFromFile(filename string) ([]byte, error) {
	input, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return render(input), nil
}

func fatal(args ...interface{}) {
	fmt.Fprintln(os.Stderr, args...)
	os.Exit(1)
}

// bopen opens some (possibly file://) url in a browser.
func bopen(url string) error {
	cmd := exec.Command(openProgram, url)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func updateListener(filename string) (<-chan bool, error) {
	// Normalize the name to "./foobar" form if it is relative like "foobar"
	if !strings.HasPrefix(filename, "./") && !strings.HasPrefix(filename, "/") {
		filename = "./" + filename
	}
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}

	c := make(chan bool)
	go func() {
		for {
			select {
			case e := <-watcher.Event:
				if e.Name != filename {
					continue
				}
				if e.IsDelete() {
					// Need to reset the watch. No way to report errors if this doesn't work for some reason.
					watcher.RemoveWatch(filename)
					watcher.Watch(filename)
				}
				// Nonblocking send (if nobody's listening, skip it and move on).
				select {
				case c <- true:
				default:
				}
			case err := <-watcher.Error:
				fmt.Println("fsnotify error:", err)
			}
		}
	}()

	if err := watcher.Watch(filepath.Dir(filename)); err != nil {
		return nil, err
	}
	return reRender(c), nil
}

// reRender re-renders the markdown on updates. It handles bursty events by batching slightly.
func reRender(updates <-chan bool) <-chan bool {
	out := make(chan bool)
	go func() {
		for _ = range updates {
			t := time.NewTimer(50 * time.Millisecond)
		loop:
			for {
				select {
				case <-updates:
					// drop
				case <-t.C:
					break loop
				}
			}
			if err := renderMarkdown(); err != nil {
				fmt.Println("Warning:", err)
			}
			out <- true
		}
	}()
	return out
}

// Unification of http.ResponseWriter, http.Flusher, and http.CloseNotifier
type HTTPWriter interface {
	Header() http.Header
	Write([]byte) (int, error)
	WriteHeader(int)
	Flush()
	CloseNotify() <-chan bool
}

func makeUpdateHandler(update <-chan bool) http.HandlerFunc {
	return func(writer http.ResponseWriter, r *http.Request) {
		w, ok := writer.(HTTPWriter)
		if !ok {
			panic("HTTP server does not support Flusher and/or CloseNotifier needed for SSE.")
		}
		closed := w.CloseNotify()
		for _, header := range sseHeaders {
			w.Header().Set(header[0], header[1])
		}

		for {
			select {
			case <-update:
				fmt.Fprint(w, "data:update\n\n")
				w.Flush()
			case <-closed:
				// We're ready to exit now, because the user closed the page.
				os.Exit(0)
			}
		}
	}
}

func renderMarkdown() error {
	mu.Lock()
	defer mu.Unlock()
	if flag.NArg() > 0 {
		var err error
		rendered, err = renderFromFile(flag.Arg(0))
		if err != nil {
			return err
		}
	} else {
		input, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			return err
		}
		rendered = render(input)
	}

	// Embed the output in an HTML page with some nice CSS, unless we're printing the output directly to stdout.
	if *serve || *watch {
		rendered = append([]byte(htmlHeader), rendered...)
		rendered = append(rendered, []byte(htmlFooter)...)
	}
	return nil
}

// serve serves output on a local webserver. The caller should Close the server it gets back.
func makeServer(updates <-chan bool) *httptest.Server {
	mux := http.NewServeMux()
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		mu.RLock()
		defer mu.RUnlock()
		if r.FormValue("nojs") != "true" {
			w.Write([]byte(jsHeader))
		}
		w.Write(rendered)
	})
	mux.HandleFunc("/updates", makeUpdateHandler(updates))
	return httptest.NewServer(mux)
}

func main() {
	if (flag.NArg() == 0 && *watch) || flag.NArg() > 1 {
		usage(1)
	}

	if err := renderMarkdown(); err != nil {
		fatal(err)
	}

	switch {
	case *watch:
		updates, err := updateListener(flag.Arg(0))
		if err != nil {
			fatal(err)
		}
		server := makeServer(updates)
		defer server.Close()
		fmt.Printf("Serving markdown rendered from %s at %s\n", flag.Arg(0), server.URL)
		if err := bopen(server.URL); err != nil {
			fatal(err)
		}
		// Just sit and block infinitely.
		select {}
	case *serve:
		// Write to a temp file and open it in a browser, then exit.
		temp, err := os.Create(tempfile)
		if err != nil {
			fatal("Could not create a tempfile:", err)
		}
		if _, err := temp.Write(rendered); err != nil {
			fatal(err)
		}
		if err := bopen(temp.Name()); err != nil {
			fatal(err)
		}
	default:
		// Just write to stdout and we're done.
		os.Stdout.Write(rendered)
	}
}
<commit_msg>More path fixes.<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/cespare/blackfriday"
	"github.com/howeyc/fsnotify"
)

// TODO: Fix paths when markdownd is installed via go get. Need to look at the proper place in $GOPATH to find
// vendor. Alternative approach: use go-bindata to pack pygments into the binary; the first time markdownd
// runs, expand pygments into /usr/local (or whatever).
// TODO: Allow for specifying the browser? (bcat has -b for this.)

const (
	tempfile   = "/tmp/markdownd_tempfile.html"
	pygmentPath = "vendor/pygments/pygmentize"
)

var (
	serve = flag.Bool("s", false, "Open the output in your browser.")
	watch = flag.Bool("w", false, "Open the output in a browser and watch the input file for changes to reload.")

	sseHeaders = [][2]string{
		{"Content-Type", "text/event-stream"},
		{"Cache-Control", "no-cache"},
		{"Connection", "keep-alive"},
	}

	pygmentize string
	validLanguages = make(map[string]struct{})

	mu       sync.RWMutex // protects rendered
	rendered []byte
)

func init() {
	flag.Parse()

	// Try to figure out where pygments is.
	exe, err := exec.LookPath(os.Args[0])
	if err != nil {
		fatal("Cannot locate markdownd executable.")
	}
	pygmentize = filepath.Join(filepath.Dir(exe), pygmentPath)

	if _, err := os.Stat(pygmentize); err != nil {
		fatal("Pygments not found:", err)
	}

	rawLexerList, err := exec.Command(pygmentize, "-L", "lexers").Output()
	if err != nil {
		fatal(err)
	}
	for _, line := range bytes.Split(rawLexerList, []byte("\n")) {
		if len(line) == 0 || line[0] != '*' {
			continue
		}
		for _, l := range bytes.Split(bytes.Trim(line, "* :"), []byte(",")) {
			lexer := string(bytes.TrimSpace(l))
			if len(lexer) != 0 {
				validLanguages[lexer] = struct{}{}
			}
		}
	}
}

func usage(status int) {
	fmt.Printf(`Usage:
  $ %s [OPTIONS] [MARKDOWN_FILE]
where OPTIONS are:
`, os.Args[0])
	flag.PrintDefaults()
	fmt.Println(`and MARKDOWN_FILE is some file containing markdown.
- If MARKDOWN_FILE is not given, markdownd will read markdown text from stdin.
- If you specify -w, you must specify MARKDOWN_FILE (stdin doesn't make sense).
- -w implies -s.
- If neither -w nor -s are given, the output is written to stdout.`)
	os.Exit(status)
}

func syntaxHighlight(out io.Writer, in io.Reader, language string) {
	_, ok := validLanguages[language]
	if !ok || language == "" {
		language = "text"
	}
	pygmentsCmd := exec.Command(pygmentize, "-l", language, "-f", "html", "-P", "encoding=utf-8")
	pygmentsCmd.Stdin = in
	pygmentsCmd.Stdout = out
	var stderr bytes.Buffer
	pygmentsCmd.Stderr = &stderr
	if err := pygmentsCmd.Run(); err != nil {
		fatal(err)
	}
}

// Render renders some markdown with syntax highlighting. It would be nicer if blackfriday.Markdown operated
// on io.Readers/Writers, but it uses []bytes so we need to fully buffer everything.
func render(input []byte) []byte {
	flags := 0
	flags |= blackfriday.HTML_GITHUB_BLOCKCODE
	renderer := blackfriday.HtmlRenderer(flags, "", "")
	renderer.SetBlockCodeProcessor(syntaxHighlight)

	extensions := 0
	extensions |= blackfriday.EXTENSION_FENCED_CODE
	extensions |= blackfriday.EXTENSION_TABLES
	extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS
	extensions |= blackfriday.EXTENSION_AUTOLINK

	return blackfriday.Markdown(input, renderer, extensions)
}

func renderFromFile(filename string) ([]byte, error) {
	input, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return render(input), nil
}

func fatal(args ...interface{}) {
	fmt.Fprintln(os.Stderr, args...)
	os.Exit(1)
}

// bopen opens some (possibly file://) url in a browser.
func bopen(url string) error {
	cmd := exec.Command(openProgram, url)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func updateListener(filename string) (<-chan bool, error) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}

	c := make(chan bool)
	go func() {
		for {
			select {
			case e := <-watcher.Event:
				if strings.TrimPrefix(e.Name, "./") != strings.TrimPrefix(filename, "./") {
					continue
				}
				if e.IsDelete() {
					// Need to reset the watch. No way to report errors if this doesn't work for some reason.
					watcher.RemoveWatch(filename)
					watcher.Watch(filename)
				}
				// Nonblocking send (if nobody's listening, skip it and move on).
				select {
				case c <- true:
				default:
				}
			case err := <-watcher.Error:
				fmt.Println("fsnotify error:", err)
			}
		}
	}()

	if err := watcher.Watch(filepath.Dir(filename)); err != nil {
		return nil, err
	}
	return reRender(c), nil
}

// reRender re-renders the markdown on updates. It handles bursty events by batching slightly.
func reRender(updates <-chan bool) <-chan bool {
	out := make(chan bool)
	go func() {
		for _ = range updates {
			t := time.NewTimer(50 * time.Millisecond)
		loop:
			for {
				select {
				case <-updates:
					// drop
				case <-t.C:
					break loop
				}
			}
			if err := renderMarkdown(); err != nil {
				fmt.Println("Warning:", err)
			}
			out <- true
		}
	}()
	return out
}

// Unification of http.ResponseWriter, http.Flusher, and http.CloseNotifier
type HTTPWriter interface {
	Header() http.Header
	Write([]byte) (int, error)
	WriteHeader(int)
	Flush()
	CloseNotify() <-chan bool
}

func makeUpdateHandler(update <-chan bool) http.HandlerFunc {
	return func(writer http.ResponseWriter, r *http.Request) {
		w, ok := writer.(HTTPWriter)
		if !ok {
			panic("HTTP server does not support Flusher and/or CloseNotifier needed for SSE.")
		}
		closed := w.CloseNotify()
		for _, header := range sseHeaders {
			w.Header().Set(header[0], header[1])
		}

		for {
			select {
			case <-update:
				fmt.Fprint(w, "data:update\n\n")
				w.Flush()
			case <-closed:
				// We're ready to exit now, because the user closed the page.
				os.Exit(0)
			}
		}
	}
}

func renderMarkdown() error {
	mu.Lock()
	defer mu.Unlock()
	if flag.NArg() > 0 {
		var err error
		rendered, err = renderFromFile(flag.Arg(0))
		if err != nil {
			return err
		}
	} else {
		input, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			return err
		}
		rendered = render(input)
	}

	// Embed the output in an HTML page with some nice CSS, unless we're printing the output directly to stdout.
	if *serve || *watch {
		rendered = append([]byte(htmlHeader), rendered...)
		rendered = append(rendered, []byte(htmlFooter)...)
	}
	return nil
}

// serve serves output on a local webserver. The caller should Close the server it gets back.
func makeServer(updates <-chan bool) *httptest.Server {
	mux := http.NewServeMux()
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		mu.RLock()
		defer mu.RUnlock()
		if r.FormValue("nojs") != "true" {
			w.Write([]byte(jsHeader))
		}
		w.Write(rendered)
	})
	mux.HandleFunc("/updates", makeUpdateHandler(updates))
	return httptest.NewServer(mux)
}

func main() {
	if (flag.NArg() == 0 && *watch) || flag.NArg() > 1 {
		usage(1)
	}

	if err := renderMarkdown(); err != nil {
		fatal(err)
	}

	switch {
	case *watch:
		updates, err := updateListener(flag.Arg(0))
		if err != nil {
			fatal(err)
		}
		server := makeServer(updates)
		defer server.Close()
		fmt.Printf("Serving markdown rendered from %s at %s\n", flag.Arg(0), server.URL)
		if err := bopen(server.URL); err != nil {
			fatal(err)
		}
		// Just sit and block infinitely.
		select {}
	case *serve:
		// Write to a temp file and open it in a browser, then exit.
		temp, err := os.Create(tempfile)
		if err != nil {
			fatal("Could not create a tempfile:", err)
		}
		if _, err := temp.Write(rendered); err != nil {
			fatal(err)
		}
		if err := bopen(temp.Name()); err != nil {
			fatal(err)
		}
	default:
		// Just write to stdout and we're done.
		os.Stdout.Write(rendered)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"strconv"
	"time"

	"github.com/brotherlogic/goserver"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	pbd "github.com/brotherlogic/discovery/proto"
	pb "github.com/brotherlogic/gobuildmaster/proto"
	pbs "github.com/brotherlogic/gobuildslave/proto"
	pbg "github.com/brotherlogic/goserver/proto"
	"github.com/brotherlogic/goserver/utils"
)

const (
	intentWait = time.Second
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	config     *pb.Config
	serving    bool
	LastIntent time.Time
}

type mainChecker struct {
	prev   []string
	logger func(string)
}

func getIP(servertype, servername string) (string, int) {
	conn, _ := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err != nil {
		return "", -1
	}
	for _, s := range r.Services {
		if s.Name == servertype && s.Identifier == servername {
			return s.Ip, int(s.Port)
		}
	}

	return "", -1
}

func (t *mainChecker) getprev() []string {
	return t.prev
}
func (t *mainChecker) setprev(v []string) {
	t.prev = v
}

func (t *mainChecker) assess(server string) (*pbs.JobList, *pbs.Config) {
	list := &pbs.JobList{}
	conf := &pbs.Config{}

	ip, port := getIP("gobuildslave", server)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	conn, err := grpc.DialContext(ctx, ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	defer conn.Close()
	if err != nil {
		return list, conf
	}

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(ctx, &pbs.Empty{}, grpc.FailFast(false))
	if err != nil {
		return list, conf
	}

	r2, err := slave.GetConfig(ctx, &pbs.Empty{}, grpc.FailFast(false))
	if err != nil {
		return list, conf
	}

	return r, r2
}

func (t *mainChecker) master(entry *pbd.RegistryEntry, master bool) bool {
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	conn, _ := grpc.Dial(entry.GetIp()+":"+strconv.Itoa(int(entry.GetPort())), grpc.WithInsecure())
	defer conn.Close()

	server := pbg.NewGoserverServiceClient(conn)
	_, err := server.Mote(ctx, &pbg.MoteRequest{Master: master}, grpc.FailFast(false))
	if err != nil {
		t.logger(fmt.Sprintf("Master REJECT: %v", err))
	}

	return err == nil
}

func runJob(job *pbs.JobSpec, server string) {
	if server != "" {
		ip, port := getIP("gobuildslave", server)
		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
		defer cancel()
		conn, _ := grpc.DialContext(ctx, ip+":"+strconv.Itoa(port), grpc.WithInsecure())
		defer conn.Close()

		slave := pbs.NewGoBuildSlaveClient(conn)
		job.Server = server
		slave.Run(ctx, job, grpc.FailFast(false))
	}
}

func (t *mainChecker) discover() *pbd.ServiceList {
	ret := &pbd.ServiceList{}

	conn, _ := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err == nil {
		for _, s := range r.Services {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	pb.RegisterGoBuildMasterServer(server, &s)
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

// Mote promotes/demotes this server
func (s Server) Mote(master bool) error {
	return nil
}

//GetState gets the state of the server
func (s Server) GetState() []*pbg.State {
	return []*pbg.State{&pbg.State{Key: "last_intent", TimeValue: s.LastIntent.Unix()}}
}

//Compare compares current state to desired state
func (s Server) Compare(ctx context.Context, in *pb.Empty) (*pb.CompareResponse, error) {
	resp := &pb.CompareResponse{}
	list, _ := getFleetStatus(&mainChecker{logger: s.Log})
	cc := &pb.Config{}
	for _, jlist := range list {
		for _, job := range jlist.GetDetails() {
			cc.Intents = append(cc.Intents, &pb.Intent{Spec: job.GetSpec()})
		}
	}
	resp.Current = cc
	resp.Desired = s.config

	return resp, nil
}

func getConfig(c checker) *pb.Config {
	list, _ := getFleetStatus(c)
	config := &pb.Config{}

	for _, jlist := range list {
		for _, job := range jlist.Details {
			found := false
			for _, ij := range config.Intents {
				if job.Spec.Name == ij.Spec.Name {
					ij.Count++
					found = true
				}
			}

			if !found {
				config.Intents = append(config.Intents, &pb.Intent{Spec: &pbs.JobSpec{Name: job.Spec.Name}, Count: 1})
			}
		}
	}

	return config
}

// MatchIntent tries to match the intent with the state of production
func (s *Server) MatchIntent() {
	checker := &mainChecker{logger: s.Log}
	for s.serving {
		time.Sleep(intentWait)
		s.LastIntent = time.Now()

		state := getConfig(checker)
		diff := configDiff(s.config, state)
		joblist := runJobs(diff)
		for _, job := range joblist {
			runJob(job, chooseServer(job, checker))
		}
	}
}

// SetMaster sets up the master settings
func (s *Server) SetMaster() {
	checker := &mainChecker{logger: s.Log}
	for s.serving {
		time.Sleep(intentWait)

		fleet := checker.discover()
		matcher := make(map[string][]*pbd.RegistryEntry)
		hasMaster := make(map[string]int)
		for _, entry := range fleet.GetServices() {
			if _, ok := matcher[entry.GetName()]; !ok {
				if entry.GetMaster() {
					hasMaster[entry.GetName()]++
				}
				matcher[entry.GetName()] = []*pbd.RegistryEntry{entry}
			} else {
				if entry.GetMaster() {
					hasMaster[entry.GetName()] = 1
					matcher[entry.GetName()] = append(matcher[entry.GetName()], entry)
				}
			}
		}

		for key, entries := range matcher {
			if hasMaster[key] > 1 {
				hasMaster[key] = 1
				seen := false
				for _, entry := range entries {
					if seen && entry.GetMaster() {
						checker.master(entry, false)
					} else if entry.GetMaster() {
						seen = true
					}
				}
			}

			if hasMaster[key] == 0 {
				for _, entry := range entries {
					if checker.master(entry, true) {
						entry.Master = true
						break
					}
				}
			}
		}
	}
}

func main() {
	config, err := loadConfig("config.pb")
	if err != nil {
		log.Fatalf("Fatal loading of config: %v", err)
	}

	var sync = flag.Bool("once", false, "One pass intent match")
	s := &Server{&goserver.GoServer{}, config, true, time.Now()}

	var quiet = flag.Bool("quiet", false, "Show all output")
	flag.Parse()

	if *quiet {
		log.SetFlags(0)
		log.SetOutput(ioutil.Discard)
	}

	if *sync {
		s.MatchIntent()
	} else {
		s.Register = s
		s.PrepServer()
		s.GoServer.Killme = false
		s.RegisterServer("gobuildmaster", false)
		s.RegisterServingTask(s.MatchIntent)
		s.RegisterServingTask(s.SetMaster)
		err := s.Serve()
		if err != nil {
			log.Fatalf("Serve error: %v", err)
		}
	}
}
<commit_msg>Removes dial contexts<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"strconv"
	"time"

	"github.com/brotherlogic/goserver"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	pbd "github.com/brotherlogic/discovery/proto"
	pb "github.com/brotherlogic/gobuildmaster/proto"
	pbs "github.com/brotherlogic/gobuildslave/proto"
	pbg "github.com/brotherlogic/goserver/proto"
	"github.com/brotherlogic/goserver/utils"
)

const (
	intentWait = time.Second
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	config     *pb.Config
	serving    bool
	LastIntent time.Time
}

type mainChecker struct {
	prev   []string
	logger func(string)
}

func getIP(servertype, servername string) (string, int) {
	conn, _ := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err != nil {
		return "", -1
	}
	for _, s := range r.Services {
		if s.Name == servertype && s.Identifier == servername {
			return s.Ip, int(s.Port)
		}
	}

	return "", -1
}

func (t *mainChecker) getprev() []string {
	return t.prev
}
func (t *mainChecker) setprev(v []string) {
	t.prev = v
}

func (t *mainChecker) assess(server string) (*pbs.JobList, *pbs.Config) {
	list := &pbs.JobList{}
	conf := &pbs.Config{}

	ip, port := getIP("gobuildslave", server)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	conn, err := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	defer conn.Close()
	if err != nil {
		return list, conf
	}

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(ctx, &pbs.Empty{}, grpc.FailFast(false))
	if err != nil {
		return list, conf
	}

	r2, err := slave.GetConfig(ctx, &pbs.Empty{}, grpc.FailFast(false))
	if err != nil {
		return list, conf
	}

	return r, r2
}

func (t *mainChecker) master(entry *pbd.RegistryEntry, master bool) bool {
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	conn, _ := grpc.Dial(entry.GetIp()+":"+strconv.Itoa(int(entry.GetPort())), grpc.WithInsecure())
	defer conn.Close()

	server := pbg.NewGoserverServiceClient(conn)
	_, err := server.Mote(ctx, &pbg.MoteRequest{Master: master}, grpc.FailFast(false))
	if err != nil {
		t.logger(fmt.Sprintf("Master REJECT: %v", err))
	}

	return err == nil
}

func runJob(job *pbs.JobSpec, server string) {
	if server != "" {
		ip, port := getIP("gobuildslave", server)
		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
		defer cancel()
		conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
		defer conn.Close()

		slave := pbs.NewGoBuildSlaveClient(conn)
		job.Server = server
		slave.Run(ctx, job, grpc.FailFast(false))
	}
}

func (t *mainChecker) discover() *pbd.ServiceList {
	ret := &pbd.ServiceList{}

	conn, _ := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err == nil {
		for _, s := range r.Services {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	pb.RegisterGoBuildMasterServer(server, &s)
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

// Mote promotes/demotes this server
func (s Server) Mote(master bool) error {
	return nil
}

//GetState gets the state of the server
func (s Server) GetState() []*pbg.State {
	return []*pbg.State{&pbg.State{Key: "last_intent", TimeValue: s.LastIntent.Unix()}}
}

//Compare compares current state to desired state
func (s Server) Compare(ctx context.Context, in *pb.Empty) (*pb.CompareResponse, error) {
	resp := &pb.CompareResponse{}
	list, _ := getFleetStatus(&mainChecker{logger: s.Log})
	cc := &pb.Config{}
	for _, jlist := range list {
		for _, job := range jlist.GetDetails() {
			cc.Intents = append(cc.Intents, &pb.Intent{Spec: job.GetSpec()})
		}
	}
	resp.Current = cc
	resp.Desired = s.config

	return resp, nil
}

func getConfig(c checker) *pb.Config {
	list, _ := getFleetStatus(c)
	config := &pb.Config{}

	for _, jlist := range list {
		for _, job := range jlist.Details {
			found := false
			for _, ij := range config.Intents {
				if job.Spec.Name == ij.Spec.Name {
					ij.Count++
					found = true
				}
			}

			if !found {
				config.Intents = append(config.Intents, &pb.Intent{Spec: &pbs.JobSpec{Name: job.Spec.Name}, Count: 1})
			}
		}
	}

	return config
}

// MatchIntent tries to match the intent with the state of production
func (s *Server) MatchIntent() {
	checker := &mainChecker{logger: s.Log}
	for s.serving {
		time.Sleep(intentWait)
		s.LastIntent = time.Now()

		state := getConfig(checker)
		diff := configDiff(s.config, state)
		joblist := runJobs(diff)
		for _, job := range joblist {
			runJob(job, chooseServer(job, checker))
		}
	}
}

// SetMaster sets up the master settings
func (s *Server) SetMaster() {
	checker := &mainChecker{logger: s.Log}
	for s.serving {
		time.Sleep(intentWait)

		fleet := checker.discover()
		matcher := make(map[string][]*pbd.RegistryEntry)
		hasMaster := make(map[string]int)
		for _, entry := range fleet.GetServices() {
			if _, ok := matcher[entry.GetName()]; !ok {
				if entry.GetMaster() {
					hasMaster[entry.GetName()]++
				}
				matcher[entry.GetName()] = []*pbd.RegistryEntry{entry}
			} else {
				if entry.GetMaster() {
					hasMaster[entry.GetName()] = 1
					matcher[entry.GetName()] = append(matcher[entry.GetName()], entry)
				}
			}
		}

		for key, entries := range matcher {
			if hasMaster[key] > 1 {
				hasMaster[key] = 1
				seen := false
				for _, entry := range entries {
					if seen && entry.GetMaster() {
						checker.master(entry, false)
					} else if entry.GetMaster() {
						seen = true
					}
				}
			}

			if hasMaster[key] == 0 {
				for _, entry := range entries {
					if checker.master(entry, true) {
						entry.Master = true
						break
					}
				}
			}
		}
	}
}

func main() {
	config, err := loadConfig("config.pb")
	if err != nil {
		log.Fatalf("Fatal loading of config: %v", err)
	}

	var sync = flag.Bool("once", false, "One pass intent match")
	s := &Server{&goserver.GoServer{}, config, true, time.Now()}

	var quiet = flag.Bool("quiet", false, "Show all output")
	flag.Parse()

	if *quiet {
		log.SetFlags(0)
		log.SetOutput(ioutil.Discard)
	}

	if *sync {
		s.MatchIntent()
	} else {
		s.Register = s
		s.PrepServer()
		s.GoServer.Killme = false
		s.RegisterServer("gobuildmaster", false)
		s.RegisterServingTask(s.MatchIntent)
		s.RegisterServingTask(s.SetMaster)
		err := s.Serve()
		if err != nil {
			log.Fatalf("Serve error: %v", err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"strconv"

	"fmt"
	"io/ioutil"
	"net/url"
	"strings"
	"unicode/utf8"

	"bytes"
	"text/template"

	"github.com/kyokomi/go-gitlab-client/gogitlab"
	color "github.com/mitchellh/colorstring"
)

const (
	titleCount          = 60
	nameCount           = 16
	labelCount           = 20
	outTitleReplaceText = " ..."
)

type templateExec struct {
	Issue *gogitlab.Issue
	Notes []*gogitlab.Note
}

const issueDetailTemplate = `
{{.Issue.Title}}

Author: [green]@{{.Issue.Author.Name}} [blue]{{.Issue.State}} [white]{{.Issue.CreatedAt}}
-------------------------------------------------------------------------------------------
{{.Issue.Description}}
-------------------------------------------------------------------------------------------
{{range $idx, $note := .Notes}}[green]@{{$note.Author.Name}}: [white]{{$note.Body}}
{{end}}`

// 対象ProjectのProjectNameを取得する.
func (gitLab *gitLabCli) GetProjectID(projectName string) (int, error) {
	projects, err := gitLab.Projects()
	if err != nil {
		return 0, err
	}

	for _, project := range projects {
		if project.Name == projectName {
			return project.ID, nil
		}
	}
	return 0, nil
}

func (gitLab *gitLabCli) GetProjectName(projectID int) (string, error) {
	project, err := gitLab.Project(strconv.Itoa(projectID))
	if err != nil {
		return "", err
	}
	return project.Name, nil
}

func (gitLab *gitLabCli) GetUserName(userID int) (string, error) {
	user, err := gitLab.User(strconv.Itoa(userID))
	if err != nil {
		return "", err
	}
	return user.Name, nil
}

//	/projects/:id/milestones/:milestone_id
func (gitLab *gitLabCli) GetMilestoneTitle(projectID, milestoneID int) (string, error) {
	milestone, err := gitLab.ProjectMilestone(strconv.Itoa(projectID), strconv.Itoa(milestoneID))
	if err != nil {
		return "", err
	}
	return milestone.Title, nil
}

func (gitLab *gitLabCli) CreateIssue(projectID int, values url.Values) ([]byte, error) {
	reader := strings.NewReader(values.Encode())
	data, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}
	res, err := gitLab.ProjectCreateIssues(projectID, data)
	if err != nil {
		return nil, err
	}

	return res, nil
}

func (gitLab *gitLabCli) EditIssue(projectID, issueID int, values url.Values) ([]byte, error) {
	reader := strings.NewReader(values.Encode())
	data, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}

	// localIssueID => issueID
	issue := gitLab.findIssueByID(projectID, issueID)
	if issue == nil {
		return nil, fmt.Errorf("issue not found")
	}

	res, err := gitLab.ProjectEditIssues(projectID, issue.ID, data)
	if err != nil {
		return nil, err
	}

	return res, nil
}

func (gitLab *gitLabCli) PrintIssue(projectID int, state string) {
	c := make(chan []*gogitlab.Issue)
	go gitLab.findIssueState(c, projectID, func(issues []*gogitlab.Issue) bool {
		if state != "" && issues[len(issues)-1].State != state {
			return true
		}
		return false
	})

	for {
		issues, ok := <-c
		if !ok {
			break
		}

		for _, issue := range issues {
			if state != "" && issue.State != state {
				continue
			}

			title := issue.Title
			if checkTrim(title) {
				title = trimPrefixIndex(title)
			}

			name := issue.Assignee.Name
			if checkTrim(name) {
				name = trimPrefixIndex(name)
			}

			labels := fmt.Sprint(issue.Labels)
			if checkTrim(name) {
				labels = trimPrefixIndex(labels)
			}

			titleCount := titleCount + ((utf8.RuneCountInString(title) - len(title)) / 2)
			nameCount := nameCount + ((utf8.RuneCountInString(name) - len(name)) / 2)
			labelCount := labelCount + ((utf8.RuneCountInString(labels) - len(labels)) / 2)
			t := fmt.Sprintf("[blue]#%%-4d %%-7s [white]%%-%ds [green]%%-%ds [red]%%-%ds [white]%%-33s / %%-33s", titleCount, nameCount, labelCount)

			fmt.Println(color.Color(fmt.Sprintf(t,
				issue.LocalID,
				issue.State,
				title,
				name,
				labels,
				issue.CreatedAt,
				issue.UpdatedAt)))
		}
	}
}

func (gitLab *gitLabCli) findIssueState(s chan<- []*gogitlab.Issue, projectID int, findFunc func([]*gogitlab.Issue) bool) {
	page := 1
	for {
		issues, err := gitLab.ProjectIssues(projectID, page)
		if err != nil || len(issues) == 0 {
			break
		}
		page++

		s <- issues

		if findFunc(issues) {
			break
		}
	}
	close(s)
}

func (gitLab *gitLabCli) findIssueByID(projectID, issueID int) *gogitlab.Issue {
	c := make(chan []*gogitlab.Issue)
	go gitLab.findIssueState(c, projectID, func(issues []*gogitlab.Issue) bool {
		for _, issue := range issues {
			if issue.LocalID == issueID {
				return true
			}
		}
		return false
	})

	for {
		issues, ok := <-c
		if !ok {
			break
		}

		for _, issue := range issues {
			if issue.LocalID != issueID {
				continue
			}

			return issue
		}
	}
	return nil
}

func (gitLab *gitLabCli) PrintIssueDetail(projectID, issueID int) error {
	if issueID == 0 {
		fmt.Println("not issue_id")
		return nil
	}

	issue := gitLab.findIssueByID(projectID, issueID)
	if issue == nil {
		fmt.Println("not found issue")
		return nil
	}

	notes, err := gitLab.IssuesNotes(projectID, issue.ID)
	if err != nil {
		return err
	}

	t := template.Must(template.New("issueDetail").Parse(issueDetailTemplate))
	var buf bytes.Buffer
	if err := t.Execute(&buf, templateExec{Issue: issue, Notes: notes}); err != nil {
		return err
	}
	fmt.Println(color.Color(buf.String()))

	return nil
}

func trimPrefixIndex(s string) string {
	t := ""
	for _, r := range s {
		if checkTrim(t + string(r) + outTitleReplaceText) {
			break
		}
		t += string(r)
	}
	return t + outTitleReplaceText
}

func checkTrim(t string) bool {
	diff := ((utf8.RuneCountInString(t) - len(t)) / 2)
	if utf8.RuneCountInString(t) > (titleCount + diff) {
		return true
	}
	return false
}
<commit_msg>refactor: IssueDetailTemplate<commit_after>package main

import (
	"strconv"

	"fmt"
	"io/ioutil"
	"net/url"
	"strings"
	"unicode/utf8"

	"bytes"
	"text/template"

	"github.com/kyokomi/go-gitlab-client/gogitlab"
	color "github.com/mitchellh/colorstring"
)

const (
	titleCount          = 60
	nameCount           = 16
	labelCount           = 20
	outTitleReplaceText = " ..."
)

type IssueDetailTemplate struct {
	*template.Template
	Issue *gogitlab.Issue
	Notes []*gogitlab.Note
}

func NewIssueDetailTemplate(issue *gogitlab.Issue, notes []*gogitlab.Note) *IssueDetailTemplate {
	return &IssueDetailTemplate{
		Template: template.Must(template.New("issueDetail").Parse(issueDetailTemplate)),
		Issue: issue,
		Notes: notes,
	}
}

func (t *IssueDetailTemplate) Print() error {
	var buf bytes.Buffer
	if err := t.Execute(&buf, t); err != nil {
		return err
	}
	fmt.Println(color.Color(buf.String()))

	return nil
}

const issueDetailTemplate = `
{{.Issue.Title}}

Author: [green]@{{.Issue.Author.Name}} [blue]{{.Issue.State}} [white]{{.Issue.CreatedAt}}
-------------------------------------------------------------------------------------------
{{.Issue.Description}}
-------------------------------------------------------------------------------------------
{{range $idx, $note := .Notes}}[green]@{{$note.Author.Name}}: [white]{{$note.Body}}
{{end}}`

// 対象ProjectのProjectNameを取得する.
func (gitLab *gitLabCli) GetProjectID(projectName string) (int, error) {
	projects, err := gitLab.Projects()
	if err != nil {
		return 0, err
	}

	for _, project := range projects {
		if project.Name == projectName {
			return project.ID, nil
		}
	}
	return 0, nil
}

func (gitLab *gitLabCli) GetProjectName(projectID int) (string, error) {
	project, err := gitLab.Project(strconv.Itoa(projectID))
	if err != nil {
		return "", err
	}
	return project.Name, nil
}

func (gitLab *gitLabCli) GetUserName(userID int) (string, error) {
	user, err := gitLab.User(strconv.Itoa(userID))
	if err != nil {
		return "", err
	}
	return user.Name, nil
}

//	/projects/:id/milestones/:milestone_id
func (gitLab *gitLabCli) GetMilestoneTitle(projectID, milestoneID int) (string, error) {
	milestone, err := gitLab.ProjectMilestone(strconv.Itoa(projectID), strconv.Itoa(milestoneID))
	if err != nil {
		return "", err
	}
	return milestone.Title, nil
}

func (gitLab *gitLabCli) CreateIssue(projectID int, values url.Values) ([]byte, error) {
	reader := strings.NewReader(values.Encode())
	data, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}
	res, err := gitLab.ProjectCreateIssues(projectID, data)
	if err != nil {
		return nil, err
	}

	return res, nil
}

func (gitLab *gitLabCli) EditIssue(projectID, issueID int, values url.Values) ([]byte, error) {
	reader := strings.NewReader(values.Encode())
	data, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}

	// localIssueID => issueID
	issue := gitLab.findIssueByID(projectID, issueID)
	if issue == nil {
		return nil, fmt.Errorf("issue not found")
	}

	res, err := gitLab.ProjectEditIssues(projectID, issue.ID, data)
	if err != nil {
		return nil, err
	}

	return res, nil
}

func (gitLab *gitLabCli) PrintIssue(projectID int, state string) {
	c := make(chan []*gogitlab.Issue)
	go gitLab.findIssueState(c, projectID, func(issues []*gogitlab.Issue) bool {
		if state != "" && issues[len(issues)-1].State != state {
			return true
		}
		return false
	})

	for {
		issues, ok := <-c
		if !ok {
			break
		}

		for _, issue := range issues {
			if state != "" && issue.State != state {
				continue
			}

			title := issue.Title
			if checkTrim(title) {
				title = trimPrefixIndex(title)
			}

			name := issue.Assignee.Name
			if checkTrim(name) {
				name = trimPrefixIndex(name)
			}

			labels := fmt.Sprint(issue.Labels)
			if checkTrim(name) {
				labels = trimPrefixIndex(labels)
			}

			titleCount := titleCount + ((utf8.RuneCountInString(title) - len(title)) / 2)
			nameCount := nameCount + ((utf8.RuneCountInString(name) - len(name)) / 2)
			labelCount := labelCount + ((utf8.RuneCountInString(labels) - len(labels)) / 2)
			t := fmt.Sprintf("[blue]#%%-4d %%-7s [white]%%-%ds [green]%%-%ds [red]%%-%ds [white]%%-33s / %%-33s", titleCount, nameCount, labelCount)

			fmt.Println(color.Color(fmt.Sprintf(t,
				issue.LocalID,
				issue.State,
				title,
				name,
				labels,
				issue.CreatedAt,
				issue.UpdatedAt)))
		}
	}
}

func (gitLab *gitLabCli) findIssueState(s chan<- []*gogitlab.Issue, projectID int, findFunc func([]*gogitlab.Issue) bool) {
	page := 1
	for {
		issues, err := gitLab.ProjectIssues(projectID, page)
		if err != nil || len(issues) == 0 {
			break
		}
		page++

		s <- issues

		if findFunc(issues) {
			break
		}
	}
	close(s)
}

func (gitLab *gitLabCli) findIssueByID(projectID, issueID int) *gogitlab.Issue {
	c := make(chan []*gogitlab.Issue)
	go gitLab.findIssueState(c, projectID, func(issues []*gogitlab.Issue) bool {
		for _, issue := range issues {
			if issue.LocalID == issueID {
				return true
			}
		}
		return false
	})

	for {
		issues, ok := <-c
		if !ok {
			break
		}

		for _, issue := range issues {
			if issue.LocalID != issueID {
				continue
			}

			return issue
		}
	}
	return nil
}

func (gitLab *gitLabCli) PrintIssueDetail(projectID, issueID int) error {
	if issueID == 0 {
		fmt.Println("not issue_id")
		return nil
	}

	issue := gitLab.findIssueByID(projectID, issueID)
	if issue == nil {
		fmt.Println("not found issue")
		return nil
	}

	notes, err := gitLab.IssuesNotes(projectID, issue.ID)
	if err != nil {
		return err
	}

	NewIssueDetailTemplate(issue, notes).Print()

	return nil
}

func trimPrefixIndex(s string) string {
	t := ""
	for _, r := range s {
		if checkTrim(t + string(r) + outTitleReplaceText) {
			break
		}
		t += string(r)
	}
	return t + outTitleReplaceText
}

func checkTrim(t string) bool {
	diff := ((utf8.RuneCountInString(t) - len(t)) / 2)
	if utf8.RuneCountInString(t) > (titleCount + diff) {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

func main() {

	tapi := NewTwitterAPI("sDggzGbHbyAfl5fJ87XOCA", "MOCQDL7ot7qIxMwYL5x1mMAqiYBYxNTxPWS6tc6hw")

	auth_url, err := tapi.GetRequestAuthorizationURL()
	if err != nil {
		fmt.Println(err.String())
		return
	}

	var pin string
	fmt.Printf("Open %s\n", auth_url)
	fmt.Printf("PIN Number: ")
	fmt.Scanln(&pin)

	tapi.SetPIN(pin)

	home_tl, err := tapi.HomeTimeline()

	if err != nil {
		fmt.Println(err.String())
		return
	}

	for _, tweet := range home_tl.Tweets {
		rt_count, okstr := tweet.Retweet_count.(string)
		if !okstr {
			if rt_count_int, okint := tweet.Retweet_count.(int64); okint {
				rt_count = fmt.Sprintf("%d", rt_count_int)
			}
		}
		fmt.Printf("[%s] %s (%s)\n", *tweet.User.Screen_name, *tweet.Text, rt_count)
	}

//	foo, posterr := goauthcon.Post(
//		"http://api.twitter.com/1/statuses/update.json",
//		oauth.Params{
//			&oauth.Pair{
//				Key:"status",
//				Value:"Test posting using Gockel prototype",
//			},
//		}, at )
//
//	fmt.Printf("foo = %v\n", foo)
//
//	if posterr != nil {
//		fmt.Println(err.String())
//		return
//	}
//
//	fmt.Println("Twitter Status is updated")
}
<commit_msg>load/save access token.<commit_after>package main

import (
	"fmt"
	"os"
	"json"
	"io/ioutil"
	oauth "github.com/hokapoka/goauth"
)

func main() {

	tapi := NewTwitterAPI("sDggzGbHbyAfl5fJ87XOCA", "MOCQDL7ot7qIxMwYL5x1mMAqiYBYxNTxPWS6tc6hw")

	at, aterr := LoadAccessToken()

	if aterr == nil {
		tapi.SetAccessToken(at)
	} else {
		auth_url, err := tapi.GetRequestAuthorizationURL()
		if err != nil {
			fmt.Println(err.String())
			return
		}

		var pin string
		fmt.Printf("Open %s\n", auth_url)
		fmt.Printf("PIN Number: ")
		fmt.Scanln(&pin)

		tapi.SetPIN(pin)

		if saveerr := SaveAccessToken(tapi.GetAccessToken()); saveerr != nil {
			fmt.Printf("saving access token failed: %s\n", saveerr.String())
			return
		}
	}

	home_tl, err := tapi.HomeTimeline()

	if err != nil {
		fmt.Println(err.String())
		return
	}

	for _, tweet := range home_tl.Tweets {
		rt_count, okstr := tweet.Retweet_count.(string)
		if !okstr {
			if rt_count_int, okint := tweet.Retweet_count.(int64); okint {
				rt_count = fmt.Sprintf("%d", rt_count_int)
			}
		}
		fmt.Printf("[%s] %s (%s)\n", *tweet.User.Screen_name, *tweet.Text, rt_count)
	}

//	foo, posterr := goauthcon.Post(
//		"http://api.twitter.com/1/statuses/update.json",
//		oauth.Params{
//			&oauth.Pair{
//				Key:"status",
//				Value:"Test posting using Gockel prototype",
//			},
//		}, at )
//
//	fmt.Printf("foo = %v\n", foo)
//
//	if posterr != nil {
//		fmt.Println(err.String())
//		return
//	}
//
//	fmt.Println("Twitter Status is updated")
}

func SaveAccessToken(at *oauth.AccessToken) os.Error {
	data, marshalerr := json.Marshal(at)
	if marshalerr != nil {
		return marshalerr
	}

	f, ferr := os.OpenFile("access_token.json", os.O_WRONLY | os.O_CREATE, 0600)
	if ferr != nil {
		return ferr
	}
	defer f.Close()

	_, werr := f.Write(data)
	if werr != nil {
		return werr
	}

	return nil
}

func LoadAccessToken() (*oauth.AccessToken, os.Error) {
	f, ferr := os.Open("access_token.json")
	if ferr != nil {
		return nil, ferr
	}
	defer f.Close()

	data, readerr := ioutil.ReadAll(f)
	if readerr != nil {
		return nil, readerr
	}

	at := &oauth.AccessToken{}

	err := json.Unmarshal(data, at)
	if err != nil {
		return nil, err
	}

	return at, nil
}
<|endoftext|>


<commit_before>package gohost

import (
	"fmt"
	"net"
	"net/http"

	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	"github.com/rs/cors"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

// ServeGRPC starts a gRPC endpoint for the given server.
func ServeGRPC(server Server, grpcAddr string, opts []grpc.ServerOption) error {
	// start listening
	lis, err := net.Listen("tcp", grpcAddr)
	if err != nil {
		return fmt.Errorf("failed to listen: %v", err)
	}

	// register server
	grpcServer := grpc.NewServer(opts...)
	server.RegisterServer(grpcServer)

	// start server
	if err := grpcServer.Serve(lis); err != nil {
		return fmt.Errorf("failed to serve gRPC endpoint: %v", err)
	}

	return nil
}

// ServeGRPCWithTLS starts a gRPC endpoint for the given server with TLS enabled.
func ServeGRPCWithTLS(server Server, grpcAddr string, opts []grpc.ServerOption, certFile string, keyFile string) error {
	// create TLS credentials
	creds, err := credentials.NewServerTLSFromFile(certFile, keyFile)
	if err != nil {
		return fmt.Errorf("failed to generate TLS credentials: %v", err)
	}

	// add TLS credentials to options
	opts = append(opts, grpc.Creds(creds))

	// start server
	return ServeGRPC(server, grpcAddr, opts)
}

// ServeHTTP starts an HTTP endpoint for a given server. This is a gateway pointing to a gRPC endpoint.
func ServeHTTP(server Server, httpAddr string, grpcAddr string, enableCORS bool, opts []grpc.DialOption) error {
	// create context
	ctx := context.Background()
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	// register server
	var handler http.Handler
	handler = runtime.NewServeMux()
	err := server.RegisterHandler(ctx, handler, grpcAddr, opts)
	if err != nil {
		return fmt.Errorf("failed to register HTTP endpoint: %v", err)
	}

	// enable CORS if requested
	if enableCORS {
		handler = cors.AllowAll().Handler(handler)
	}

	// start server
	err = http.ListenAndServe(httpAddr, handler)
	if err != nil {
		return fmt.Errorf("failed to serve HTTP endpoint: %v", err)
	}

	return nil
}
<commit_msg>Added TLS function for the http endpoint.<commit_after>package gohost

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/http"

	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	"github.com/rs/cors"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

// ServeGRPC starts a gRPC endpoint for the given server.
func ServeGRPC(server Server, grpcAddr string, opts []grpc.ServerOption) error {
	// start listening
	lis, err := net.Listen("tcp", grpcAddr)
	if err != nil {
		return fmt.Errorf("failed to listen: %v", err)
	}

	// register server
	grpcServer := grpc.NewServer(opts...)
	server.RegisterServer(grpcServer)

	// start server
	if err := grpcServer.Serve(lis); err != nil {
		return fmt.Errorf("failed to serve gRPC endpoint: %v", err)
	}

	return nil
}

// ServeGRPCWithTLS starts a gRPC endpoint for the given server with TLS enabled.
func ServeGRPCWithTLS(server Server, grpcAddr string, opts []grpc.ServerOption, certFile string, keyFile string) error {
	// create TLS credentials
	creds, err := credentials.NewServerTLSFromFile(certFile, keyFile)
	if err != nil {
		return fmt.Errorf("failed to generate TLS credentials: %v", err)
	}

	// add TLS credentials to options
	opts = append(opts, grpc.Creds(creds))

	// start server
	return ServeGRPC(server, grpcAddr, opts)
}

// ServeHTTP starts an HTTP endpoint for a given server. This is a gateway pointing to a gRPC endpoint.
func ServeHTTP(server Server, httpAddr string, grpcAddr string, enableCORS bool, opts []grpc.DialOption) error {
	// start server
	return serveHTTP(server, httpAddr, grpcAddr, enableCORS, opts, func(addr string, handler http.Handler) error {
		return http.ListenAndServe(addr, handler)
	})
}

// ServeHTTPWithTLS starts an HTTP endpoint for a given server with TLS enabled. This is a gateway pointing to a gRPC endpoint.
func ServeHTTPWithTLS(server Server, httpAddr string, grpcAddr string, enableCORS bool, opts []grpc.DialOption, certFile string, keyFile string, insecureSkipVerify bool) error {
	// create TLS credentials
	creds := credentials.NewTLS(&tls.Config{
		InsecureSkipVerify: insecureSkipVerify,
	})

	// add TLS credentials to options
	opts = append(opts, grpc.WithTransportCredentials(creds))

	// start server
	return serveHTTP(server, httpAddr, grpcAddr, enableCORS, opts, func(addr string, handler http.Handler) error {
		return http.ListenAndServeTLS(addr, certFile, keyFile, handler)
	})
}

// serveHTTP starts an HTTP endpoint for a given server. This is a gateway pointing to a gRPC endpoint.
func serveHTTP(server Server, httpAddr string, grpcAddr string, enableCORS bool, opts []grpc.DialOption, listenAndServe func(addr string, handler http.Handler) error) error {
	// create context
	ctx := context.Background()
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	// register server
	var handler http.Handler
	handler = runtime.NewServeMux()
	err := server.RegisterHandler(ctx, handler, grpcAddr, opts)
	if err != nil {
		return fmt.Errorf("failed to register HTTP endpoint: %v", err)
	}

	// enable CORS if requested
	if enableCORS {
		handler = cors.AllowAll().Handler(handler)
	}

	// start server
	err = listenAndServe(httpAddr, handler)
	if err != nil {
		return fmt.Errorf("failed to serve HTTP endpoint: %v", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"time"

	"github.com/fatih/color"
	"github.com/tatsushid/go-fastping"
)

var pingIP, listenPort string
var flagExpandDNS, flagShowRTT, flagEnableWeb, flagNoColor bool
var width = "2"

var ipList []string
var ipListMap map[string][]string
var pingStats map[string]stats

func init() {
	flag.BoolVar(&flagNoColor, "nocolor", false, "disable color output")
	flag.BoolVar(&flagEnableWeb, "web", false, "enable webserver")
	flag.BoolVar(&flagExpandDNS, "expand", false, "use all available ip's (ipv4/ipv6) of a hostname (multiple A, AAAA)")
	flag.StringVar(&listenPort, "port", "8887", "port the webserver listens on")
	flag.StringVar(&pingIP, "hosts", "", "ip addresses/hosts to ping, space seperated (e.g \"8.8.8.8 8.8.4.4 google.com 2a00:1450:400c:c07::66\")")
	flag.BoolVar(&flagShowRTT, "showrtt", false, "show roundtrip time in ms")
	flag.Parse()
	if flag.NFlag() == 0 {
		fmt.Println("usage: ")
		flag.PrintDefaults()
		os.Exit(2)
	}
	if flagShowRTT {
		width = "3"
	}
	if flagNoColor {
		color.NoColor = true
	}
}

type milliDuration time.Duration

func (hd milliDuration) String() string {
	milliseconds := time.Duration(hd).Nanoseconds()
	milliseconds = milliseconds / 1000000
	if milliseconds > 1000 {
		return fmt.Sprintf(">1s")
	}
	return fmt.Sprintf("%"+width+"d", milliseconds)
}

func (hd milliDuration) Int() int {
	milliseconds := time.Duration(hd).Nanoseconds()
	milliseconds = milliseconds / 1000000
	return int(milliseconds)
}

type gomphs struct {
	latestEntry   []byte
	pingIP        string
	flagShowRTT   bool
	flagExpandDNS bool
	IPList        []string
	IPListMap     map[string][]string
}

type stats struct {
	min   int
	max   int
	avg   int
	count int
	rtts  []int
}

func (g *gomphs) update(result map[string]string) {
	record := []string{time.Now().Format("2006/01/02 15:04:05")}
	for _, key := range g.IPList {
		for _, value := range g.IPListMap[key] {
			if result[value] != "" {
				res := strings.Replace(result[value], " ", "", -1)
				record = append(record, res)
			} else {
				record = append(record, "-10")
			}
		}
	}
	g.latestEntry, _ = json.Marshal(record)
}

func main() {
	var rowcounter int
	ipListMap = make(map[string][]string)
	pingStats = make(map[string]stats)
	g := &gomphs{}

	if flagEnableWeb {
		listener, err := net.Listen("tcp", ":"+listenPort)
		if err != nil {
			log.Fatal(err)
		}
		go http.Serve(listener, nil)
		http.HandleFunc("/read.json", webReadJSONHandler(g))
		http.HandleFunc("/stream", webStreamHandler)
	}

	result := make(map[string]string)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		for range c {
			fmt.Printf("\n%-38s: %4s %4s %4s %5s\n", "source", "min", "max", "avg", "ploss")
			for key, stat := range pingStats {
				ploss := 0
				for _, entry := range stat.rtts[0:rowcounter] {
					if entry != -1 {
						if entry > stat.max {
							stat.max = entry
						}
						if entry < stat.min {
							stat.min = entry
						}
						stat.avg = stat.avg + entry
					} else {
						ploss++
					}
				}
				if ploss == rowcounter {
					stat.min = -1
					stat.max = -1
					stat.avg = -1
				} else {
					stat.avg = stat.avg / (rowcounter - ploss)
				}
				plosspct := float32(ploss) / float32(rowcounter) * 100
				fmt.Printf("%-38s: %4d %4d %4d %5d(%.2f%%)\n", key, stat.min, stat.max, stat.avg, ploss, plosspct)
			}
			os.Exit(0)
		}
	}()
	p := fastping.NewPinger()

	for _, host := range strings.Fields(pingIP) {
		if flagExpandDNS {
			lookups, err := net.LookupIP(host)
			checkHostErr(host, err)
			ipList = append(ipList, host)
			for _, ip := range lookups {
				ra := &net.IPAddr{IP: ip}
				p.AddIPAddr(ra)
				ipListMap[host] = append(ipListMap[host], ra.String())
			}
		} else {
			ra, err := net.ResolveIPAddr("ip:icmp", host)
			checkHostErr(host, err)
			p.AddIPAddr(ra)
			ipList = append(ipList, ra.String())
			ipListMap[ra.String()] = append(ipListMap[ra.String()], ra.String())
		}
	}
	g.IPList = ipList
	g.IPListMap = ipListMap
	p.OnRecv = func(addr *net.IPAddr, rtt time.Duration) {
		result[addr.String()] = milliDuration(rtt).String()
		stats := pingStats[addr.String()]
		if stats.count == 0 {
			stats.min = 100000
		}
		stats.rtts = append(stats.rtts, milliDuration(rtt).Int())
		pingStats[addr.String()] = stats
	}
	p.OnIdle = func() {
		if rowcounter%25 == 0 {
			printHeader()
			if rowcounter == 10000 {
				rowcounter = 0
			}
		}
		g.update(result)
		fmt.Printf("%04d", rowcounter+1)
		for _, key := range ipList {
			for _, value := range ipListMap[key] {
				fmt.Printf(" ")
				if result[value] != "" {
					color.Set(color.BgGreen, color.FgYellow, color.Bold)
					if flagShowRTT {
						fmt.Printf("%"+width+"s", result[value])
					} else {
						fmt.Printf("%"+width+"s", ".")
					}
					color.Unset()
				} else {
					stats := pingStats[value]
					stats.rtts = append(stats.rtts, -1)
					pingStats[value] = stats
					color.Set(color.BgRed, color.FgYellow, color.Bold)
					fmt.Printf("%"+width+"s", "!")
					color.Unset()
				}
			}
		}
		fmt.Println(" ")
		result = make(map[string]string)
		rowcounter++
	}
	if flagExpandDNS {
		printFirstHeader()
	}
	for {
		err := p.Run()
		if err != nil {
			log.Fatal(err)
		}
	}
}
<commit_msg>Display ploss/count<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"time"

	"github.com/fatih/color"
	"github.com/tatsushid/go-fastping"
)

var pingIP, listenPort string
var flagExpandDNS, flagShowRTT, flagEnableWeb, flagNoColor bool
var width = "2"

var ipList []string
var ipListMap map[string][]string
var pingStats map[string]stats

func init() {
	flag.BoolVar(&flagNoColor, "nocolor", false, "disable color output")
	flag.BoolVar(&flagEnableWeb, "web", false, "enable webserver")
	flag.BoolVar(&flagExpandDNS, "expand", false, "use all available ip's (ipv4/ipv6) of a hostname (multiple A, AAAA)")
	flag.StringVar(&listenPort, "port", "8887", "port the webserver listens on")
	flag.StringVar(&pingIP, "hosts", "", "ip addresses/hosts to ping, space seperated (e.g \"8.8.8.8 8.8.4.4 google.com 2a00:1450:400c:c07::66\")")
	flag.BoolVar(&flagShowRTT, "showrtt", false, "show roundtrip time in ms")
	flag.Parse()
	if flag.NFlag() == 0 {
		fmt.Println("usage: ")
		flag.PrintDefaults()
		os.Exit(2)
	}
	if flagShowRTT {
		width = "3"
	}
	if flagNoColor {
		color.NoColor = true
	}
}

type milliDuration time.Duration

func (hd milliDuration) String() string {
	milliseconds := time.Duration(hd).Nanoseconds()
	milliseconds = milliseconds / 1000000
	if milliseconds > 1000 {
		return fmt.Sprintf(">1s")
	}
	return fmt.Sprintf("%"+width+"d", milliseconds)
}

func (hd milliDuration) Int() int {
	milliseconds := time.Duration(hd).Nanoseconds()
	milliseconds = milliseconds / 1000000
	return int(milliseconds)
}

type gomphs struct {
	latestEntry   []byte
	pingIP        string
	flagShowRTT   bool
	flagExpandDNS bool
	IPList        []string
	IPListMap     map[string][]string
}

type stats struct {
	min   int
	max   int
	avg   int
	count int
	rtts  []int
}

func (g *gomphs) update(result map[string]string) {
	record := []string{time.Now().Format("2006/01/02 15:04:05")}
	for _, key := range g.IPList {
		for _, value := range g.IPListMap[key] {
			if result[value] != "" {
				res := strings.Replace(result[value], " ", "", -1)
				record = append(record, res)
			} else {
				record = append(record, "-10")
			}
		}
	}
	g.latestEntry, _ = json.Marshal(record)
}

func main() {
	var rowcounter int
	ipListMap = make(map[string][]string)
	pingStats = make(map[string]stats)
	g := &gomphs{}

	if flagEnableWeb {
		listener, err := net.Listen("tcp", ":"+listenPort)
		if err != nil {
			log.Fatal(err)
		}
		go http.Serve(listener, nil)
		http.HandleFunc("/read.json", webReadJSONHandler(g))
		http.HandleFunc("/stream", webStreamHandler)
	}

	result := make(map[string]string)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		for range c {
			fmt.Printf("\n%-38s: %4s %4s %4s %5s\n", "source", "min", "max", "avg", "ploss")
			for key, stat := range pingStats {
				ploss := 0
				for _, entry := range stat.rtts[0:rowcounter] {
					if entry != -1 {
						if entry > stat.max {
							stat.max = entry
						}
						if entry < stat.min {
							stat.min = entry
						}
						stat.avg = stat.avg + entry
					} else {
						ploss++
					}
				}
				if ploss == rowcounter {
					stat.min = -1
					stat.max = -1
					stat.avg = -1
				} else {
					stat.avg = stat.avg / (rowcounter - ploss)
				}
				plosspct := float32(ploss) / float32(rowcounter) * 100
				fmt.Printf("%-38s: %4d %4d %4d %5d/%d (%.2f%%)\n", key, stat.min, stat.max, stat.avg, ploss, rowcounter, plosspct)
			}
			os.Exit(0)
		}
	}()
	p := fastping.NewPinger()

	for _, host := range strings.Fields(pingIP) {
		if flagExpandDNS {
			lookups, err := net.LookupIP(host)
			checkHostErr(host, err)
			ipList = append(ipList, host)
			for _, ip := range lookups {
				ra := &net.IPAddr{IP: ip}
				p.AddIPAddr(ra)
				ipListMap[host] = append(ipListMap[host], ra.String())
			}
		} else {
			ra, err := net.ResolveIPAddr("ip:icmp", host)
			checkHostErr(host, err)
			p.AddIPAddr(ra)
			ipList = append(ipList, ra.String())
			ipListMap[ra.String()] = append(ipListMap[ra.String()], ra.String())
		}
	}
	g.IPList = ipList
	g.IPListMap = ipListMap
	p.OnRecv = func(addr *net.IPAddr, rtt time.Duration) {
		result[addr.String()] = milliDuration(rtt).String()
		stats := pingStats[addr.String()]
		if stats.count == 0 {
			stats.min = 100000
		}
		stats.rtts = append(stats.rtts, milliDuration(rtt).Int())
		pingStats[addr.String()] = stats
	}
	p.OnIdle = func() {
		if rowcounter%25 == 0 {
			printHeader()
			if rowcounter == 10000 {
				rowcounter = 0
			}
		}
		g.update(result)
		fmt.Printf("%04d", rowcounter+1)
		for _, key := range ipList {
			for _, value := range ipListMap[key] {
				fmt.Printf(" ")
				if result[value] != "" {
					color.Set(color.BgGreen, color.FgYellow, color.Bold)
					if flagShowRTT {
						fmt.Printf("%"+width+"s", result[value])
					} else {
						fmt.Printf("%"+width+"s", ".")
					}
					color.Unset()
				} else {
					stats := pingStats[value]
					stats.rtts = append(stats.rtts, -1)
					pingStats[value] = stats
					color.Set(color.BgRed, color.FgYellow, color.Bold)
					fmt.Printf("%"+width+"s", "!")
					color.Unset()
				}
			}
		}
		fmt.Println(" ")
		result = make(map[string]string)
		rowcounter++
	}
	if flagExpandDNS {
		printFirstHeader()
	}
	for {
		err := p.Run()
		if err != nil {
			log.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"github.com/k0kubun/gosick/scheme"
	"log"
	"os"
)

func printShellPrompt(indentLevel int) {
	if indentLevel == 0 {
		fmt.Print("gosick> ")
	} else if indentLevel > 0 {
		fmt.Print("gosick* ")

		for length := indentLevel; length > 0; length-- {
			fmt.Print("  ")
		}
	} else {
		panic("Negative indent level")
	}
}

func invokeInteractiveShell() {
	commandLine := bufio.NewReader(os.Stdin)

	for {
		indentLevel := 0
		expression := ""

		for {
			printShellPrompt(indentLevel)

			currentLine, err := commandLine.ReadString('\n')
			if err != nil {
				log.Fatal(err)
				return
			}
			expression += currentLine

			interpreter := scheme.NewInterpreter(expression)
			if indentLevel = interpreter.IndentLevel(); indentLevel == 0 {
				interpreter.Eval()
				break
			}
		}
	}
}

func main() {
	invokeInteractiveShell()
}
<commit_msg>Use linenoise to support cursor control<commit_after>package main

import (
	"fmt"
	"github.com/GeertJohan/go.linenoise"
	"github.com/k0kubun/gosick/scheme"
	"log"
	"strings"
)

func shellPrompt(indentLevel int) string {
	if indentLevel == 0 {
		return "gosick> "
	} else if indentLevel > 0 {
		return fmt.Sprintf("gosick* %s", strings.Repeat("  ", indentLevel))
	} else {
		panic("Negative indent level")
	}
}

func invokeInteractiveShell() {
	for {
		indentLevel := 0
		expression := ""

		for {
			currentLine, err := linenoise.Line(shellPrompt(indentLevel))
			if err != nil {
				log.Fatal(err)
				return
			}
			if len(currentLine) == 0 {
				continue
			}
			expression += currentLine

			interpreter := scheme.NewInterpreter(expression)
			if indentLevel = interpreter.IndentLevel(); indentLevel == 0 {
				interpreter.Eval()
				break
			}
		}
	}
}

func main() {
	invokeInteractiveShell()
}
<|endoftext|>