<commit_before>package db

import (
	"encoding/binary"
	"fmt"
	"os"
	"strings"
	"time"

	turtle "github.com/gtfierro/hod/goraptor"
	"github.com/op/go-logging"
	"github.com/pkg/errors"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/zhangxinngang/murmur"
)

// logger
var log *logging.Logger
var emptyHash = [4]byte{0, 0, 0, 0}

func init() {
	log = logging.MustGetLogger("hod")
	var format = "%{color}%{level} %{shortfile} %{time:Jan 02 15:04:05} %{color:reset} ▶ %{message}"
	var logBackend = logging.NewLogBackend(os.Stderr, "", 0)
	logBackendLeveled := logging.AddModuleLevel(logBackend)
	logging.SetBackend(logBackendLeveled)
	logging.SetFormatter(logging.MustStringFormatter(format))
}

type DB struct {
	// store []byte(entity URI) => primary key
	entityDB *leveldb.DB
	// store primary key => [](entity URI)
	pkDB *leveldb.DB
	// predicate index: stores "children" of predicates
	predDB    *leveldb.DB
	predIndex map[turtle.URI]*PredicateEntity
	// graph structure
	graphDB *leveldb.DB
	// store relationships and their inverses
	relationships map[turtle.URI]turtle.URI
	// store the namespace prefixes as strings
	namespaces map[string]string
}

func NewDB(path string) (*DB, error) {
	path = strings.TrimSuffix(path, "/")

	// set up entity, pk databases
	entityDBPath := path + "/db-entities"
	entityDB, err := leveldb.OpenFile(entityDBPath, nil)
	if err != nil {
		return nil, errors.Wrapf(err, "Could not open entityDB file %s", entityDBPath)
	}

	pkDBPath := path + "/db-pk"
	pkDB, err := leveldb.OpenFile(pkDBPath, nil)
	if err != nil {
		return nil, errors.Wrapf(err, "Could not open pkDB file %s", pkDBPath)
	}

	graphDBPath := path + "/db-graph"
	graphDB, err := leveldb.OpenFile(graphDBPath, nil)
	if err != nil {
		return nil, errors.Wrapf(err, "Could not open graphDB file %s", graphDBPath)
	}
	predDBPath := path + "/db-pred"
	predDB, err := leveldb.OpenFile(predDBPath, nil)
	if err != nil {
		return nil, errors.Wrapf(err, "Could not open predDB file %s", predDBPath)
	}

	db := &DB{
		entityDB:      entityDB,
		pkDB:          pkDB,
		graphDB:       graphDB,
		predDB:        predDB,
		predIndex:     make(map[turtle.URI]*PredicateEntity),
		relationships: make(map[turtle.URI]turtle.URI),
	}

	return db, nil
}

// hashes the given URI into the byte array
func (db *DB) hashURI(u turtle.URI, dest []byte, salt uint32) {
	var hash uint32
	if len(dest) < 4 {
		dest = make([]byte, 4)
	}
	if salt > 0 {
		saltbytes := make([]byte, 4)
		binary.LittleEndian.PutUint32(saltbytes, salt)
		hash = murmur.Murmur3(append(u.Bytes(), saltbytes...))
	} else {
		hash = murmur.Murmur3(u.Bytes())
	}
	binary.LittleEndian.PutUint32(dest, hash)
}

// for each part of the triple (subject, predicate, object), we check if its already in the entity database.
// If it is, we can skip it. If not, we generate a murmur3 hash for the entity, and then
// 0. check if we've already inserted the entity (skip if we already have)
// 1. check if the hash is unique (check membership in pk db) - if it isn't then we add a salt and check again
// 2. insert hash => []byte(entity) into pk db
// 3. insert []byte(entity) => hash into entity db
func (db *DB) insertEntity(entity turtle.URI, hashdest []byte, enttx, pktx *leveldb.Transaction) error {
	// check if we've inserted Subject already
	if exists, err := enttx.Has(entity.Bytes(), nil); err == nil && exists {
		// populate hash anyway
		hash, err := enttx.Get(entity.Bytes(), nil)
		copy(hashdest, hash[:])
		return err
	} else if err != nil {
		return errors.Wrapf(err, "Error checking db membership for %s", entity.String())
	}
	// generate the hash
	var salt = uint32(0)
	db.hashURI(entity, hashdest, salt)
	for {
		if exists, err := pktx.Has(hashdest, nil); err == nil && exists {
			log.Warning("hash exists")
			salt += 1
			db.hashURI(entity, hashdest, salt)
		} else if err != nil {
			return errors.Wrapf(err, "Error checking db membership for %v", hashdest)
		} else {
			break
		}
	}

	// insert the hash into the entity and prefix dbs
	if err := enttx.Put(entity.Bytes(), hashdest, nil); err != nil {
		return errors.Wrapf(err, "Error inserting entity %s", entity.String())
	}
	if err := pktx.Put(hashdest, entity.Bytes(), nil); err != nil {
		return errors.Wrapf(err, "Error inserting pk %s", hashdest)
	}
	return nil
}

func (db *DB) loadPredicateEntity(predicate turtle.URI, _predicateHash, _subjectHash, _objectHash []byte, predtx *leveldb.Transaction) error {
	var (
		pred          *PredicateEntity
		found         bool
		predicateHash [4]byte
		subjectHash   [4]byte
		objectHash    [4]byte
	)
	copy(predicateHash[:], _predicateHash)
	copy(subjectHash[:], _subjectHash)
	copy(objectHash[:], _objectHash)

	//log.Debug(predicate)
	if pred, found = db.predIndex[predicate]; !found {
		pred = NewPredicateEntity()
		pred.PK = predicateHash
	}

	pred.AddSubjectObject(subjectHash, objectHash)
	db.predIndex[predicate] = pred
	//TODO: save predicate index

	return nil
}

func (db *DB) LoadRelationships(dataset turtle.DataSet) error {
	// iterate through dataset, and pull out all that have a "rdf:type" of "owl:ObjectProperty"
	// then we want to find the mapping that has "owl:inverseOf"
	var relationships = make(map[turtle.URI]struct{})

	rdf_namespace, found := dataset.Namespaces["rdf"]
	if !found {
		return errors.New("Relationships has no rdf namespace")
	}
	owl_namespace, found := dataset.Namespaces["owl"]
	if !found {
		return errors.New("Relationships has no owl namespace")
	}

	for _, triple := range dataset.Triples {
		if triple.Predicate.Namespace == rdf_namespace &&
			triple.Predicate.Value == "type" &&
			triple.Object.Namespace == owl_namespace &&
			triple.Object.Value == "ObjectProperty" {
			relationships[triple.Subject] = struct{}{}
		}
	}

	for _, triple := range dataset.Triples {
		if triple.Predicate.Namespace == owl_namespace && triple.Predicate.Value == "inverseOf" {
			// check that the subject/object of the inverseOf relationships are both actually relationships
			if _, found := relationships[triple.Subject]; !found {
				continue
			}
			if _, found := relationships[triple.Object]; !found {
				continue
			}
			db.relationships[triple.Subject] = triple.Object
			db.relationships[triple.Object] = triple.Subject
		}
	}

	return nil
}

func (db *DB) LoadDataset(dataset turtle.DataSet) error {
	start := time.Now()
	db.namespaces = dataset.Namespaces
	// start transactions
	enttx, err := db.entityDB.OpenTransaction()
	if err != nil {
		return errors.Wrap(err, "Could not open transaction on entity dataset")
	}
	pktx, err := db.pkDB.OpenTransaction()
	if err != nil {
		return errors.Wrap(err, "Could not open transaction on pk dataset")
	}
	predtx, err := db.predDB.OpenTransaction()
	if err != nil {
		return errors.Wrap(err, "Could not open transaction on pred dataset")
	}
	// load triples and primary keys
	var (
		subjectHash   = make([]byte, 4)
		predicateHash = make([]byte, 4)
		objectHash    = make([]byte, 4)
	)
	for _, triple := range dataset.Triples {
		if err := db.insertEntity(triple.Subject, subjectHash, enttx, pktx); err != nil {
			return err
		}
		if err := db.insertEntity(triple.Predicate, predicateHash, enttx, pktx); err != nil {
			return err
		}
		if err := db.insertEntity(triple.Object, objectHash, enttx, pktx); err != nil {
			return err
		}
		if err := db.loadPredicateEntity(triple.Predicate, predicateHash, subjectHash, objectHash, predtx); err != nil {
			return err
		}
	}

	// finish those transactions
	if err := enttx.Commit(); err != nil {
		return errors.Wrap(err, "Could not commit transaction")
	}
	if err := pktx.Commit(); err != nil {
		return errors.Wrap(err, "Could not commit transaction")
	}
	if err := predtx.Commit(); err != nil {
		return errors.Wrap(err, "Could not commit transaction")
	}
	log.Infof("Built lookup tables in %s", time.Since(start))

	start = time.Now()
	if err := db.buildGraph(dataset); err != nil {
		return errors.Wrap(err, "Could not build graph")
	}
	log.Infof("Built graph in %s", time.Since(start))

	for pfx, uri := range db.namespaces {
		fmt.Printf("%s => %s\n", pfx, uri)
	}

	return nil
}

// returns the uint32 hash of the given URI (this is adjusted for uniqueness)
func (db *DB) GetHash(entity turtle.URI) ([4]byte, error) {
	var hash [4]byte
	val, err := db.entityDB.Get(entity.Bytes(), nil)
	if err != nil {
		return emptyHash, err
	}
	copy(hash[:], val)
	if hash == emptyHash {
		return emptyHash, errors.New("Got bad hash")
	}
	return hash, nil
}

func (db *DB) MustGetHash(entity turtle.URI) [4]byte {
	val, err := db.GetHash(entity)
	if err != nil {
		panic(err)
	}
	return val
}

func (db *DB) GetURI(hash [4]byte) (turtle.URI, error) {
	val, err := db.pkDB.Get(hash[:], nil)
	if err != nil {
		return turtle.URI{}, err
	}
	return turtle.ParseURI(string(val)), nil
}

func (db *DB) MustGetURI(hash [4]byte) turtle.URI {
	val, err := db.pkDB.Get(hash[:], nil)
	if err != nil {
		panic(err)
	}
	return turtle.ParseURI(string(val))
}

func (db *DB) GetEntity(uri turtle.URI) (*Entity, error) {
	var entity = NewEntity()
	hash, err := db.GetHash(uri)
	if err != nil {
		return nil, err
	}
	bytes, err := db.graphDB.Get(hash[:], nil)
	if err != nil {
		return nil, err
	}
	_, err = entity.UnmarshalMsg(bytes)
	if err != nil {
		return nil, err
	}
	return entity, nil
}

func (db *DB) GetEntityFromHash(hash [4]byte) (*Entity, error) {
	bytes, err := db.graphDB.Get(hash[:], nil)
	if err != nil {
		return nil, err
	}
	ent := NewEntity()
	_, err = ent.UnmarshalMsg(bytes)
	return ent, err
}

func (db *DB) MustGetEntityFromHash(hash [4]byte) *Entity {
	e, err := db.GetEntityFromHash(hash)
	if err != nil {
		panic(err)
	}
	return e
}

func (db *DB) GetEntityTx(graphtx *leveldb.Transaction, uri turtle.URI) (*Entity, error) {
	var entity = NewEntity()
	hash, err := db.GetHash(uri)
	if err != nil {
		return nil, err
	}
	bytes, err := graphtx.Get(hash[:], nil)
	if err != nil {
		return nil, err
	}
	_, err = entity.UnmarshalMsg(bytes)
	if err != nil {
		return nil, err
	}
	return entity, nil
}

func (db *DB) GetEntityFromHashTx(graphtx *leveldb.Transaction, hash [4]byte) (*Entity, error) {
	bytes, err := graphtx.Get(hash[:], nil)
	if err != nil {
		return nil, err
	}
	ent := NewEntity()
	_, err = ent.UnmarshalMsg(bytes)
	return ent, err
}
<commit_msg>add all relationships even if they do not appear in building<commit_after>package db

import (
	"encoding/binary"
	"fmt"
	"os"
	"strings"
	"time"

	turtle "github.com/gtfierro/hod/goraptor"
	"github.com/op/go-logging"
	"github.com/pkg/errors"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/zhangxinngang/murmur"
)

// logger
var log *logging.Logger
var emptyHash = [4]byte{0, 0, 0, 0}

func init() {
	log = logging.MustGetLogger("hod")
	var format = "%{color}%{level} %{shortfile} %{time:Jan 02 15:04:05} %{color:reset} ▶ %{message}"
	var logBackend = logging.NewLogBackend(os.Stderr, "", 0)
	logBackendLeveled := logging.AddModuleLevel(logBackend)
	logging.SetBackend(logBackendLeveled)
	logging.SetFormatter(logging.MustStringFormatter(format))
}

type DB struct {
	// store []byte(entity URI) => primary key
	entityDB *leveldb.DB
	// store primary key => [](entity URI)
	pkDB *leveldb.DB
	// predicate index: stores "children" of predicates
	predDB    *leveldb.DB
	predIndex map[turtle.URI]*PredicateEntity
	// graph structure
	graphDB *leveldb.DB
	// store relationships and their inverses
	relationships map[turtle.URI]turtle.URI
	// store the namespace prefixes as strings
	namespaces map[string]string
}

func NewDB(path string) (*DB, error) {
	path = strings.TrimSuffix(path, "/")

	// set up entity, pk databases
	entityDBPath := path + "/db-entities"
	entityDB, err := leveldb.OpenFile(entityDBPath, nil)
	if err != nil {
		return nil, errors.Wrapf(err, "Could not open entityDB file %s", entityDBPath)
	}

	pkDBPath := path + "/db-pk"
	pkDB, err := leveldb.OpenFile(pkDBPath, nil)
	if err != nil {
		return nil, errors.Wrapf(err, "Could not open pkDB file %s", pkDBPath)
	}

	graphDBPath := path + "/db-graph"
	graphDB, err := leveldb.OpenFile(graphDBPath, nil)
	if err != nil {
		return nil, errors.Wrapf(err, "Could not open graphDB file %s", graphDBPath)
	}
	predDBPath := path + "/db-pred"
	predDB, err := leveldb.OpenFile(predDBPath, nil)
	if err != nil {
		return nil, errors.Wrapf(err, "Could not open predDB file %s", predDBPath)
	}

	db := &DB{
		entityDB:      entityDB,
		pkDB:          pkDB,
		graphDB:       graphDB,
		predDB:        predDB,
		predIndex:     make(map[turtle.URI]*PredicateEntity),
		relationships: make(map[turtle.URI]turtle.URI),
	}

	return db, nil
}

// hashes the given URI into the byte array
func (db *DB) hashURI(u turtle.URI, dest []byte, salt uint32) {
	var hash uint32
	if len(dest) < 4 {
		dest = make([]byte, 4)
	}
	if salt > 0 {
		saltbytes := make([]byte, 4)
		binary.LittleEndian.PutUint32(saltbytes, salt)
		hash = murmur.Murmur3(append(u.Bytes(), saltbytes...))
	} else {
		hash = murmur.Murmur3(u.Bytes())
	}
	binary.LittleEndian.PutUint32(dest, hash)
}

// for each part of the triple (subject, predicate, object), we check if its already in the entity database.
// If it is, we can skip it. If not, we generate a murmur3 hash for the entity, and then
// 0. check if we've already inserted the entity (skip if we already have)
// 1. check if the hash is unique (check membership in pk db) - if it isn't then we add a salt and check again
// 2. insert hash => []byte(entity) into pk db
// 3. insert []byte(entity) => hash into entity db
func (db *DB) insertEntity(entity turtle.URI, hashdest []byte, enttx, pktx *leveldb.Transaction) error {
	// check if we've inserted Subject already
	if exists, err := enttx.Has(entity.Bytes(), nil); err == nil && exists {
		// populate hash anyway
		hash, err := enttx.Get(entity.Bytes(), nil)
		copy(hashdest, hash[:])
		return err
	} else if err != nil {
		return errors.Wrapf(err, "Error checking db membership for %s", entity.String())
	}
	// generate the hash
	var salt = uint32(0)
	db.hashURI(entity, hashdest, salt)
	for {
		if exists, err := pktx.Has(hashdest, nil); err == nil && exists {
			log.Warning("hash exists")
			salt += 1
			db.hashURI(entity, hashdest, salt)
		} else if err != nil {
			return errors.Wrapf(err, "Error checking db membership for %v", hashdest)
		} else {
			break
		}
	}

	// insert the hash into the entity and prefix dbs
	if err := enttx.Put(entity.Bytes(), hashdest, nil); err != nil {
		return errors.Wrapf(err, "Error inserting entity %s", entity.String())
	}
	if err := pktx.Put(hashdest, entity.Bytes(), nil); err != nil {
		return errors.Wrapf(err, "Error inserting pk %s", hashdest)
	}
	return nil
}

func (db *DB) loadPredicateEntity(predicate turtle.URI, _predicateHash, _subjectHash, _objectHash []byte, predtx *leveldb.Transaction) error {
	var (
		pred          *PredicateEntity
		found         bool
		predicateHash [4]byte
		subjectHash   [4]byte
		objectHash    [4]byte
	)
	copy(predicateHash[:], _predicateHash)
	copy(subjectHash[:], _subjectHash)
	copy(objectHash[:], _objectHash)

	//log.Debug(predicate)
	if pred, found = db.predIndex[predicate]; !found {
		pred = NewPredicateEntity()
		pred.PK = predicateHash
	}

	pred.AddSubjectObject(subjectHash, objectHash)
	db.predIndex[predicate] = pred
	//TODO: save predicate index

	return nil
}

func (db *DB) LoadRelationships(dataset turtle.DataSet) error {
	// iterate through dataset, and pull out all that have a "rdf:type" of "owl:ObjectProperty"
	// then we want to find the mapping that has "owl:inverseOf"
	var relationships = make(map[turtle.URI]struct{})

	rdf_namespace, found := dataset.Namespaces["rdf"]
	if !found {
		return errors.New("Relationships has no rdf namespace")
	}
	owl_namespace, found := dataset.Namespaces["owl"]
	if !found {
		return errors.New("Relationships has no owl namespace")
	}

	for _, triple := range dataset.Triples {
		if triple.Predicate.Namespace == rdf_namespace &&
			triple.Predicate.Value == "type" &&
			triple.Object.Namespace == owl_namespace &&
			triple.Object.Value == "ObjectProperty" {
			relationships[triple.Subject] = struct{}{}
		}
	}

	for _, triple := range dataset.Triples {
		if triple.Predicate.Namespace == owl_namespace && triple.Predicate.Value == "inverseOf" {
			// check that the subject/object of the inverseOf relationships are both actually relationships
			if _, found := relationships[triple.Subject]; !found {
				continue
			}
			if _, found := relationships[triple.Object]; !found {
				continue
			}
			db.relationships[triple.Subject] = triple.Object
			db.relationships[triple.Object] = triple.Subject
		}
	}

	return nil
}

func (db *DB) LoadDataset(dataset turtle.DataSet) error {
	start := time.Now()
	db.namespaces = dataset.Namespaces
	// start transactions
	enttx, err := db.entityDB.OpenTransaction()
	if err != nil {
		return errors.Wrap(err, "Could not open transaction on entity dataset")
	}
	pktx, err := db.pkDB.OpenTransaction()
	if err != nil {
		return errors.Wrap(err, "Could not open transaction on pk dataset")
	}
	predtx, err := db.predDB.OpenTransaction()
	if err != nil {
		return errors.Wrap(err, "Could not open transaction on pred dataset")
	}
	// load triples and primary keys
	var (
		subjectHash   = make([]byte, 4)
		predicateHash = make([]byte, 4)
		objectHash    = make([]byte, 4)
	)
	for _, triple := range dataset.Triples {
		if err := db.insertEntity(triple.Subject, subjectHash, enttx, pktx); err != nil {
			return err
		}
		if err := db.insertEntity(triple.Predicate, predicateHash, enttx, pktx); err != nil {
			return err
		}
		if err := db.insertEntity(triple.Object, objectHash, enttx, pktx); err != nil {
			return err
		}
		if err := db.loadPredicateEntity(triple.Predicate, predicateHash, subjectHash, objectHash, predtx); err != nil {
			return err
		}
	}

	for pred, _ := range db.relationships {
		log.Error(pred)
		if err := db.insertEntity(pred, predicateHash, enttx, pktx); err != nil {
			return err
		}
	}

	// finish those transactions
	if err := enttx.Commit(); err != nil {
		return errors.Wrap(err, "Could not commit transaction")
	}
	if err := pktx.Commit(); err != nil {
		return errors.Wrap(err, "Could not commit transaction")
	}
	if err := predtx.Commit(); err != nil {
		return errors.Wrap(err, "Could not commit transaction")
	}
	log.Infof("Built lookup tables in %s", time.Since(start))

	start = time.Now()
	if err := db.buildGraph(dataset); err != nil {
		return errors.Wrap(err, "Could not build graph")
	}
	log.Infof("Built graph in %s", time.Since(start))

	for pfx, uri := range db.namespaces {
		fmt.Printf("%s => %s\n", pfx, uri)
	}
	fmt.Println("Relationships")
	for rel, invrel := range db.relationships {
		fmt.Printf("%s => %s\n", rel.String(), invrel.String())
	}

	return nil
}

// returns the uint32 hash of the given URI (this is adjusted for uniqueness)
func (db *DB) GetHash(entity turtle.URI) ([4]byte, error) {
	var hash [4]byte
	val, err := db.entityDB.Get(entity.Bytes(), nil)
	if err != nil {
		return emptyHash, err
	}
	copy(hash[:], val)
	if hash == emptyHash {
		return emptyHash, errors.New("Got bad hash")
	}
	return hash, nil
}

func (db *DB) MustGetHash(entity turtle.URI) [4]byte {
	val, err := db.GetHash(entity)
	if err != nil {
		panic(err)
	}
	return val
}

func (db *DB) GetURI(hash [4]byte) (turtle.URI, error) {
	val, err := db.pkDB.Get(hash[:], nil)
	if err != nil {
		return turtle.URI{}, err
	}
	return turtle.ParseURI(string(val)), nil
}

func (db *DB) MustGetURI(hash [4]byte) turtle.URI {
	val, err := db.pkDB.Get(hash[:], nil)
	if err != nil {
		panic(err)
	}
	return turtle.ParseURI(string(val))
}

func (db *DB) GetEntity(uri turtle.URI) (*Entity, error) {
	var entity = NewEntity()
	hash, err := db.GetHash(uri)
	if err != nil {
		return nil, err
	}
	bytes, err := db.graphDB.Get(hash[:], nil)
	if err != nil {
		return nil, err
	}
	_, err = entity.UnmarshalMsg(bytes)
	if err != nil {
		return nil, err
	}
	return entity, nil
}

func (db *DB) GetEntityFromHash(hash [4]byte) (*Entity, error) {
	bytes, err := db.graphDB.Get(hash[:], nil)
	if err != nil {
		return nil, err
	}
	ent := NewEntity()
	_, err = ent.UnmarshalMsg(bytes)
	return ent, err
}

func (db *DB) MustGetEntityFromHash(hash [4]byte) *Entity {
	e, err := db.GetEntityFromHash(hash)
	if err != nil {
		panic(err)
	}
	return e
}

func (db *DB) GetEntityTx(graphtx *leveldb.Transaction, uri turtle.URI) (*Entity, error) {
	var entity = NewEntity()
	hash, err := db.GetHash(uri)
	if err != nil {
		return nil, err
	}
	bytes, err := graphtx.Get(hash[:], nil)
	if err != nil {
		return nil, err
	}
	_, err = entity.UnmarshalMsg(bytes)
	if err != nil {
		return nil, err
	}
	return entity, nil
}

func (db *DB) GetEntityFromHashTx(graphtx *leveldb.Transaction, hash [4]byte) (*Entity, error) {
	bytes, err := graphtx.Get(hash[:], nil)
	if err != nil {
		return nil, err
	}
	ent := NewEntity()
	_, err = ent.UnmarshalMsg(bytes)
	return ent, err
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"syscall"
	"time"
)

type DebugInfo struct {
	Timestamp          time.Time
	Hostname           string
	Addresses          []string
	InternetConnection bool
	Applications       []DebugApplication
}

type DebugApplication struct {
	Name       string
	DeployFile string
	LogFile    string
	PidFile    int64
	IsRunning  bool
}

func main() {
	port := parseFlags()
	http.HandleFunc("/", debugResponse)

	log.Printf("Debug server running on http://127.0.0.1:%d", port)
	http.ListenAndServe(fmt.Sprintf("127.0.0.1:%d", port), nil)
}

func debugResponse(w http.ResponseWriter, req *http.Request) {
	d := fetchDebugInfo()

	jsonReponse, err := json.Marshal(d)
	failOnError(err, "Cannot marshal JSON")

	w.Header().Set("Content-Type", "application/json")
	w.Write(jsonReponse)
}

func fetchDebugInfo() DebugInfo {
	d := DebugInfo{Timestamp: time.Now()}

	// fetch ip address(es)
	d.Hostname, d.Addresses = ipAddresses()
	// test for internet connection
	d.InternetConnection = checkConnection()
	// log active apps
	d.Applications = checkApps()

	return d
}

func ipAddresses() (hostname string, addrs []string) {
	hostname, err := os.Hostname()
	failOnError(err, "Could not determine hostname")

	addrs, err = net.LookupHost(hostname)
	failOnError(err, "Could not determine IP")
	return
}

func checkConnection() bool {
	// match the request to a known output
	success := "<TITLE>Success</TITLE>" +
		"Success"

	res, err := http.Get("http://www.apple.com/library/test/success.html")

	if err != nil {
		return false
	}

	body, err := ioutil.ReadAll(res.Body)
	defer res.Body.Close()

	if err != nil {
		return false
	}

	return string(body) == success
}

func checkApps() (apps []DebugApplication) {
	appNames := []string{
		"buttons", "cease", "example", "magic",
		"server", "updater", "debug",
	}

	for _, appName := range appNames {
		apps = append(apps, checkApp(appName))
	}

	return
}

func checkApp(appName string) (app DebugApplication) {
	app.Name = appName
	app.DeployFile = fetchDeployFile(appName)
	app.LogFile = fetchLogFile(appName)
	app.PidFile = fetchPidFile(appName)
	app.IsRunning = checkProcess(app.PidFile)
	return
}

func fetchDeployFile(appName string) (output string) {
	pathTemplate := "/opt/radiodan/apps/%s/current/.deploy"
	path := fmt.Sprintf(pathTemplate, appName)

	file, err := ioutil.ReadFile(path)

	if err != nil {
		log.Println("[!] Could not open file", path)
		return
	}

	output = string(file)

	return
}

func fetchLogFile(appName string) (output string) {
	path := fmt.Sprintf("/var/log/radiodan-%s.log", appName)
	_, err := os.Stat(path)

	if err != nil {
		log.Printf("[!] Could not open file %s", path)
		return
	}

	cmd := "tail -n 100 " + path
	stdout, err := exec.Command(cmd).Output()

	if err != nil {
		log.Printf("[!] Command failed: %s", err)
	} else {
		output = string(stdout)
	}

	return
}

func fetchPidFile(appName string) (output int64) {
	pathTemplate := "/var/run/radiodan/radiodan-%s.pid"
	path := fmt.Sprintf(pathTemplate, appName)
	file, err := ioutil.ReadFile(path)

	if err != nil {
		log.Println("[!] Could not open file", path)
		return
	}

	pidString := strings.Trim(string(file), "\n")
	output, err = strconv.ParseInt(pidString, 10, 0)

	if err != nil {
		log.Println("[!] Could not parse pid as integer", pidString)
	}

	return
}

func checkProcess(pid int64) (running bool) {
	running = false

	// this isn't the pid we're looking for
	if pid == 0 {
		return
	}

	// see if that process exists
	process, err := os.FindProcess(int(pid))
	if err != nil {
		return
	}

	// see if the process responds to sig 0
	err = process.Signal(syscall.Signal(0))
	running = (err == nil)

	return
}

func parseFlags() (port int) {
	flag.IntVar(&port, "port", 8080, "Port for server")
	flag.Parse()
	return
}

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}
<commit_msg>Add full path to tail<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"syscall"
	"time"
)

type DebugInfo struct {
	Timestamp          time.Time
	Hostname           string
	Addresses          []string
	InternetConnection bool
	Applications       []DebugApplication
}

type DebugApplication struct {
	Name       string
	DeployFile string
	LogFile    string
	PidFile    int64
	IsRunning  bool
}

func main() {
	port := parseFlags()
	http.HandleFunc("/", debugResponse)

	log.Printf("Debug server running on http://127.0.0.1:%d", port)
	http.ListenAndServe(fmt.Sprintf("127.0.0.1:%d", port), nil)
}

func debugResponse(w http.ResponseWriter, req *http.Request) {
	d := fetchDebugInfo()

	jsonReponse, err := json.Marshal(d)
	failOnError(err, "Cannot marshal JSON")

	w.Header().Set("Content-Type", "application/json")
	w.Write(jsonReponse)
}

func fetchDebugInfo() DebugInfo {
	d := DebugInfo{Timestamp: time.Now()}

	// fetch ip address(es)
	d.Hostname, d.Addresses = ipAddresses()
	// test for internet connection
	d.InternetConnection = checkConnection()
	// log active apps
	d.Applications = checkApps()

	return d
}

func ipAddresses() (hostname string, addrs []string) {
	hostname, err := os.Hostname()
	failOnError(err, "Could not determine hostname")

	addrs, err = net.LookupHost(hostname)
	failOnError(err, "Could not determine IP")
	return
}

func checkConnection() bool {
	// match the request to a known output
	success := "<TITLE>Success</TITLE>" +
		"Success"

	res, err := http.Get("http://www.apple.com/library/test/success.html")

	if err != nil {
		return false
	}

	body, err := ioutil.ReadAll(res.Body)
	defer res.Body.Close()

	if err != nil {
		return false
	}

	return string(body) == success
}

func checkApps() (apps []DebugApplication) {
	appNames := []string{
		"buttons", "cease", "example", "magic",
		"server", "updater", "debug",
	}

	for _, appName := range appNames {
		apps = append(apps, checkApp(appName))
	}

	return
}

func checkApp(appName string) (app DebugApplication) {
	app.Name = appName
	app.DeployFile = fetchDeployFile(appName)
	app.LogFile = fetchLogFile(appName)
	app.PidFile = fetchPidFile(appName)
	app.IsRunning = checkProcess(app.PidFile)
	return
}

func fetchDeployFile(appName string) (output string) {
	pathTemplate := "/opt/radiodan/apps/%s/current/.deploy"
	path := fmt.Sprintf(pathTemplate, appName)

	file, err := ioutil.ReadFile(path)

	if err != nil {
		log.Println("[!] Could not open file", path)
		return
	}

	output = string(file)

	return
}

func fetchLogFile(appName string) (output string) {
	path := fmt.Sprintf("/var/log/radiodan-%s.log", appName)
	_, err := os.Stat(path)

	if err != nil {
		log.Printf("[!] Could not open file %s", path)
		return
	}

	cmd := "/usr/bin/tail -n 100 " + path
	stdout, err := exec.Command(cmd).Output()

	if err != nil {
		log.Printf("[!] Command failed: %s", err)
	} else {
		output = string(stdout)
	}

	return
}

func fetchPidFile(appName string) (output int64) {
	pathTemplate := "/var/run/radiodan/radiodan-%s.pid"
	path := fmt.Sprintf(pathTemplate, appName)
	file, err := ioutil.ReadFile(path)

	if err != nil {
		log.Println("[!] Could not open file", path)
		return
	}

	pidString := strings.Trim(string(file), "\n")
	output, err = strconv.ParseInt(pidString, 10, 0)

	if err != nil {
		log.Println("[!] Could not parse pid as integer", pidString)
	}

	return
}

func checkProcess(pid int64) (running bool) {
	running = false

	// this isn't the pid we're looking for
	if pid == 0 {
		return
	}

	// see if that process exists
	process, err := os.FindProcess(int(pid))
	if err != nil {
		return
	}

	// see if the process responds to sig 0
	err = process.Signal(syscall.Signal(0))
	running = (err == nil)

	return
}

func parseFlags() (port int) {
	flag.IntVar(&port, "port", 8080, "Port for server")
	flag.Parse()
	return
}

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"flag"
	"io"
	"io/ioutil"
	"log"
	"os"
	"sync"
)

var fEnableDebug = flag.Bool(
	"fuse.debug",
	false,
	"Write FUSE debugging messages to stderr.")

var gDebugLogger *log.Logger
var gDebugLoggerOnce sync.Once

func initDebugDebugLogger() {
	if !flag.Parsed() {
		panic("initDebugDebugLogger called before flags available.")
	}

	var writer io.Writer = ioutil.Discard
	if *fEnableDebug {
		writer = os.Stderr
	}

	const flags = log.Ldate | log.Ltime | log.Lmicroseconds
	gDebugLogger = log.New(writer, "", flags)
}

func getDebugLogger() *log.Logger {
	gDebugLoggerOnce.Do(initDebugDebugLogger)
	return gDebugLogger
}
<commit_msg>Killed debug.go.<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package l3

import (
	"fmt"
	"net"
	"strconv"
	"strings"
)

const (
	/* ARP */

	// StaticArpKeyPrefix is a prefix used in ETCD to store configuration for Linux static ARPs.
	StaticArpKeyPrefix = "linux/config/v2/arp/"

	// staticArpKeyTemplate is a template for key representing Linux ARP entry configuration.
	staticArpKeyTemplate = StaticArpKeyPrefix + "{if}/{ip}"

	/* Route Config */

	// StaticRouteKeyPrefix is a prefix used in ETCD to store configuration for Linux static routes.
	StaticRouteKeyPrefix = "linux/config/v2/route/"

	// staticRouteKeySuffix is a suffix common to all keys representing routes.
	staticRouteKeySuffix = "{dest-net}/{dest-mask}/{out-intf}"

	// staticRouteKeyTemplate is a template for key representing Linux Route configuration.
	staticRouteKeyTemplate = StaticRouteKeyPrefix + staticRouteKeySuffix

	/* Link-local route (derived) */

	// StaticLinkLocalRouteKeyPrefix is a prefix for keys derived from link-local routes.
	StaticLinkLocalRouteKeyPrefix = "linux/link-local-route/"

	// staticLinkLocalRouteKeyTemplate is a template for key derived from link-local route.
	staticLinkLocalRouteKeyTemplate = StaticLinkLocalRouteKeyPrefix + staticRouteKeySuffix
)

/* ARP */

// StaticArpKey returns the key used in ETCD to store configuration of a particular Linux ARP entry.
func StaticArpKey(iface, ipAddr string) string {
	key := strings.Replace(staticArpKeyTemplate, "{if}", iface, 1)
	key = strings.Replace(key, "{ip}", ipAddr, 1)
	return key
}

// ParseStaticArpKey parses ARP entry from a key.
func ParseStaticArpKey(key string) (iface string, ipAddr net.IP, err error) {
	errPrefix := "invalid Linux ARP key: "
	if strings.HasPrefix(key, StaticArpKeyPrefix) {
		arpSuffix := strings.TrimPrefix(key, StaticArpKeyPrefix)
		arpComps := strings.Split(arpSuffix, "/")
		if len(arpComps) != 2 {
			return "", nil, fmt.Errorf(errPrefix + "invalid suffix")
		}
		ipAddr = net.ParseIP(arpComps[1])
		if ipAddr == nil {
			return "", nil, fmt.Errorf(errPrefix + "invalid IP address")
		}
		iface = arpComps[0]
		return
	}
	return "", nil, fmt.Errorf(errPrefix + "invalid prefix")
}

/* Route Config */

// StaticRouteKey returns the key used in ETCD to store configuration of a particular Linux route.
func StaticRouteKey(dstAddr, outgoingInterface string) string {
	return staticRouteKeyFromTemplate(staticRouteKeyTemplate, dstAddr, outgoingInterface)
}

// ParseStaticRouteKey parses Linux route attributes from a key.
func ParseStaticRouteKey(key string) (dstNetAddr *net.IPNet, outgoingInterface string, err error) {
	return parseStaticRouteFromKeySuffix(key, StaticRouteKeyPrefix, "invalid Linux Route key: ")
}

/* Link-local Route (derived) */

// StaticLinkLocalRouteKey returns a derived key used to represent link-local route.
func StaticLinkLocalRouteKey(dstAddr, outgoingInterface string) string {
	return staticRouteKeyFromTemplate(staticLinkLocalRouteKeyTemplate, dstAddr, outgoingInterface)
}

// ParseStaticLinkLocalRouteKey parses route attributes from a key derived from link-local route.
func ParseStaticLinkLocalRouteKey(key string) (dstNetAddr *net.IPNet, outgoingInterface string, err error) {
	return parseStaticRouteFromKeySuffix(key, StaticLinkLocalRouteKeyPrefix, "invalid Linux link-local Route key: ")
}

/* Route helpers */

// staticRouteKeyFromTemplate fills key template with route attributes.
func staticRouteKeyFromTemplate(template, dstAddr, outgoingInterface string) string {
	_, dstNet, _ := net.ParseCIDR(dstAddr)
	dstNetAddr := dstNet.IP.String()
	dstNetMask, _ := dstNet.Mask.Size()
	key := strings.Replace(template, "{dest-net}", dstNetAddr, 1)
	key = strings.Replace(key, "{dest-mask}", strconv.Itoa(dstNetMask), 1)
	key = strings.Replace(key, "{out-intf}", outgoingInterface, 1)
	return key
}

// parseStaticRouteFromKeySuffix parses destination network and outgoing interface from a route key suffix.
func parseStaticRouteFromKeySuffix(key, prefix, errPrefix string) (dstNetAddr *net.IPNet, outgoingInterface string, err error) {
	if strings.HasPrefix(key, prefix) {
		routeSuffix := strings.TrimPrefix(key, prefix)
		routeComps := strings.Split(routeSuffix, "/")
		if len(routeComps) != 3 {
			return nil, "", fmt.Errorf(errPrefix + "invalid suffix")
		}
		_, dstNetAddr, err = net.ParseCIDR(routeComps[0] + "/" + routeComps[1])
		if err != nil {
			return nil, "", fmt.Errorf(errPrefix + "invalid destination address")
		}
		outgoingInterface = routeComps[2]
		return
	}
	return nil, "", fmt.Errorf(errPrefix + "invalid prefix")
}
<commit_msg>Fix StaticRouteKey argument name.<commit_after>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package l3

import (
	"fmt"
	"net"
	"strconv"
	"strings"
)

const (
	/* ARP */

	// StaticArpKeyPrefix is a prefix used in ETCD to store configuration for Linux static ARPs.
	StaticArpKeyPrefix = "linux/config/v2/arp/"

	// staticArpKeyTemplate is a template for key representing Linux ARP entry configuration.
	staticArpKeyTemplate = StaticArpKeyPrefix + "{if}/{ip}"

	/* Route Config */

	// StaticRouteKeyPrefix is a prefix used in ETCD to store configuration for Linux static routes.
	StaticRouteKeyPrefix = "linux/config/v2/route/"

	// staticRouteKeySuffix is a suffix common to all keys representing routes.
	staticRouteKeySuffix = "{dest-net}/{dest-mask}/{out-intf}"

	// staticRouteKeyTemplate is a template for key representing Linux Route configuration.
	staticRouteKeyTemplate = StaticRouteKeyPrefix + staticRouteKeySuffix

	/* Link-local route (derived) */

	// StaticLinkLocalRouteKeyPrefix is a prefix for keys derived from link-local routes.
	StaticLinkLocalRouteKeyPrefix = "linux/link-local-route/"

	// staticLinkLocalRouteKeyTemplate is a template for key derived from link-local route.
	staticLinkLocalRouteKeyTemplate = StaticLinkLocalRouteKeyPrefix + staticRouteKeySuffix
)

/* ARP */

// StaticArpKey returns the key used in ETCD to store configuration of a particular Linux ARP entry.
func StaticArpKey(iface, ipAddr string) string {
	key := strings.Replace(staticArpKeyTemplate, "{if}", iface, 1)
	key = strings.Replace(key, "{ip}", ipAddr, 1)
	return key
}

// ParseStaticArpKey parses ARP entry from a key.
func ParseStaticArpKey(key string) (iface string, ipAddr net.IP, err error) {
	errPrefix := "invalid Linux ARP key: "
	if strings.HasPrefix(key, StaticArpKeyPrefix) {
		arpSuffix := strings.TrimPrefix(key, StaticArpKeyPrefix)
		arpComps := strings.Split(arpSuffix, "/")
		if len(arpComps) != 2 {
			return "", nil, fmt.Errorf(errPrefix + "invalid suffix")
		}
		ipAddr = net.ParseIP(arpComps[1])
		if ipAddr == nil {
			return "", nil, fmt.Errorf(errPrefix + "invalid IP address")
		}
		iface = arpComps[0]
		return
	}
	return "", nil, fmt.Errorf(errPrefix + "invalid prefix")
}

/* Route Config */

// StaticRouteKey returns the key used in ETCD to store configuration of a particular Linux route.
func StaticRouteKey(dstNetwork, outgoingInterface string) string {
	return staticRouteKeyFromTemplate(staticRouteKeyTemplate, dstNetwork, outgoingInterface)
}

// ParseStaticRouteKey parses Linux route attributes from a key.
func ParseStaticRouteKey(key string) (dstNetAddr *net.IPNet, outgoingInterface string, err error) {
	return parseStaticRouteFromKeySuffix(key, StaticRouteKeyPrefix, "invalid Linux Route key: ")
}

/* Link-local Route (derived) */

// StaticLinkLocalRouteKey returns a derived key used to represent link-local route.
func StaticLinkLocalRouteKey(dstAddr, outgoingInterface string) string {
	return staticRouteKeyFromTemplate(staticLinkLocalRouteKeyTemplate, dstAddr, outgoingInterface)
}

// ParseStaticLinkLocalRouteKey parses route attributes from a key derived from link-local route.
func ParseStaticLinkLocalRouteKey(key string) (dstNetAddr *net.IPNet, outgoingInterface string, err error) {
	return parseStaticRouteFromKeySuffix(key, StaticLinkLocalRouteKeyPrefix, "invalid Linux link-local Route key: ")
}

/* Route helpers */

// staticRouteKeyFromTemplate fills key template with route attributes.
func staticRouteKeyFromTemplate(template, dstAddr, outgoingInterface string) string {
	_, dstNet, _ := net.ParseCIDR(dstAddr)
	dstNetAddr := dstNet.IP.String()
	dstNetMask, _ := dstNet.Mask.Size()
	key := strings.Replace(template, "{dest-net}", dstNetAddr, 1)
	key = strings.Replace(key, "{dest-mask}", strconv.Itoa(dstNetMask), 1)
	key = strings.Replace(key, "{out-intf}", outgoingInterface, 1)
	return key
}

// parseStaticRouteFromKeySuffix parses destination network and outgoing interface from a route key suffix.
func parseStaticRouteFromKeySuffix(key, prefix, errPrefix string) (dstNetAddr *net.IPNet, outgoingInterface string, err error) {
	if strings.HasPrefix(key, prefix) {
		routeSuffix := strings.TrimPrefix(key, prefix)
		routeComps := strings.Split(routeSuffix, "/")
		if len(routeComps) != 3 {
			return nil, "", fmt.Errorf(errPrefix + "invalid suffix")
		}
		_, dstNetAddr, err = net.ParseCIDR(routeComps[0] + "/" + routeComps[1])
		if err != nil {
			return nil, "", fmt.Errorf(errPrefix + "invalid destination address")
		}
		outgoingInterface = routeComps[2]
		return
	}
	return nil, "", fmt.Errorf(errPrefix + "invalid prefix")
}
<|endoftext|>


<commit_before>package worldweather

// `{ "data": { "error": [ {"msg": "Unable to find any matching weather location to the query submitted!" } ] }}`
type ErrorDescription struct {
	Message string `json:"msg"`
}

type CommonData struct {
	Error []*ErrorDescription `json:"error"`
}

func (data *CommonData) HasError() bool {
	return len(data.Error) > 0
}

// https://developer.worldweatheronline.com/api/docs/local-city-town-weather-api.aspx#data_element
type LocalWeatherResponse struct {
	Data *WeatherData `json:"data"`
}

type WeatherData struct {
	CommonData
	Request          []*Request          `json:"request"`
	CurrentCondition []*CurrentCondition `json:"current_condition"`
	Weather          []*Weather          `json:"weather"`
}

type Request struct {
	Type  string
	Query string
}

type CurrentCondition struct {
	ObservationTime    string                `json:"observation_time"`
	Temperature        string                `json:"temp_C"`
	FeelingTemperature string                `json:"FeelsLikeC"`
	WindSpeed          string                `json:"windspeedKmph"`
	WindDirection      string                `json:"winddirDegree"`
	WeatherCode        string                `json:"weatherCode"`
	WeatherIcon        []*WeatherIcon        `json:"weatherIconUrl"`
	Description        []*WeatherDescription `json:"weatherDesc"`
	Precipitation      string                `json:"precpMM"`
	Humidity           string                `json:"humidity"`
	Visibility         string                `json:"visibility"`
	Pressure           string                `json:"pressure"`
	CloudCover         string                `json:"cloudcocver"`
}

type WeatherIcon struct {
	URL string `json:"value"`
}

type WeatherDescription struct {
	Content string `json:"value"`
}

type Weather struct {
	Astronomy []*Astronomy     `json:"astronomy"`
	Date      string           `json:"date"` // 2016-09-04
	MaxTempC  string           `json:"maxTempC"`
	MaxTempF  string           `json:"maxTempF"`
	MinTempC  string           `json:"minTempC"`
	MinTempF  string           `json:"minTempF"`
	UV        string           `json:"uvindex"`
	Hourly    []*HourlyWeather `json:"hourly"`
}

type HourlyWeather struct {
	Time               string                `json:"time"`
	Temperature        string                `json:"tempC"` // not temp_C
	FeelingTemperature string                `json:"FeelsLikeC"`
	WindSpeed          string                `json:"windspeedKmph"`
	WindDirection      string                `json:"winddirDegree"`
	WeatherCode        string                `json:"weatherCode"`
	WeatherIcon        []*WeatherIcon        `json:"weatherIconUrl"`
	Description        []*WeatherDescription `json:"weatherDesc"`
	Precipitation      string                `json:"precpMM"`
	Humidity           string                `json:"humidity"`
	Visibility         string                `json:"visibility"`
	Pressure           string                `json:"pressure"`
	CloudCover         string                `json:"cloudcocver"`
}

// TODO
type Astronomy struct {
	Sunrise  string `json:"sunrise"`
	Sunset   string `json:"sunset"`
	MoonRise string `json:"moonrise"`
	MoonSet  string `json:"moonset"`
}
<commit_msg>Add document comments for weather command's response<commit_after>package worldweather

// ErrorDescription represents error that returned by World Weather API.
// `{ "data": { "error": [ {"msg": "Unable to find any matching weather location to the query submitted!" } ] }}`
type ErrorDescription struct {
	Message string `json:"msg"`
}

// CommonData represents common response fields returned as part of API response.
type CommonData struct {
	Error []*ErrorDescription `json:"error"`
}

// HasError tells if the response contains any error.
func (data *CommonData) HasError() bool {
	return len(data.Error) > 0
}

// LocalWeatherResponse represents local weather information.
// https://developer.worldweatheronline.com/api/docs/local-city-town-weather-api.aspx#data_element
type LocalWeatherResponse struct {
	Data *WeatherData `json:"data"`
}

// WeatherData represents set of weather information.
type WeatherData struct {
	CommonData
	Request          []*Request          `json:"request"`
	CurrentCondition []*CurrentCondition `json:"current_condition"`
	Weather          []*Weather          `json:"weather"`
}

// Request represents clients request.
type Request struct {
	Type  string
	Query string
}

// CurrentCondition represents current weather condition returned by API.
type CurrentCondition struct {
	ObservationTime    string                `json:"observation_time"`
	Temperature        string                `json:"temp_C"`
	FeelingTemperature string                `json:"FeelsLikeC"`
	WindSpeed          string                `json:"windspeedKmph"`
	WindDirection      string                `json:"winddirDegree"`
	WeatherCode        string                `json:"weatherCode"`
	WeatherIcon        []*WeatherIcon        `json:"weatherIconUrl"`
	Description        []*WeatherDescription `json:"weatherDesc"`
	Precipitation      string                `json:"precpMM"`
	Humidity           string                `json:"humidity"`
	Visibility         string                `json:"visibility"`
	Pressure           string                `json:"pressure"`
	CloudCover         string                `json:"cloudcocver"`
}

// WeatherIcon is an icon url that represents corresponding weather.
type WeatherIcon struct {
	URL string `json:"value"`
}

// WeatherDescription represents weather description.
type WeatherDescription struct {
	Content string `json:"value"`
}

// Weather represents set of weather information.
type Weather struct {
	Astronomy []*Astronomy     `json:"astronomy"`
	Date      string           `json:"date"` // 2016-09-04
	MaxTempC  string           `json:"maxTempC"`
	MaxTempF  string           `json:"maxTempF"`
	MinTempC  string           `json:"minTempC"`
	MinTempF  string           `json:"minTempF"`
	UV        string           `json:"uvindex"`
	Hourly    []*HourlyWeather `json:"hourly"`
}

// HourlyWeather represents hourly weather information.
type HourlyWeather struct {
	Time               string                `json:"time"`
	Temperature        string                `json:"tempC"` // not temp_C
	FeelingTemperature string                `json:"FeelsLikeC"`
	WindSpeed          string                `json:"windspeedKmph"`
	WindDirection      string                `json:"winddirDegree"`
	WeatherCode        string                `json:"weatherCode"`
	WeatherIcon        []*WeatherIcon        `json:"weatherIconUrl"`
	Description        []*WeatherDescription `json:"weatherDesc"`
	Precipitation      string                `json:"precpMM"`
	Humidity           string                `json:"humidity"`
	Visibility         string                `json:"visibility"`
	Pressure           string                `json:"pressure"`
	CloudCover         string                `json:"cloudcocver"`
}

// Astronomy represents astronomical information.
type Astronomy struct {
	Sunrise  string `json:"sunrise"`
	Sunset   string `json:"sunset"`
	MoonRise string `json:"moonrise"`
	MoonSet  string `json:"moonset"`
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package api

import (
	"github.com/juju/juju/process"
	"gopkg.in/juju/charm.v5"
)

// API2Proc converts an API Process info struct into a process.Info struct.
func API2Proc(p Process) process.Info {
	return process.Info{
		Process: charm.Process{
			Name:        p.Definition.Name,
			Description: p.Definition.Description,
			Type:        p.Definition.Type,
			TypeOptions: p.Definition.TypeOptions,
			Command:     p.Definition.Command,
			Image:       p.Definition.Image,
			Ports:       API2charmPorts(p.Definition.Ports),
			Volumes:     API2charmVolumes(p.Definition.Volumes),
			EnvVars:     p.Definition.EnvVars,
		},
		Details: process.Details{
			ID:     p.Details.ID,
			Status: APIStatus2Status(p.Details.Status),
		},
	}
}

// Proc2api converts a process.Info struct into an api.Process struct.
func Proc2api(p process.Info) Process {
	return Process{
		Definition: ProcessDefinition{
			Name:        p.Process.Name,
			Description: p.Process.Description,
			Type:        p.Process.Type,
			TypeOptions: p.Process.TypeOptions,
			Command:     p.Process.Command,
			Image:       p.Process.Image,
			Ports:       Charm2apiPorts(p.Process.Ports),
			Volumes:     Charm2apiVolumes(p.Process.Volumes),
			EnvVars:     p.Process.EnvVars,
		},
		Details: ProcessDetails{
			ID:     p.Details.ID,
			Status: Status2apiStatus(p.Details.Status),
		},
	}
}

// APIStatus2Status converts an API ProcessStatus struct into a
// process.Status struct.
func APIStatus2Status(status ProcessStatus) process.PluginStatus {
	return process.PluginStatus{
		Label: status.Label,
	}
}

// Status2APIStatus converts a process.Status struct into an
// API ProcessStatus struct.
func Status2apiStatus(status process.PluginStatus) ProcessStatus {
	return ProcessStatus{
		Label: status.Label,
	}
}

// API2charmPorts converts a slice of api.ProcessPorts into a slice of
// charm.ProcessPorts.
func API2charmPorts(ports []ProcessPort) []charm.ProcessPort {
	ret := make([]charm.ProcessPort, len(ports))
	for i, p := range ports {
		ret[i] = charm.ProcessPort{
			Internal: p.Internal,
			External: p.External,
			Endpoint: p.Endpoint,
		}
	}
	return ret
}

// API2charmVolumes converts a slice of api.ProcessVolume into a slice of charm.ProcessVolume.
func API2charmVolumes(vols []ProcessVolume) []charm.ProcessVolume {
	ret := make([]charm.ProcessVolume, len(vols))
	for i, v := range vols {
		ret[i] = charm.ProcessVolume{
			ExternalMount: v.ExternalMount,
			InternalMount: v.InternalMount,
			Mode:          v.Mode,
			Name:          v.Name,
		}
	}
	return ret
}

// Charm2apiPorts converts a slice of charm.ProcessPorts into a slice of api.ProcessPort.
func Charm2apiPorts(ports []charm.ProcessPort) []ProcessPort {
	ret := make([]ProcessPort, len(ports))
	for i, p := range ports {
		ret[i] = ProcessPort{
			Internal: p.Internal,
			External: p.External,
			Endpoint: p.Endpoint,
		}
	}
	return ret
}

// Charm2apiVolumes converts a slice of charm.ProcessVolume into a slice of api.ProcessVolume.
func Charm2apiVolumes(vols []charm.ProcessVolume) []ProcessVolume {
	ret := make([]ProcessVolume, len(vols))
	for i, v := range vols {
		ret[i] = ProcessVolume{
			ExternalMount: v.ExternalMount,
			InternalMount: v.InternalMount,
			Mode:          v.Mode,
			Name:          v.Name,
		}
	}
	return ret
}
<commit_msg>Factor out API2Definition and Definition2api helpers.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package api

import (
	"github.com/juju/juju/process"
	"gopkg.in/juju/charm.v5"
)

// API2Definition converts an API process definition struct into
// a charm.Process struct.
func API2Definition(d ProcessDefinition) charm.Process {
	return charm.Process{
		Name:        d.Name,
		Description: d.Description,
		Type:        d.Type,
		TypeOptions: d.TypeOptions,
		Command:     d.Command,
		Image:       d.Image,
		Ports:       API2charmPorts(d.Ports),
		Volumes:     API2charmVolumes(d.Volumes),
		EnvVars:     d.EnvVars,
	}
}

// Definition2api converts a charm.Process struct into an
// api.ProcessDefinition struct.
func Definition2api(d charm.Process) ProcessDefinition {
	return ProcessDefinition{
		Name:        d.Name,
		Description: d.Description,
		Type:        d.Type,
		TypeOptions: d.TypeOptions,
		Command:     d.Command,
		Image:       d.Image,
		Ports:       Charm2apiPorts(d.Ports),
		Volumes:     Charm2apiVolumes(d.Volumes),
		EnvVars:     d.EnvVars,
	}
}

// API2Proc converts an API Process info struct into a process.Info struct.
func API2Proc(p Process) process.Info {
	return process.Info{
		Process: API2Definition(p.Definition),
		Details: process.Details{
			ID:     p.Details.ID,
			Status: APIStatus2Status(p.Details.Status),
		},
	}
}

// Proc2api converts a process.Info struct into an api.Process struct.
func Proc2api(p process.Info) Process {
	return Process{
		Definition: Definition2api(p.Process),
		Details: ProcessDetails{
			ID:     p.Details.ID,
			Status: Status2apiStatus(p.Details.Status),
		},
	}
}

// APIStatus2Status converts an API ProcessStatus struct into a
// process.Status struct.
func APIStatus2Status(status ProcessStatus) process.PluginStatus {
	return process.PluginStatus{
		Label: status.Label,
	}
}

// Status2APIStatus converts a process.Status struct into an
// API ProcessStatus struct.
func Status2apiStatus(status process.PluginStatus) ProcessStatus {
	return ProcessStatus{
		Label: status.Label,
	}
}

// API2charmPorts converts a slice of api.ProcessPorts into a slice of
// charm.ProcessPorts.
func API2charmPorts(ports []ProcessPort) []charm.ProcessPort {
	ret := make([]charm.ProcessPort, len(ports))
	for i, p := range ports {
		ret[i] = charm.ProcessPort{
			Internal: p.Internal,
			External: p.External,
			Endpoint: p.Endpoint,
		}
	}
	return ret
}

// API2charmVolumes converts a slice of api.ProcessVolume into a slice of charm.ProcessVolume.
func API2charmVolumes(vols []ProcessVolume) []charm.ProcessVolume {
	ret := make([]charm.ProcessVolume, len(vols))
	for i, v := range vols {
		ret[i] = charm.ProcessVolume{
			ExternalMount: v.ExternalMount,
			InternalMount: v.InternalMount,
			Mode:          v.Mode,
			Name:          v.Name,
		}
	}
	return ret
}

// Charm2apiPorts converts a slice of charm.ProcessPorts into a slice of api.ProcessPort.
func Charm2apiPorts(ports []charm.ProcessPort) []ProcessPort {
	ret := make([]ProcessPort, len(ports))
	for i, p := range ports {
		ret[i] = ProcessPort{
			Internal: p.Internal,
			External: p.External,
			Endpoint: p.Endpoint,
		}
	}
	return ret
}

// Charm2apiVolumes converts a slice of charm.ProcessVolume into a slice of api.ProcessVolume.
func Charm2apiVolumes(vols []charm.ProcessVolume) []ProcessVolume {
	ret := make([]ProcessVolume, len(vols))
	for i, v := range vols {
		ret[i] = ProcessVolume{
			ExternalMount: v.ExternalMount,
			InternalMount: v.InternalMount,
			Mode:          v.Mode,
			Name:          v.Name,
		}
	}
	return ret
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package processors

import (
	"net/url"
	"time"

	"github.com/golang/glog"

	kube_config "k8s.io/heapster/common/kubernetes"
	"k8s.io/heapster/metrics/core"
	kube_api "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/cache"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
)

type NamespaceBasedEnricher struct {
	store     cache.Store
	reflector *cache.Reflector
}

func (this *NamespaceBasedEnricher) Name() string {
	return "namespace_based_enricher"
}

func (this *NamespaceBasedEnricher) Process(batch *core.DataBatch) (*core.DataBatch, error) {
	for _, ms := range batch.MetricSets {
		this.addNamespaceInfo(ms)
	}
	return batch, nil
}

// Adds UID to all namespaced elements.
func (this *NamespaceBasedEnricher) addNamespaceInfo(metricSet *core.MetricSet) {
	if metricSetType, found := metricSet.Labels[core.LabelMetricSetType.Key]; found &&
		(metricSetType == core.MetricSetTypePodContainer ||
			metricSetType == core.MetricSetTypePod ||
			metricSetType == core.MetricSetTypeNamespace) {

		if namespaceName, found := metricSet.Labels[core.LabelNamespaceName.Key]; found {
			nsObj, exists, err := this.store.GetByKey(namespaceName)
			if exists && err == nil {
				namespace, ok := nsObj.(*kube_api.Namespace)
				if ok {
					metricSet.Labels[core.LabelPodNamespaceUID.Key] = string(namespace.UID)
				} else {
					glog.Errorf("Wrong namespace store content")
				}
			} else {
				glog.Warningf("Failed to get namespace: exist: %s error: %v", exists, err)
			}
		}
	}
}

func NewNamespaceBasedEnricher(url *url.URL) (*NamespaceBasedEnricher, error) {
	kubeConfig, err := kube_config.GetKubeClientConfig(url)
	if err != nil {
		return nil, err
	}
	kubeClient := kube_client.NewOrDie(kubeConfig)

	// watch nodes
	lw := cache.NewListWatchFromClient(kubeClient, "namespaces", kube_api.NamespaceAll, fields.Everything())
	store := cache.NewStore(cache.MetaNamespaceKeyFunc)
	reflector := cache.NewReflector(lw, &kube_api.Namespace{}, store, time.Hour)
	reflector.Run()

	return &NamespaceBasedEnricher{
		store:     store,
		reflector: reflector,
	}, nil
}
<commit_msg>Fix warning formatting in NamespaceBasedEnricher<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package processors

import (
	"net/url"
	"time"

	"github.com/golang/glog"

	kube_config "k8s.io/heapster/common/kubernetes"
	"k8s.io/heapster/metrics/core"
	kube_api "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/cache"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
)

type NamespaceBasedEnricher struct {
	store     cache.Store
	reflector *cache.Reflector
}

func (this *NamespaceBasedEnricher) Name() string {
	return "namespace_based_enricher"
}

func (this *NamespaceBasedEnricher) Process(batch *core.DataBatch) (*core.DataBatch, error) {
	for _, ms := range batch.MetricSets {
		this.addNamespaceInfo(ms)
	}
	return batch, nil
}

// Adds UID to all namespaced elements.
func (this *NamespaceBasedEnricher) addNamespaceInfo(metricSet *core.MetricSet) {
	if metricSetType, found := metricSet.Labels[core.LabelMetricSetType.Key]; found &&
		(metricSetType == core.MetricSetTypePodContainer ||
			metricSetType == core.MetricSetTypePod ||
			metricSetType == core.MetricSetTypeNamespace) {

		if namespaceName, found := metricSet.Labels[core.LabelNamespaceName.Key]; found {
			nsObj, exists, err := this.store.GetByKey(namespaceName)
			if exists && err == nil {
				namespace, ok := nsObj.(*kube_api.Namespace)
				if ok {
					metricSet.Labels[core.LabelPodNamespaceUID.Key] = string(namespace.UID)
				} else {
					glog.Errorf("Wrong namespace store content")
				}
			} else {
				if err != nil {
					glog.Warningf("Failed to get namespace %s: %v", namespaceName, err)
				} else if !exists {
					glog.Warningf("Namespace doesn't exist: %s", namespaceName)
				}
			}
		}
	}
}

func NewNamespaceBasedEnricher(url *url.URL) (*NamespaceBasedEnricher, error) {
	kubeConfig, err := kube_config.GetKubeClientConfig(url)
	if err != nil {
		return nil, err
	}
	kubeClient := kube_client.NewOrDie(kubeConfig)

	// watch nodes
	lw := cache.NewListWatchFromClient(kubeClient, "namespaces", kube_api.NamespaceAll, fields.Everything())
	store := cache.NewStore(cache.MetaNamespaceKeyFunc)
	reflector := cache.NewReflector(lw, &kube_api.Namespace{}, store, time.Hour)
	reflector.Run()

	return &NamespaceBasedEnricher{
		store:     store,
		reflector: reflector,
	}, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package render

import (
	"bytes"
	"fmt"
	"io"
	"path"
	"sort"
	"strconv"
	"strings"

	"github.com/googlecodelabs/tools/claat/types"
)

// MD renders nodes as markdown for the target env.
func MD(ctx Context, nodes ...types.Node) (string, error) {
	var buf bytes.Buffer
	if err := WriteMD(&buf, ctx.Env, nodes...); err != nil {
		return "", err
	}
	return buf.String(), nil
}

// WriteMD does the same as MD but outputs rendered markup to w.
func WriteMD(w io.Writer, env string, nodes ...types.Node) error {
	mw := mdWriter{w: w, env: env, Prefix: ""}
	return mw.write(nodes...)
}

type mdWriter struct {
	w                  io.Writer // output writer
	env                string    // target environment
	err                error     // error during any writeXxx methods
	lineStart          bool
	isWritingTableCell bool   // used to override lineStart for correct cell formatting
	Prefix             string // prefix for e.g. blockquote content
}

func (mw *mdWriter) writeBytes(b []byte) {
	if mw.err != nil {
		return
	}
	mw.lineStart = len(b) > 0 && b[len(b)-1] == '\n'
	_, mw.err = mw.w.Write(b)
}

func (mw *mdWriter) writeString(s string) {
	if mw.lineStart {
		s = mw.Prefix + s
	}
	mw.writeBytes([]byte(s))
}

func (mw *mdWriter) space() {
	if !mw.lineStart {
		mw.writeString(" ")
	}
}

func (mw *mdWriter) newBlock() {
	if !mw.lineStart {
		mw.writeBytes(newLine)
	}
	mw.writeBytes(newLine)
}

func (mw *mdWriter) matchEnv(v []string) bool {
	if len(v) == 0 || mw.env == "" {
		return true
	}
	i := sort.SearchStrings(v, mw.env)
	return i < len(v) && v[i] == mw.env
}

func (mw *mdWriter) write(nodes ...types.Node) error {
	for _, n := range nodes {
		if !mw.matchEnv(n.Env()) {
			continue
		}
		switch n := n.(type) {
		case *types.TextNode:
			mw.text(n)
		case *types.ImageNode:
			mw.image(n)
		case *types.URLNode:
			mw.url(n)
		case *types.ButtonNode:
			mw.write(n.Content.Nodes...)
		case *types.CodeNode:
			mw.code(n)
		case *types.ListNode:
			mw.list(n)
		case *types.ImportNode:
			if len(n.Content.Nodes) == 0 {
				break
			}
			mw.write(n.Content.Nodes...)
		case *types.ItemsListNode:
			mw.itemsList(n)
		case *types.GridNode:
			mw.table(n)
		case *types.InfoboxNode:
			mw.infobox(n)
		//case *types.SurveyNode:
		//	mw.survey(n)
		case *types.HeaderNode:
			mw.header(n)
		case *types.YouTubeNode:
			mw.youtube(n)
		}
		if mw.err != nil {
			return mw.err
		}
	}
	return nil
}

func (mw *mdWriter) text(n *types.TextNode) {
	tr := strings.TrimLeft(n.Value, " \t\n\r\f\v")
	left := n.Value[0:(len(n.Value) - len(tr))]
	t := strings.TrimRight(tr, " \t\n\r\f\v")
	right := tr[len(t):len(tr)]

	mw.writeString(left)

	if n.Bold {
		mw.writeString("**")
	}
	if n.Italic {
		mw.writeString("*")
	}
	if n.Code {
		mw.writeString("`")
	}

	mw.writeString(t)

	if n.Code {
		mw.writeString("`")
	}
	if n.Italic {
		mw.writeString("*")
	}
	if n.Bold {
		mw.writeString("**")
	}

	mw.writeString(right)
}

func (mw *mdWriter) image(n *types.ImageNode) {
	mw.space()
	mw.writeString("<img ")
	mw.writeString(fmt.Sprintf("src=\"%s\" ", n.Src))

	if n.Alt != "" {
		mw.writeString(fmt.Sprintf("alt=\"%s\" ", n.Alt))
	} else {
		mw.writeString(fmt.Sprintf("alt=\"%s\" ", path.Base(n.Src)))
	}

	if n.Title != "" {
		mw.writeString(fmt.Sprintf("title=\"%q\" ", n.Title))
	}

	// If available append width to the src string of the image.
	if n.Width > 0 {
		mw.writeString(fmt.Sprintf(" width=\"%.2f\" ", n.Width))
	}

	mw.writeString("/>")
}

func (mw *mdWriter) url(n *types.URLNode) {
	mw.space()
	if n.URL != "" {
		// Look-ahead for button syntax.
		if _, ok := n.Content.Nodes[0].(*types.ButtonNode); ok {
			mw.writeString("<button>")
		}
		mw.writeString("[")
	}
	mw.write(n.Content.Nodes...)
	if n.URL != "" {
		mw.writeString("](")
		mw.writeString(n.URL)
		mw.writeString(")")
		if _, ok := n.Content.Nodes[0].(*types.ButtonNode); ok {
			// Look-ahead for button syntax.
			mw.writeString("</button>")
		}
	}
}

func (mw *mdWriter) code(n *types.CodeNode) {
	mw.newBlock()
	defer mw.writeBytes(newLine)
	mw.writeString("```")
	if n.Term {
		mw.writeString("console")
	} else {
		mw.writeString(n.Lang)
	}
	mw.writeBytes(newLine)
	mw.writeString(n.Value)
	if !mw.lineStart {
		mw.writeBytes(newLine)
	}
	mw.writeString("```")
}

func (mw *mdWriter) list(n *types.ListNode) {
	if n.Block() == true {
		mw.newBlock()
	}
	mw.write(n.Nodes...)
	if !mw.lineStart && !mw.isWritingTableCell {
		mw.writeBytes(newLine)
	}
}

func (mw *mdWriter) itemsList(n *types.ItemsListNode) {
	if n.Block() == true {
		mw.newBlock()
	}
	for i, item := range n.Items {
		s := "* "
		if n.Type() == types.NodeItemsList && n.Start > 0 {
			s = strconv.Itoa(i+n.Start) + ". "
		}
		mw.writeString(s)
		mw.write(item.Nodes...)
		if !mw.lineStart {
			mw.writeBytes(newLine)
		}
	}
}

func (mw *mdWriter) infobox(n *types.InfoboxNode) {
	// InfoBoxes are comprised of a ListNode with the contents of the InfoBox.
	// Writing the ListNode directly results in extra newlines in the md output
	// which breaks the formatting. So instead, write the ListNode's children
	// directly and don't write the ListNode itself.
	mw.newBlock()
	k := "aside positive"
	if n.Kind == types.InfoboxNegative {
		k = "aside negative"
	}
	mw.Prefix = "> "
	mw.writeString(k)
	mw.writeString("\n")

	for _, cn := range n.Content.Nodes {
		cn.MutateBlock(false)
		mw.write(cn)
	}

	mw.Prefix = ""
}

func (mw *mdWriter) header(n *types.HeaderNode) {
	mw.newBlock()
	mw.writeString(strings.Repeat("#", n.Level+1))
	mw.writeString(" ")
	mw.write(n.Content.Nodes...)
	if !mw.lineStart {
		mw.writeBytes(newLine)
	}
}

func (mw *mdWriter) youtube(n *types.YouTubeNode) {
	mw.newBlock()
	mw.writeString(fmt.Sprintf(`<video id="%s"></video>`, n.VideoID))
}

func (mw *mdWriter) table(n *types.GridNode) {
	for rowIndex, row := range n.Rows {
		mw.writeString("| ")
		for _, cell := range row {
			mw.isWritingTableCell = true

			for _, cn := range cell.Content.Nodes {
				cn.MutateBlock(false) // don't treat content as a new block
				mw.write(cn)
			}
		}
		mw.writeString(" |")
		mw.writeBytes(newLine)

		// Write header bottom border
		if rowIndex == 0 {
			mw.writeString("| ")
			for range row {
				mw.writeString("---")
			}
			mw.writeString(" |")
			mw.writeBytes(newLine)
		}

		mw.isWritingTableCell = false
	}
}
<commit_msg>Add newline before table<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package render

import (
	"bytes"
	"fmt"
	"io"
	"path"
	"sort"
	"strconv"
	"strings"

	"github.com/googlecodelabs/tools/claat/types"
)

// MD renders nodes as markdown for the target env.
func MD(ctx Context, nodes ...types.Node) (string, error) {
	var buf bytes.Buffer
	if err := WriteMD(&buf, ctx.Env, nodes...); err != nil {
		return "", err
	}
	return buf.String(), nil
}

// WriteMD does the same as MD but outputs rendered markup to w.
func WriteMD(w io.Writer, env string, nodes ...types.Node) error {
	mw := mdWriter{w: w, env: env, Prefix: ""}
	return mw.write(nodes...)
}

type mdWriter struct {
	w                  io.Writer // output writer
	env                string    // target environment
	err                error     // error during any writeXxx methods
	lineStart          bool
	isWritingTableCell bool   // used to override lineStart for correct cell formatting
	Prefix             string // prefix for e.g. blockquote content
}

func (mw *mdWriter) writeBytes(b []byte) {
	if mw.err != nil {
		return
	}
	mw.lineStart = len(b) > 0 && b[len(b)-1] == '\n'
	_, mw.err = mw.w.Write(b)
}

func (mw *mdWriter) writeString(s string) {
	if mw.lineStart {
		s = mw.Prefix + s
	}
	mw.writeBytes([]byte(s))
}

func (mw *mdWriter) space() {
	if !mw.lineStart {
		mw.writeString(" ")
	}
}

func (mw *mdWriter) newBlock() {
	if !mw.lineStart {
		mw.writeBytes(newLine)
	}
	mw.writeBytes(newLine)
}

func (mw *mdWriter) matchEnv(v []string) bool {
	if len(v) == 0 || mw.env == "" {
		return true
	}
	i := sort.SearchStrings(v, mw.env)
	return i < len(v) && v[i] == mw.env
}

func (mw *mdWriter) write(nodes ...types.Node) error {
	for _, n := range nodes {
		if !mw.matchEnv(n.Env()) {
			continue
		}
		switch n := n.(type) {
		case *types.TextNode:
			mw.text(n)
		case *types.ImageNode:
			mw.image(n)
		case *types.URLNode:
			mw.url(n)
		case *types.ButtonNode:
			mw.write(n.Content.Nodes...)
		case *types.CodeNode:
			mw.code(n)
		case *types.ListNode:
			mw.list(n)
		case *types.ImportNode:
			if len(n.Content.Nodes) == 0 {
				break
			}
			mw.write(n.Content.Nodes...)
		case *types.ItemsListNode:
			mw.itemsList(n)
		case *types.GridNode:
			mw.table(n)
		case *types.InfoboxNode:
			mw.infobox(n)
		//case *types.SurveyNode:
		//	mw.survey(n)
		case *types.HeaderNode:
			mw.header(n)
		case *types.YouTubeNode:
			mw.youtube(n)
		}
		if mw.err != nil {
			return mw.err
		}
	}
	return nil
}

func (mw *mdWriter) text(n *types.TextNode) {
	tr := strings.TrimLeft(n.Value, " \t\n\r\f\v")
	left := n.Value[0:(len(n.Value) - len(tr))]
	t := strings.TrimRight(tr, " \t\n\r\f\v")
	right := tr[len(t):len(tr)]

	mw.writeString(left)

	if n.Bold {
		mw.writeString("**")
	}
	if n.Italic {
		mw.writeString("*")
	}
	if n.Code {
		mw.writeString("`")
	}

	mw.writeString(t)

	if n.Code {
		mw.writeString("`")
	}
	if n.Italic {
		mw.writeString("*")
	}
	if n.Bold {
		mw.writeString("**")
	}

	mw.writeString(right)
}

func (mw *mdWriter) image(n *types.ImageNode) {
	mw.space()
	mw.writeString("<img ")
	mw.writeString(fmt.Sprintf("src=\"%s\" ", n.Src))

	if n.Alt != "" {
		mw.writeString(fmt.Sprintf("alt=\"%s\" ", n.Alt))
	} else {
		mw.writeString(fmt.Sprintf("alt=\"%s\" ", path.Base(n.Src)))
	}

	if n.Title != "" {
		mw.writeString(fmt.Sprintf("title=\"%q\" ", n.Title))
	}

	// If available append width to the src string of the image.
	if n.Width > 0 {
		mw.writeString(fmt.Sprintf(" width=\"%.2f\" ", n.Width))
	}

	mw.writeString("/>")
}

func (mw *mdWriter) url(n *types.URLNode) {
	mw.space()
	if n.URL != "" {
		// Look-ahead for button syntax.
		if _, ok := n.Content.Nodes[0].(*types.ButtonNode); ok {
			mw.writeString("<button>")
		}
		mw.writeString("[")
	}
	mw.write(n.Content.Nodes...)
	if n.URL != "" {
		mw.writeString("](")
		mw.writeString(n.URL)
		mw.writeString(")")
		if _, ok := n.Content.Nodes[0].(*types.ButtonNode); ok {
			// Look-ahead for button syntax.
			mw.writeString("</button>")
		}
	}
}

func (mw *mdWriter) code(n *types.CodeNode) {
	mw.newBlock()
	defer mw.writeBytes(newLine)
	mw.writeString("```")
	if n.Term {
		mw.writeString("console")
	} else {
		mw.writeString(n.Lang)
	}
	mw.writeBytes(newLine)
	mw.writeString(n.Value)
	if !mw.lineStart {
		mw.writeBytes(newLine)
	}
	mw.writeString("```")
}

func (mw *mdWriter) list(n *types.ListNode) {
	if n.Block() == true {
		mw.newBlock()
	}
	mw.write(n.Nodes...)
	if !mw.lineStart && !mw.isWritingTableCell {
		mw.writeBytes(newLine)
	}
}

func (mw *mdWriter) itemsList(n *types.ItemsListNode) {
	if n.Block() == true {
		mw.newBlock()
	}
	for i, item := range n.Items {
		s := "* "
		if n.Type() == types.NodeItemsList && n.Start > 0 {
			s = strconv.Itoa(i+n.Start) + ". "
		}
		mw.writeString(s)
		mw.write(item.Nodes...)
		if !mw.lineStart {
			mw.writeBytes(newLine)
		}
	}
}

func (mw *mdWriter) infobox(n *types.InfoboxNode) {
	// InfoBoxes are comprised of a ListNode with the contents of the InfoBox.
	// Writing the ListNode directly results in extra newlines in the md output
	// which breaks the formatting. So instead, write the ListNode's children
	// directly and don't write the ListNode itself.
	mw.newBlock()
	k := "aside positive"
	if n.Kind == types.InfoboxNegative {
		k = "aside negative"
	}
	mw.Prefix = "> "
	mw.writeString(k)
	mw.writeString("\n")

	for _, cn := range n.Content.Nodes {
		cn.MutateBlock(false)
		mw.write(cn)
	}

	mw.Prefix = ""
}

func (mw *mdWriter) header(n *types.HeaderNode) {
	mw.newBlock()
	mw.writeString(strings.Repeat("#", n.Level+1))
	mw.writeString(" ")
	mw.write(n.Content.Nodes...)
	if !mw.lineStart {
		mw.writeBytes(newLine)
	}
}

func (mw *mdWriter) youtube(n *types.YouTubeNode) {
	mw.newBlock()
	mw.writeString(fmt.Sprintf(`<video id="%s"></video>`, n.VideoID))
}

func (mw *mdWriter) table(n *types.GridNode) {
	mw.writeBytes(newLine)
	for rowIndex, row := range n.Rows {
		mw.writeString("|")
		for _, cell := range row {
			mw.isWritingTableCell = true
			mw.writeString(" ")
			for _, cn := range cell.Content.Nodes {
				cn.MutateBlock(false) // don't treat content as a new block
				mw.write(cn)
			}
			mw.writeString(" |")
		}
		mw.writeBytes(newLine)

		// Write header bottom border
		if rowIndex == 0 {
			mw.writeString("|")
			for range row {
				mw.writeString(" --- |")
			}
			mw.writeBytes(newLine)
		}

		mw.isWritingTableCell = false
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The StudyGolang Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// http://studygolang.com
// Author: polaris	polaris@studygolang.com

package controller

import (
	"bytes"
	"html/template"
	"logic"
	"math/rand"
	"model"
	"net/http"
	"net/url"
	"strings"

	. "http"

	"github.com/labstack/echo"
	"github.com/polaris1119/config"
	"github.com/polaris1119/goutils"
	"github.com/polaris1119/logger"
)

type IndexController struct{}

// 注册路由
func (self IndexController) RegisterRoute(g *echo.Group) {
	g.GET("/", self.Index)
	g.GET("/wr", self.WrapUrl)
	g.GET("/pkgdoc", self.Pkgdoc)
	g.GET("/markdown", self.Markdown)
	g.GET("/link", self.Link)
}

func (IndexController) Index(ctx echo.Context) error {
	if len(logic.WebsiteSetting.IndexNavs) == 0 {
		return render(ctx, "index.html", nil)
	}

	tab := ctx.QueryParam("tab")
	if tab == "" {
		tab = GetFromCookie(ctx, "INDEX_TAB")
	}

	if tab == "" {
		tab = logic.WebsiteSetting.IndexNavs[0].Tab
	}
	paginator := logic.NewPaginator(goutils.MustInt(ctx.QueryParam("p"), 1))

	data := logic.DefaultIndex.FindData(ctx, tab, paginator)

	SetCookie(ctx, "INDEX_TAB", data["tab"].(string))

	data["all_nodes"] = logic.GenNodes()

	if tab == "all" {
		pageHtml := paginator.SetTotal(logic.DefaultFeed.GetTotalCount(ctx)).GetPageHtml(ctx.Request().URL().Path())

		data["page"] = template.HTML(pageHtml)

		data["total"] = paginator.GetTotal()

	}

	return render(ctx, "index.html", data)
}

// Index 首页
func (IndexController) OldIndex(ctx echo.Context) error {
	num := 10
	paginator := logic.NewPaginatorWithPerPage(1, num)
	topicsList := make([]map[string]interface{}, num)

	// 置顶的topic
	topTopics := logic.DefaultTopic.FindAll(ctx, paginator, "ctime DESC", "top=1")
	if len(topTopics) < num {
		// 获取最新帖子
		paginator.SetPerPage(num - len(topTopics))
		newTopics := logic.DefaultTopic.FindAll(ctx, paginator, "ctime DESC", "top=0")

		topicsList = append(topTopics, newTopics...)
	}

	// 获得最新博文
	recentArticles := logic.DefaultArticle.FindBy(ctx, 10)
	// 获取当前用户喜欢对象信息
	var likeFlags map[int]int

	if len(recentArticles) > 0 {
		curUser, ok := ctx.Get("user").(*model.Me)
		if ok {
			likeFlags, _ = logic.DefaultLike.FindUserLikeObjects(ctx, curUser.Uid, model.TypeArticle, recentArticles[0].Id, recentArticles[len(recentArticles)-1].Id)
		}
	}

	// 资源
	resources := logic.DefaultResource.FindBy(ctx, 10)

	books := logic.DefaultGoBook.FindBy(ctx, 24)
	if len(books) > 8 {
		bookNum := 8
		bookStart := rand.Intn(len(books) - bookNum)
		books = books[bookStart : bookStart+bookNum]
	}

	// 学习资料
	materials := logic.DefaultLearningMaterial.FindAll(ctx)

	return render(ctx, "index.html",
		map[string]interface{}{
			"topics":    topicsList,
			"articles":  recentArticles,
			"likeflags": likeFlags,
			"resources": resources,
			"books":     books,
			"materials": materials,
		})
}

// WrapUrl 包装链接
func (IndexController) WrapUrl(ctx echo.Context) error {
	tUrl := ctx.QueryParam("u")
	if tUrl == "" {
		return ctx.Redirect(http.StatusSeeOther, "/")
	}

	// 本站
	if strings.Contains(tUrl, logic.WebsiteSetting.Domain) {
		return ctx.Redirect(http.StatusSeeOther, tUrl)
	}

	if strings.Contains(tUrl, "?") {
		tUrl += "&"
	} else {
		tUrl += "?"
	}
	tUrl += "hmsr=studygolang.com&utm_medium=studygolang.com&utm_source=studygolang.com"

	if CheckIsHttps(ctx) {
		return ctx.Redirect(http.StatusSeeOther, tUrl)
	}

	var (
		pUrl *url.URL
		err  error
	)

	if pUrl, err = url.Parse(tUrl); err != nil {
		return ctx.Redirect(http.StatusSeeOther, tUrl)
	}

	iframeDeny := config.ConfigFile.MustValue("crawl", "iframe_deny")
	// 检测是否禁止了 iframe 加载
	// 看是否在黑名单中
	for _, denyHost := range strings.Split(iframeDeny, ",") {
		if strings.Contains(pUrl.Host, denyHost) {
			return ctx.Redirect(http.StatusSeeOther, tUrl)
		}
	}

	// 检测会比较慢，进行异步检测，记录下来，以后分析再加黑名单
	go func() {
		resp, err := http.Head(tUrl)
		if err != nil {
			logger.Errorln("[iframe] head url:", tUrl, "error:", err)
			return
		}
		defer resp.Body.Close()
		if resp.Header.Get("X-Frame-Options") != "" {
			logger.Errorln("[iframe] deny:", tUrl)
			return
		}
	}()

	return render(ctx, "wr.html", map[string]interface{}{"url": tUrl})
}

// PkgdocHandler Go 语言文档中文版
func (IndexController) Pkgdoc(ctx echo.Context) error {
	// return render(ctx, "pkgdoc.html", map[string]interface{}{"activeDoc": "active"})
	tpl, err := template.ParseFiles(config.TemplateDir + "pkgdoc.html")
	if err != nil {
		logger.Errorln("parse file error:", err)
		return err
	}

	buf := new(bytes.Buffer)
	err = tpl.Execute(buf, nil)
	if err != nil {
		logger.Errorln("execute template error:", err)
		return err
	}

	return ctx.HTML(http.StatusOK, buf.String())
}

func (IndexController) Markdown(ctx echo.Context) error {
	return render(ctx, "markdown.html", nil)
}

// Link 用于重定向外部链接，比如广告链接
func (IndexController) Link(ctx echo.Context) error {
	tUrl := ctx.QueryParam("url")
	return ctx.Redirect(http.StatusSeeOther, tUrl)
}
<commit_msg>加上媒体信息<commit_after>// Copyright 2016 The StudyGolang Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// http://studygolang.com
// Author: polaris	polaris@studygolang.com

package controller

import (
	"bytes"
	"html/template"
	"logic"
	"math/rand"
	"model"
	"net/http"
	"net/url"
	"strings"

	. "http"

	"github.com/labstack/echo"
	"github.com/polaris1119/config"
	"github.com/polaris1119/goutils"
	"github.com/polaris1119/logger"
)

type IndexController struct{}

// 注册路由
func (self IndexController) RegisterRoute(g *echo.Group) {
	g.GET("/", self.Index)
	g.GET("/wr", self.WrapUrl)
	g.GET("/pkgdoc", self.Pkgdoc)
	g.GET("/markdown", self.Markdown)
	g.GET("/link", self.Link)
}

func (IndexController) Index(ctx echo.Context) error {
	if len(logic.WebsiteSetting.IndexNavs) == 0 {
		return render(ctx, "index.html", nil)
	}

	tab := ctx.QueryParam("tab")
	if tab == "" {
		tab = GetFromCookie(ctx, "INDEX_TAB")
	}

	if tab == "" {
		tab = logic.WebsiteSetting.IndexNavs[0].Tab
	}
	paginator := logic.NewPaginator(goutils.MustInt(ctx.QueryParam("p"), 1))

	data := logic.DefaultIndex.FindData(ctx, tab, paginator)

	SetCookie(ctx, "INDEX_TAB", data["tab"].(string))

	data["all_nodes"] = logic.GenNodes()

	if tab == "all" {
		pageHtml := paginator.SetTotal(logic.DefaultFeed.GetTotalCount(ctx)).GetPageHtml(ctx.Request().URL().Path())

		data["page"] = template.HTML(pageHtml)

		data["total"] = paginator.GetTotal()

	}

	return render(ctx, "index.html", data)
}

// Index 首页
func (IndexController) OldIndex(ctx echo.Context) error {
	num := 10
	paginator := logic.NewPaginatorWithPerPage(1, num)
	topicsList := make([]map[string]interface{}, num)

	// 置顶的topic
	topTopics := logic.DefaultTopic.FindAll(ctx, paginator, "ctime DESC", "top=1")
	if len(topTopics) < num {
		// 获取最新帖子
		paginator.SetPerPage(num - len(topTopics))
		newTopics := logic.DefaultTopic.FindAll(ctx, paginator, "ctime DESC", "top=0")

		topicsList = append(topTopics, newTopics...)
	}

	// 获得最新博文
	recentArticles := logic.DefaultArticle.FindBy(ctx, 10)
	// 获取当前用户喜欢对象信息
	var likeFlags map[int]int

	if len(recentArticles) > 0 {
		curUser, ok := ctx.Get("user").(*model.Me)
		if ok {
			likeFlags, _ = logic.DefaultLike.FindUserLikeObjects(ctx, curUser.Uid, model.TypeArticle, recentArticles[0].Id, recentArticles[len(recentArticles)-1].Id)
		}
	}

	// 资源
	resources := logic.DefaultResource.FindBy(ctx, 10)

	books := logic.DefaultGoBook.FindBy(ctx, 24)
	if len(books) > 8 {
		bookNum := 8
		bookStart := rand.Intn(len(books) - bookNum)
		books = books[bookStart : bookStart+bookNum]
	}

	// 学习资料
	materials := logic.DefaultLearningMaterial.FindAll(ctx)

	return render(ctx, "index.html",
		map[string]interface{}{
			"topics":    topicsList,
			"articles":  recentArticles,
			"likeflags": likeFlags,
			"resources": resources,
			"books":     books,
			"materials": materials,
		})
}

// WrapUrl 包装链接
func (IndexController) WrapUrl(ctx echo.Context) error {
	tUrl := ctx.QueryParam("u")
	if tUrl == "" {
		return ctx.Redirect(http.StatusSeeOther, "/")
	}

	// 本站
	if strings.Contains(tUrl, logic.WebsiteSetting.Domain) {
		return ctx.Redirect(http.StatusSeeOther, tUrl)
	}

	if strings.Contains(tUrl, "?") {
		tUrl += "&"
	} else {
		tUrl += "?"
	}
	tUrl += "utm_campaign=studygolang.com&utm_medium=studygolang.com&utm_source=studygolang.com"

	if CheckIsHttps(ctx) {
		return ctx.Redirect(http.StatusSeeOther, tUrl)
	}

	var (
		pUrl *url.URL
		err  error
	)

	if pUrl, err = url.Parse(tUrl); err != nil {
		return ctx.Redirect(http.StatusSeeOther, tUrl)
	}

	iframeDeny := config.ConfigFile.MustValue("crawl", "iframe_deny")
	// 检测是否禁止了 iframe 加载
	// 看是否在黑名单中
	for _, denyHost := range strings.Split(iframeDeny, ",") {
		if strings.Contains(pUrl.Host, denyHost) {
			return ctx.Redirect(http.StatusSeeOther, tUrl)
		}
	}

	// 检测会比较慢，进行异步检测，记录下来，以后分析再加黑名单
	go func() {
		resp, err := http.Head(tUrl)
		if err != nil {
			logger.Errorln("[iframe] head url:", tUrl, "error:", err)
			return
		}
		defer resp.Body.Close()
		if resp.Header.Get("X-Frame-Options") != "" {
			logger.Errorln("[iframe] deny:", tUrl)
			return
		}
	}()

	return render(ctx, "wr.html", map[string]interface{}{"url": tUrl})
}

// PkgdocHandler Go 语言文档中文版
func (IndexController) Pkgdoc(ctx echo.Context) error {
	// return render(ctx, "pkgdoc.html", map[string]interface{}{"activeDoc": "active"})
	tpl, err := template.ParseFiles(config.TemplateDir + "pkgdoc.html")
	if err != nil {
		logger.Errorln("parse file error:", err)
		return err
	}

	buf := new(bytes.Buffer)
	err = tpl.Execute(buf, nil)
	if err != nil {
		logger.Errorln("execute template error:", err)
		return err
	}

	return ctx.HTML(http.StatusOK, buf.String())
}

func (IndexController) Markdown(ctx echo.Context) error {
	return render(ctx, "markdown.html", nil)
}

// Link 用于重定向外部链接，比如广告链接
func (IndexController) Link(ctx echo.Context) error {
	tUrl := ctx.QueryParam("url")
	if strings.Contains(tUrl, "?") {
		tUrl += "&"
	} else {
		tUrl += "?"
	}
	tUrl += "utm_campaign=studygolang.com&utm_medium=studygolang.com&utm_source=studygolang.com"
	return ctx.Redirect(http.StatusSeeOther, tUrl)
}
<|endoftext|>