<commit_before>package main

import (
  "fmt"
  "net/http"
  "encoding/json"
  "bytes"
  "strconv"
)

func GetMiner(id int) (miner Miner, err error) {

  rows := QueryMinerForId(id)
  for rows.Next() {
    var uid int
    var name string
    var source string
    var location string
    var url string
    var geoCoord Point
    var locationHash int
    var stopwords string
    err := rows.Scan(&uid, &name, &source, &location, &url, &geoCoord, &locationHash, &stopwords)
    checkErr(err)
    miner = Miner {
      Uid: uid,
      Name: name,
      Source: source,
      Location: location,
      GeoCoord: geoCoord,
      Url: url,
      Stopwords: stopwords,
    }
  }
  return
}

// Miners admin home page
func AdminMiners(w http.ResponseWriter, r *http.Request) {

  sess, err := globalSessions.SessionStart(w, r)
  if err != nil {
      //need logging here instead of print
      fmt.Printf("Error, could not start session %v\n", err)
      return
  }
  defer sess.SessionRelease(w)
  username := sess.Get("username")
  if username == nil {
    AdminLogin(w, r)
  } else {
    miners, err :=  MinersCollection()
    content := make(map[string]interface{})

    content["Title"] = "Miners Admin"
    if err != nil {
      content["Error"] = "Miners database table not yet created"
    } else {
      content["Miners"] = miners
    }
    renderTemplate(w, "admin/miners/index", content)
  }
}

func AdminNewMiner(w http.ResponseWriter, r *http.Request) {
  sess, err := globalSessions.SessionStart(w, r)
  if err != nil {
      //need logging here instead of print
      fmt.Printf("Error, could not start session %v\n", err)
      return
  }
  defer sess.SessionRelease(w)
  username := sess.Get("username")
  if username == nil {
    AdminLogin(w, r)
  } else {
    content := make(map[string]interface{})
    content["Title"] = "Miners Admin: Add New Miner"
    fmt.Fprintf(w, "
NEW MINER
")
    //renderTemplate(w, "admin/miners/new", content)
  }
}

func AdminEditMiner(w http.ResponseWriter, r *http.Request) {
  sess, err := globalSessions.SessionStart(w, r)
  if err != nil {
      //need logging here instead of print
      fmt.Printf("Error, could not start session %v\n", err)
      return
  }
  defer sess.SessionRelease(w)
  username := sess.Get("username")
  if username == nil {
    AdminLogin(w, r)
  } else {
    content := make(map[string]interface{})
    content["Title"] = "Miners Admin: Add New Miner"

    uidParam := r.URL.Query().Get("uid")
    uidConv, _ := strconv.ParseInt(uidParam, 10, 0)
    miner, err := GetMiner(int(uidConv))

    if err != nil {
      content["Error"] = "Could not retrieve miner"
    } else {
      content["Miner"] = miner
    }

    fmt.Fprintf(w, "
EDIT MINER {{ .Miner.uid }}
")
    //renderTemplate(w, "admin/miners/edit", content)
  }
}

func AdminUpdateMiner(w http.ResponseWriter, r *http.Request) {
  sess, err := globalSessions.SessionStart(w, r)
  if err != nil {
      //need logging here instead of print
      fmt.Printf("Error, could not start session %v\n", err)
      return
  }
  defer sess.SessionRelease(w)
  username := sess.Get("username")
  if username == nil {
    AdminLogin(w, r)
  } else {
    content := make(map[string]interface{})
    content["Title"] = "Miners Admin: Update Miner"

    uidParam := r.URL.Query().Get("uid")
    uidConv, _ := strconv.ParseInt(uidParam, 10, 0)
    miner, err := GetMiner(int(uidConv))

    if err != nil {
      content["Error"] = "Could not retrieve miner"
    } else {
      content["Miner"] = miner
    }

    fmt.Fprintf(w, "
UPDATE MINER AND REDIRECT
")
  }
}

func AdminDeleteMiner(w http.ResponseWriter, r *http.Request) {
  sess, err := globalSessions.SessionStart(w, r)
  if err != nil {
      //need logging here instead of print
      fmt.Printf("Error, could not start session %v\n", err)
      return
  }
  defer sess.SessionRelease(w)
  username := sess.Get("username")
  if username == nil {
    AdminLogin(w, r)
  } else {
    content := make(map[string]interface{})
    content["Title"] = "Miners Admin: Delete Miner"

    uidParam := r.URL.Query().Get("uid")
    uidConv, _ := strconv.ParseInt(uidParam, 10, 0)
    miner, err := GetMiner(int(uidConv))

    if err != nil {
      content["Error"] = "Could not retrieve miner"
    } else {
      content["Miner"] = miner
    }

    fmt.Fprintf(w, "
DELETE MINER AND REDIRECT
")
  }
}

func AdminMinersResetDatabase(w http.ResponseWriter, r *http.Request) {

  sess, err := globalSessions.SessionStart(w, r)
  if err != nil {
      //need logging here instead of print
      fmt.Printf("Error, could not start session %v\n", err)
      return
  }
  defer sess.SessionRelease(w)
  username := sess.Get("username")
  if username == nil {
    AdminLogin(w, r)
  } else {
    ResetMinersDatabase()

    AdminMiners(w, r)
  }
}

// Creates a new miner
func AdminCreateMiner(w http.ResponseWriter, r *http.Request) {
  sess, err := globalSessions.SessionStart(w, r)
  if err != nil {
      //need logging here instead of print
      fmt.Printf("Error, could not start session %v\n", err)
      return
  }
  defer sess.SessionRelease(w)
  username := sess.Get("username")
  if username == nil {
    AdminLogin(w, r)
  } else {

    err := r.ParseForm()
    if err != nil {
      fmt.Println(err)
    }

    content := make(map[string]interface{})

    name := r.PostFormValue("name")
    url := r.PostFormValue("url")
    location := r.PostFormValue("location")
    latitude := r.PostFormValue("latitude")
    longitude := r.PostFormValue("longitude")
    source := r.PostFormValue("source")
    stopwords := r.PostFormValue("stopwords")
    lastInsertId, err := InsertMiner(name, location, latitude, longitude, source, url, stopwords)
    if err != nil {
      content["MinerError"] = err
    }
    sendIdUrl := fmt.Sprintf("%s/categories", url)
    idData := fmt.Sprintf("{\"id\":\"%d\"}", lastInsertId)

    var jsonStr = []byte(idData)
    req, err := http.NewRequest("POST", sendIdUrl, bytes.NewBuffer(jsonStr))
    req.Header.Set("Content-Type", "application/json")

    client := &http.Client{}
    resp, err := client.Do(req)
    if err == nil {
      defer resp.Body.Close()
    }

    content["Title"] = "Miners Admin"
    if err != nil {
      content["MinerError"] = err
    }

    miners, err :=  MinersCollection()
    if err != nil {
      content["Error"] = "Miners database table not yet created"
    } else {
      content["Miners"] = miners
    }

    renderTemplate(w, "admin/miners/index", content)
  }
}

// Handles receipt of post from a Miner
func MinerPost(w http.ResponseWriter, r *http.Request) {
  decoder := json.NewDecoder(r.Body)
  var posts MinerPostsJSON
  err := decoder.Decode(&posts)
  if err != nil {
    fmt.Println("Error:", err)
    http.Error(w, err.Error(), 500)
    return
  }

  http.Error(w, "OK", 200)

  //fmt.Println("Post from Miner Id: ", posts.MinerId)
  //fmt.Println(posts)

  minerConv, _ := strconv.ParseInt(posts.MinerId, 10, 0)
  var miner Miner
  rows := QueryMinerForId(int(minerConv))
  for rows.Next() {
    var uid int
    var name string
    var source string
    var location string
    var url string
    var geoCoord Point
    var locationHash int
    var stopwords string
    err := rows.Scan(&uid, &name, &source, &location, &url, &geoCoord, &locationHash, &stopwords)
    checkErr(err)
    miner = Miner {
      Uid: uid,
      Name: name,
      Source: source,
      Location: location,
      GeoCoord: geoCoord,
      Url: url,
      Stopwords: stopwords,
    }
  }

  for _, post := range posts.Posts {
    url := post.Url
    posted := post.Datetime
    mined := post.MinedAt
    /*
    fmt.Println("SourceURI ", url)
    fmt.Println("Posted ", posted)
    fmt.Println("Mined ", mined)
    fmt.Println("Terms and their counts")
    fmt.Println("Miner location ", miner.Location)
    fmt.Println("Miner source", miner.Source)
    */
    lastInsertId := InsertPost(miner.Source, miner.Location, url, posted.Time, mined.Time)
    if lastInsertId != 0 {
      for k, v := range post.Terms {
        //fmt.Println(k, v)
        InsertTerm(miner.Location, k, v, lastInsertId, posted.Time)
      }
    }
  }
}<commit_msg>adds delete logic to handler<commit_after>package main

import (
  "fmt"
  "net/http"
  "encoding/json"
  "bytes"
  "strconv"
)

func GetMiner(id int) (miner Miner, err error) {

  rows := QueryMinerForId(id)
  for rows.Next() {
    var uid int
    var name string
    var source string
    var location string
    var url string
    var geoCoord Point
    var locationHash int
    var stopwords string
    err := rows.Scan(&uid, &name, &source, &location, &url, &geoCoord, &locationHash, &stopwords)
    checkErr(err)
    miner = Miner {
      Uid: uid,
      Name: name,
      Source: source,
      Location: location,
      GeoCoord: geoCoord,
      Url: url,
      Stopwords: stopwords,
    }
  }
  return
}

// Miners admin home page
func AdminMiners(w http.ResponseWriter, r *http.Request) {

  sess, err := globalSessions.SessionStart(w, r)
  if err != nil {
      //need logging here instead of print
      fmt.Printf("Error, could not start session %v\n", err)
      return
  }
  defer sess.SessionRelease(w)
  username := sess.Get("username")
  if username == nil {
    AdminLogin(w, r)
  } else {
    miners, err :=  MinersCollection()
    content := make(map[string]interface{})

    content["Title"] = "Miners Admin"
    if err != nil {
      content["Error"] = "Miners database table not yet created"
    } else {
      content["Miners"] = miners
    }
    renderTemplate(w, "admin/miners/index", content)
  }
}

func AdminNewMiner(w http.ResponseWriter, r *http.Request) {
  sess, err := globalSessions.SessionStart(w, r)
  if err != nil {
      //need logging here instead of print
      fmt.Printf("Error, could not start session %v\n", err)
      return
  }
  defer sess.SessionRelease(w)
  username := sess.Get("username")
  if username == nil {
    AdminLogin(w, r)
  } else {
    content := make(map[string]interface{})
    content["Title"] = "Miners Admin: Add New Miner"
    fmt.Fprintf(w, "
NEW MINER
")
    //renderTemplate(w, "admin/miners/new", content)
  }
}

func AdminEditMiner(w http.ResponseWriter, r *http.Request) {
  sess, err := globalSessions.SessionStart(w, r)
  if err != nil {
      //need logging here instead of print
      fmt.Printf("Error, could not start session %v\n", err)
      return
  }
  defer sess.SessionRelease(w)
  username := sess.Get("username")
  if username == nil {
    AdminLogin(w, r)
  } else {
    content := make(map[string]interface{})
    content["Title"] = "Miners Admin: Add New Miner"

    uidParam := r.URL.Query().Get("uid")
    uidConv, _ := strconv.ParseInt(uidParam, 10, 0)
    miner, err := GetMiner(int(uidConv))

    if err != nil {
      content["Error"] = "Could not retrieve miner"
    } else {
      content["Miner"] = miner
    }

    fmt.Fprintf(w, "
EDIT MINER {{ .Miner.uid }}
")
    //renderTemplate(w, "admin/miners/edit", content)
  }
}

func AdminUpdateMiner(w http.ResponseWriter, r *http.Request) {
  sess, err := globalSessions.SessionStart(w, r)
  if err != nil {
      //need logging here instead of print
      fmt.Printf("Error, could not start session %v\n", err)
      return
  }
  defer sess.SessionRelease(w)
  username := sess.Get("username")
  if username == nil {
    AdminLogin(w, r)
  } else {
    content := make(map[string]interface{})
    content["Title"] = "Miners Admin"

    uidParam := r.URL.Query().Get("uid")
    uidConv, _ := strconv.ParseInt(uidParam, 10, 0)
    miner, err := GetMiner(int(uidConv))

    if err != nil {
      content["Error"] = "Could not retrieve miner"
    } else {
      content["Miner"] = miner
    }

    fmt.Fprintf(w, "
UPDATE MINER AND REDIRECT
")
  }
}

func AdminDeleteMiner(w http.ResponseWriter, r *http.Request) {
  sess, err := globalSessions.SessionStart(w, r)
  if err != nil {
      //need logging here instead of print
      fmt.Printf("Error, could not start session %v\n", err)
      return
  }
  defer sess.SessionRelease(w)
  username := sess.Get("username")
  if username == nil {
    AdminLogin(w, r)
  } else {
    content := make(map[string]interface{})
    content["Title"] = "Miners Admin"

    //Delete miner
    uidParam := r.URL.Query().Get("uid")
    uidConv, _ := strconv.ParseInt(uidParam, 10, 0)
    affected, derr := DeleteMiner(int(uidConv))
    if (derr != nil) {
      content["Error"] = "Could not delete miner"
    }

    //Get remaining collection
    miners, err :=  MinersCollection()
    if err != nil {
      content["Error"] = "Miners database table not yet created"
    } else {
      content["Miners"] = miners
    }

    renderTemplate(w, "admin/miners/index", content)
  }
}

func AdminMinersResetDatabase(w http.ResponseWriter, r *http.Request) {

  sess, err := globalSessions.SessionStart(w, r)
  if err != nil {
      //need logging here instead of print
      fmt.Printf("Error, could not start session %v\n", err)
      return
  }
  defer sess.SessionRelease(w)
  username := sess.Get("username")
  if username == nil {
    AdminLogin(w, r)
  } else {
    ResetMinersDatabase()

    AdminMiners(w, r)
  }
}

// Creates a new miner
func AdminCreateMiner(w http.ResponseWriter, r *http.Request) {
  sess, err := globalSessions.SessionStart(w, r)
  if err != nil {
      //need logging here instead of print
      fmt.Printf("Error, could not start session %v\n", err)
      return
  }
  defer sess.SessionRelease(w)
  username := sess.Get("username")
  if username == nil {
    AdminLogin(w, r)
  } else {

    err := r.ParseForm()
    if err != nil {
      fmt.Println(err)
    }

    content := make(map[string]interface{})

    name := r.PostFormValue("name")
    url := r.PostFormValue("url")
    location := r.PostFormValue("location")
    latitude := r.PostFormValue("latitude")
    longitude := r.PostFormValue("longitude")
    source := r.PostFormValue("source")
    stopwords := r.PostFormValue("stopwords")
    lastInsertId, err := InsertMiner(name, location, latitude, longitude, source, url, stopwords)
    if err != nil {
      content["MinerError"] = err
    }
    sendIdUrl := fmt.Sprintf("%s/categories", url)
    idData := fmt.Sprintf("{\"id\":\"%d\"}", lastInsertId)

    var jsonStr = []byte(idData)
    req, err := http.NewRequest("POST", sendIdUrl, bytes.NewBuffer(jsonStr))
    req.Header.Set("Content-Type", "application/json")

    client := &http.Client{}
    resp, err := client.Do(req)
    if err == nil {
      defer resp.Body.Close()
    }

    content["Title"] = "Miners Admin"
    if err != nil {
      content["MinerError"] = err
    }

    miners, err :=  MinersCollection()
    if err != nil {
      content["Error"] = "Miners database table not yet created"
    } else {
      content["Miners"] = miners
    }

    renderTemplate(w, "admin/miners/index", content)
  }
}

// Handles receipt of post from a Miner
func MinerPost(w http.ResponseWriter, r *http.Request) {
  decoder := json.NewDecoder(r.Body)
  var posts MinerPostsJSON
  err := decoder.Decode(&posts)
  if err != nil {
    fmt.Println("Error:", err)
    http.Error(w, err.Error(), 500)
    return
  }

  http.Error(w, "OK", 200)

  //fmt.Println("Post from Miner Id: ", posts.MinerId)
  //fmt.Println(posts)

  minerConv, _ := strconv.ParseInt(posts.MinerId, 10, 0)
  var miner Miner
  rows := QueryMinerForId(int(minerConv))
  for rows.Next() {
    var uid int
    var name string
    var source string
    var location string
    var url string
    var geoCoord Point
    var locationHash int
    var stopwords string
    err := rows.Scan(&uid, &name, &source, &location, &url, &geoCoord, &locationHash, &stopwords)
    checkErr(err)
    miner = Miner {
      Uid: uid,
      Name: name,
      Source: source,
      Location: location,
      GeoCoord: geoCoord,
      Url: url,
      Stopwords: stopwords,
    }
  }

  for _, post := range posts.Posts {
    url := post.Url
    posted := post.Datetime
    mined := post.MinedAt
    /*
    fmt.Println("SourceURI ", url)
    fmt.Println("Posted ", posted)
    fmt.Println("Mined ", mined)
    fmt.Println("Terms and their counts")
    fmt.Println("Miner location ", miner.Location)
    fmt.Println("Miner source", miner.Source)
    */
    lastInsertId := InsertPost(miner.Source, miner.Location, url, posted.Time, mined.Time)
    if lastInsertId != 0 {
      for k, v := range post.Terms {
        //fmt.Println(k, v)
        InsertTerm(miner.Location, k, v, lastInsertId, posted.Time)
      }
    }
  }
}<|endoftext|>


<commit_before>package hashing

import (
	"crypto/hmac"
	"crypto/sha1"
	"crypto/sha256"
	"fmt"
	"hash"
	"strconv"
	"strings"
)

/*

The PBKDF2 algorithm is a slightly modified version of:
https://bitbucket.org/taruti/pbkdf2/

Copyright (c) 2010-2011 Taru Karttunen <taruti@taruti.net>

Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following
conditions:

The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.

*/

func Pbkdf2(cleartext []byte, salt []byte, rounds int, hash func() hash.Hash, outlen int) []byte {
	// TODO necessity of outlen? For the app we'll be using the hash Size
	hashSize := hash().Size()
	if outlen == 0 {
		outlen = hashSize
	}
	out := make([]byte, outlen)
	ibuf := make([]byte, 4)
	block := 1
	p := out
	for outlen > 0 {
		clen := outlen
		if clen > hashSize {
			clen = hashSize
		}
		ibuf[0] = byte((block >> 24) & 0xff)
		ibuf[1] = byte((block >> 16) & 0xff)
		ibuf[2] = byte((block >> 8) & 0xff)
		ibuf[3] = byte((block) & 0xff)
		hmac := hmac.New(hash, cleartext)
		hmac.Write(salt)
		hmac.Write(ibuf)
		tmp := hmac.Sum(nil)
		for i := 0; i < clen; i++ {
			p[i] = tmp[i]
		}
		for j := 1; j < rounds; j++ {
			hmac.Reset()
			hmac.Write(tmp)
			tmp = hmac.Sum(nil)
			for k := 0; k < clen; k++ {
				p[k] ^= tmp[k]
			}
		}
		outlen -= clen
		block++
		p = p[clen:]
	}
	return out
}

// TODO declare private?
type PBKDF2_Base struct {
	BaseHasher
	Digest func() hash.Hash // TODO move to base hasher?
}

func (pbkH *PBKDF2_Base) Encode(cleartext, salt string) string {
	// TODO these []byte conversions are a bit silly
	rounds := 10000
	hashed := EncodeBase64String(Pbkdf2([]byte(cleartext), []byte(salt), rounds, pbkH.Digest, 0))
	return strings.Join([]string{pbkH.GetAlgorithm(), fmt.Sprintf("%d", rounds), salt, hashed}, "$")
}

func (pbkH *PBKDF2_Base) Verify(cleartext, encoded string) bool {
	// Split the saved hash apart
	splitHash := strings.SplitN(encoded, "$", 4)

	// The algorithm should match this hasher
	algo := splitHash[0]
	if algo != pbkH.Algorithm {
		return false
	}
	rounds64, err := strconv.ParseInt(splitHash[1], 10, 0)
	if err != nil {
		return false
	}
	rounds := int(rounds64)
	salt := splitHash[2]

	// Generate a new hash using the given cleartext
	hashed := Pbkdf2([]byte(cleartext), []byte(salt), rounds, pbkH.Digest, 0)
	return ConstantTimeStringCompare(EncodeBase64String(hashed), splitHash[3])
}

func init() {
	pbkdf2_sha256 := &PBKDF2_Base{BaseHasher{Algorithm: "pbkdf2_sha256", Rounds: 10000}, sha256.New}
	Register(pbkdf2_sha256.Algorithm, pbkdf2_sha256)

	pbkdf2_sha1 := &PBKDF2_Base{BaseHasher{Algorithm: "pbkdf2_sha1", Rounds: 10000}, sha1.New}
	Register(pbkdf2_sha1.Algorithm, pbkdf2_sha1)
}<commit_msg>Switched PBKDF2 source to that of the Go authors<commit_after>package hashing

import (
	"crypto/hmac"
	"crypto/sha1"
	"crypto/sha256"
	"fmt"
	"hash"
	"strconv"
	"strings"
)

/*
Copyright (c) 2009 The Go Authors. All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

   * Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
   * Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
   * Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

*/

func key(password, salt []byte, iter, keyLen int, h func() hash.Hash) []byte {
	prf := hmac.New(h, password)
	hashLen := prf.Size()
	numBlocks := (keyLen + hashLen - 1) / hashLen

	var buf [4]byte
	dk := make([]byte, 0, numBlocks*hashLen)
	U := make([]byte, hashLen)
	for block := 1; block <= numBlocks; block++ {
		// N.B.: || means concatenation, ^ means XOR
		// for each block T_i = U_1 ^ U_2 ^ ... ^ U_iter
		// U_1 = PRF(password, salt || uint(i))
		prf.Reset()
		prf.Write(salt)
		buf[0] = byte(block >> 24)
		buf[1] = byte(block >> 16)
		buf[2] = byte(block >> 8)
		buf[3] = byte(block)
		prf.Write(buf[:4])
		dk = prf.Sum(dk)
		T := dk[len(dk)-hashLen:]
		copy(U, T)

		// U_n = PRF(password, U_(n-1))
		for n := 2; n <= iter; n++ {
			prf.Reset()
			prf.Write(U)
			U = U[:0]
			U = prf.Sum(U)
			for x := range U {
				T[x] ^= U[x]
			}
		}
	}
	return dk[:keyLen]
}

func Pbkdf2(cleartext, salt []byte, rounds int, h func() hash.Hash) []byte {
	// Use the hash Size as the keyLen
	return key(cleartext, salt, rounds, h().Size(), h)
}

// TODO declare private?
type PBKDF2_Base struct {
	BaseHasher
	Digest func() hash.Hash // TODO move to base hasher?
}

func (pbkH *PBKDF2_Base) Encode(cleartext, salt string) string {
	// TODO these []byte conversions are a bit silly
	rounds := 10000
	hashed := EncodeBase64String(Pbkdf2([]byte(cleartext), []byte(salt), rounds, pbkH.Digest))
	return strings.Join([]string{pbkH.GetAlgorithm(), fmt.Sprintf("%d", rounds), salt, hashed}, "$")
}

func (pbkH *PBKDF2_Base) Verify(cleartext, encoded string) bool {
	// Split the saved hash apart
	splitHash := strings.SplitN(encoded, "$", 4)

	// The algorithm should match this hasher
	algo := splitHash[0]
	if algo != pbkH.Algorithm {
		return false
	}
	rounds64, err := strconv.ParseInt(splitHash[1], 10, 0)
	if err != nil {
		return false
	}
	rounds := int(rounds64)
	salt := splitHash[2]

	// Generate a new hash using the given cleartext
	hashed := Pbkdf2([]byte(cleartext), []byte(salt), rounds, pbkH.Digest)
	return ConstantTimeStringCompare(EncodeBase64String(hashed), splitHash[3])
}

func init() {
	pbkdf2_sha256 := &PBKDF2_Base{BaseHasher{Algorithm: "pbkdf2_sha256", Rounds: 10000}, sha256.New}
	Register(pbkdf2_sha256.Algorithm, pbkdf2_sha256)

	pbkdf2_sha1 := &PBKDF2_Base{BaseHasher{Algorithm: "pbkdf2_sha1", Rounds: 10000}, sha1.New}
	Register(pbkdf2_sha1.Algorithm, pbkdf2_sha1)
}<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/

package hawk

import (
	"bytes"
	"encoding/base64"
	"encoding/hex"
	"net/http"
	"strings"
	"testing"
)

func Test_parseParameters(t *testing.T) {
	parameters, err := parseParameters(`id="dh37fgj492je", ts="1353832234", nonce="j4h3g2", hash="Yi9LfIIFRtBEPt74PVmbTF/xVAwPn7ub15ePICfgnuY=", ext="some-app-ext-data", mac="aSe1DERmZuRl3pI36/9BdZmnErTw3sNzOOAUlfeKjVw="`)
	if err != nil {
		t.Error("Cannot parse Hawk parameters", err)
	}

	if parameters.Id != "dh37fgj492je" {
		t.Error("id mismatch")
	}

	if parameters.Timestamp != 1353832234 {
		t.Error("ts mismatch")
	}

	if parameters.Nonce != "j4h3g2" {
		t.Error("nonce mismatch")
	}

	if parameters.Ext != "some-app-ext-data" {
		t.Error("ext mismatch")
	}

	expectedHash, _ := hex.DecodeString("622f4b7c820546d0443edef83d599b4c5ff1540c0f9fbb9bd7978f2027e09ee6")
	if !bytes.Equal(parameters.Hash, expectedHash) {
		t.Error("mac mismatch")
	}

	expectedMac, _ := hex.DecodeString("6927b50c446666e465de9237ebff417599a712b4f0dec37338e01495f78a8d5c")
	if !bytes.Equal(parameters.Mac, expectedMac) {
		t.Error("mac mismatch")
	}
}

func Test_validateParameters(t *testing.T) {
}

func Test_getRequestHost(t *testing.T) {
	test := func(url string, expectedHost string, headers map[string]string) {
		request, err := http.NewRequest("GET", url, nil)
		if err != nil {
			t.Error(err)
		}
		host := getRequestHost(request)
		if host != expectedHost {
			t.Errorf("Expected host %s for %s but got %s", expectedHost, url, host)
		}
	}
	test("http://localhost/foo", "localhost", nil)
	test("https://127.0.0.1/foo", "127.0.0.1", nil)
	test("http://localhost:8080/foo", "localhost", nil)
	test("https://127.0.0.1:8443/foo", "127.0.0.1", nil)
	// TODO: Add tests here that mimic a typical front proxy (X-Forwarded-Proto?)
	test("http://localhost/foo", "localhost", map[string]string{})
	test("https://127.0.0.1/foo", "127.0.0.1", map[string]string{})
	test("http://localhost:8080/foo", "localhost", map[string]string{})
	test("https://127.0.0.1:8443/foo", "127.0.0.1", map[string]string{})
}

func Test_getRequestPort(t *testing.T) {
	test := func(url string, expectedPort int) {
		request, err := http.NewRequest("GET", url, nil)
		if err != nil {
			t.Error(err)
		}
		port := getRequestPort(request)
		if port != expectedPort {
			t.Errorf("Expected port %d for %s but got %d", expectedPort, url, port)
		}
	}
	test("http://localhost/foo", 80)
	test("https://localhost/foo", 443)
	test("http://localhost:8080/foo", 8080)
	test("https://localhost:8443/foo", 8443)
}

func Test_calculatePayloadHash(t *testing.T) {
	r, err := http.NewRequest("POST", "http://localhost", strings.NewReader("Thank you for flying Hawk"))
	if err != nil {
		t.Error(err)
	}
	r.Header.Add("Content-Type", "text/plain")
	hash, err := calculatePayloadHash(r)
	if err != nil {
		t.Error(err)
	}
	expectedHash, _ := base64.StdEncoding.DecodeString("Yi9LfIIFRtBEPt74PVmbTF/xVAwPn7ub15ePICfgnuY=")
	if !bytes.Equal(hash, expectedHash) {
		t.Error("Hash mismatch")
	}
}

func Test_calculateRequestSignature(t *testing.T) {
	r, err := http.NewRequest("GET", "http://example.com:8000/resource/1?b=1&a=2", nil)
	if err != nil {
		t.Error(err)
	}

	parameters := Parameters{
		Timestamp: 1353832234,
		Ext:       "some-app-ext-data",
		Nonce:     "j4h3g2",
		Hash:      nil,
		// TODO: Rest is not important for this test
	}

	credentials := Credentials{
		KeyIdentifier: "dh37fgj492je",
		Key:           []byte("werxhqb98rpaxn39848xrunpaw3489ruxnpa98w4rxn"),
		Algorithm:     "sha256",
	}

	mac, err := calculateRequestSignature(r, parameters, credentials)
	if err != nil {
		t.Error(err)
	}

	expectedMac, _ := base64.StdEncoding.DecodeString("6R4rV5iE+NPoym+WwjeHzjAGXUtLNIxmo1vpMofpLAE=")
	if !bytes.Equal(mac, expectedMac) {
		t.Error("Mac mismatch")
	}
}
<commit_msg>Test for hawk.calculateRequestSignature for POST req<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/

package hawk

import (
	"bytes"
	"encoding/base64"
	"encoding/hex"
	"net/http"
	"strings"
	"testing"
)

func Test_parseParameters(t *testing.T) {
	parameters, err := parseParameters(`id="dh37fgj492je", ts="1353832234", nonce="j4h3g2", hash="Yi9LfIIFRtBEPt74PVmbTF/xVAwPn7ub15ePICfgnuY=", ext="some-app-ext-data", mac="aSe1DERmZuRl3pI36/9BdZmnErTw3sNzOOAUlfeKjVw="`)
	if err != nil {
		t.Error("Cannot parse Hawk parameters", err)
	}

	if parameters.Id != "dh37fgj492je" {
		t.Error("id mismatch")
	}

	if parameters.Timestamp != 1353832234 {
		t.Error("ts mismatch")
	}

	if parameters.Nonce != "j4h3g2" {
		t.Error("nonce mismatch")
	}

	if parameters.Ext != "some-app-ext-data" {
		t.Error("ext mismatch")
	}

	expectedHash, _ := hex.DecodeString("622f4b7c820546d0443edef83d599b4c5ff1540c0f9fbb9bd7978f2027e09ee6")
	if !bytes.Equal(parameters.Hash, expectedHash) {
		t.Error("mac mismatch")
	}

	expectedMac, _ := hex.DecodeString("6927b50c446666e465de9237ebff417599a712b4f0dec37338e01495f78a8d5c")
	if !bytes.Equal(parameters.Mac, expectedMac) {
		t.Error("mac mismatch")
	}
}

func Test_validateParameters(t *testing.T) {
}

func Test_getRequestHost(t *testing.T) {
	test := func(url string, expectedHost string, headers map[string]string) {
		request, err := http.NewRequest("GET", url, nil)
		if err != nil {
			t.Error(err)
		}
		host := getRequestHost(request)
		if host != expectedHost {
			t.Errorf("Expected host %s for %s but got %s", expectedHost, url, host)
		}
	}
	test("http://localhost/foo", "localhost", nil)
	test("https://127.0.0.1/foo", "127.0.0.1", nil)
	test("http://localhost:8080/foo", "localhost", nil)
	test("https://127.0.0.1:8443/foo", "127.0.0.1", nil)
	// TODO: Add tests here that mimic a typical front proxy (X-Forwarded-Proto?)
	test("http://localhost/foo", "localhost", map[string]string{})
	test("https://127.0.0.1/foo", "127.0.0.1", map[string]string{})
	test("http://localhost:8080/foo", "localhost", map[string]string{})
	test("https://127.0.0.1:8443/foo", "127.0.0.1", map[string]string{})
}

func Test_getRequestPort(t *testing.T) {
	test := func(url string, expectedPort int) {
		request, err := http.NewRequest("GET", url, nil)
		if err != nil {
			t.Error(err)
		}
		port := getRequestPort(request)
		if port != expectedPort {
			t.Errorf("Expected port %d for %s but got %d", expectedPort, url, port)
		}
	}
	test("http://localhost/foo", 80)
	test("https://localhost/foo", 443)
	test("http://localhost:8080/foo", 8080)
	test("https://localhost:8443/foo", 8443)
}

func Test_calculatePayloadHash(t *testing.T) {
	r, err := http.NewRequest("POST", "http://localhost", strings.NewReader("Thank you for flying Hawk"))
	if err != nil {
		t.Error(err)
	}
	r.Header.Add("Content-Type", "text/plain")
	hash, err := calculatePayloadHash(r)
	if err != nil {
		t.Error(err)
	}
	expectedHash, _ := base64.StdEncoding.DecodeString("Yi9LfIIFRtBEPt74PVmbTF/xVAwPn7ub15ePICfgnuY=")
	if !bytes.Equal(hash, expectedHash) {
		t.Error("Hash mismatch")
	}
}

func Test_calculateRequestSignatureWithGET(t *testing.T) {
	r, err := http.NewRequest("GET", "http://example.com:8000/resource/1?b=1&a=2", nil)
	if err != nil {
		t.Error(err)
	}

	parameters := Parameters{
		Timestamp: 1353832234,
		Ext:       "some-app-ext-data",
		Nonce:     "j4h3g2",
		Hash:      nil,
		// TODO: Rest is not important for this test
	}

	credentials := Credentials{
		KeyIdentifier: "dh37fgj492je",
		Key:           []byte("werxhqb98rpaxn39848xrunpaw3489ruxnpa98w4rxn"),
		Algorithm:     "sha256",
	}

	mac, err := calculateRequestSignature(r, parameters, credentials)
	if err != nil {
		t.Error(err)
	}

	expectedMac, _ := base64.StdEncoding.DecodeString("6R4rV5iE+NPoym+WwjeHzjAGXUtLNIxmo1vpMofpLAE=")
	if !bytes.Equal(mac, expectedMac) {
		t.Error("Mac mismatch")
	}
}

func Test_calculateRequestSignatureWithPOST(t *testing.T) {
	r, err := http.NewRequest("POST", "http://example.com:8000/resource/1?b=1&a=2", strings.NewReader("Thank you for flying Hawk"))
	if err != nil {
		t.Error(err)
	}
	r.Header.Add("Content-Type", "text/plain")

	payloadHash, _ := base64.StdEncoding.DecodeString("Yi9LfIIFRtBEPt74PVmbTF/xVAwPn7ub15ePICfgnuY=")

	parameters := Parameters{
		Timestamp: 1353832234,
		Ext:       "some-app-ext-data",
		Nonce:     "j4h3g2",
		Hash:      payloadHash,
		// TODO: Rest is not important for this test
	}

	credentials := Credentials{
		KeyIdentifier: "dh37fgj492je",
		Key:           []byte("werxhqb98rpaxn39848xrunpaw3489ruxnpa98w4rxn"),
		Algorithm:     "sha256",
	}

	mac, err := calculateRequestSignature(r, parameters, credentials)
	if err != nil {
		t.Error(err)
	}

	expectedMac, _ := base64.StdEncoding.DecodeString("aSe1DERmZuRl3pI36/9BdZmnErTw3sNzOOAUlfeKjVw=")
	if !bytes.Equal(mac, expectedMac) {
		t.Error("Mac mismatch")
	}
}
<|endoftext|>


<commit_before>/*
	This file supports the DVID REST API, breaking down URLs into
	commands and massaging attached data into appropriate data types.
*/

package server

import (
	"encoding/json"
	"fmt"
	"net/http"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/janelia-flyem/dvid/datastore"
	"github.com/janelia-flyem/dvid/dvid"
	"github.com/janelia-flyem/dvid/storage"
)

const webClientUnavailableMessage = `
DVID Web Client Unavailable!  To make the web client available, you have two choices:

1) Invoke the DVID server using the full path to the DVID executable to use
   the built-in web client.

2) Specify a path to web pages that implement a web client via the "-webclient=PATH"
   option to dvid.  Example:
   % dvid -webclient=/path/to/html/files -datastore=/path/to/db serve
`

const WebAPIHelp = `
DVID's HTTP API is a Level 2 REST API that roughly translates the dvid terminal
commands into URL form.

Commands that set or create data use POST.  Commands that return data use GET,
and the returned format will be in JSON except for "help" which returns HTML.

    GET /api/help
    GET /api/load

    GET /api/server/info
    GET /api/server/types

    GET /api/datasets/info
    GET /api/datasets/list
    POST /api/datasets/new  (Returns JSON like {"Root": "My Root UUID"})

    GET /api/dataset/<UUID>/info
    POST /api/dataset/<UUID>/new/<datatype name>/<data name>
        Type-specific configuration settings should be sent via JSON.

    GET /api/dataset/<UUID>/<data name>/<type-specific commands>

    POST /api/node/<UUID>/lock
    POST /api/node/<UUID>/branch

    GET /api/node/<UUID>/<data name>/<type-specific commands>
    POST /api/node/<UUID>/<data name>/<type-specific commands>

To examine the data type-specific API commands available, use GET /api/dataset/.../help
shown above.
`

func BadRequest(w http.ResponseWriter, r *http.Request, message string) {
	errorMsg := fmt.Sprintf("ERROR using REST API: %s (%s).", message, r.URL.Path)
	errorMsg += "  Use 'dvid help' to get proper API request format.\n"
	dvid.Error(errorMsg)
	http.Error(w, errorMsg, http.StatusBadRequest)
}

// Index file redirection.
func indexHandler(w http.ResponseWriter, r *http.Request) {
	http.Redirect(w, r, "/index.html", http.StatusMovedPermanently)
}

// Handler for web client
func mainHandler(w http.ResponseWriter, r *http.Request) {
	if runningService.WebClientPath != "" {
		consoleFile := strings.TrimPrefix(r.URL.Path, "/console/")
		filename := filepath.Join(runningService.WebClientPath, consoleFile)
		dvid.Fmt(dvid.Debug, "Web client: %s -> %s\n", r.URL.Path, filename)
		http.ServeFile(w, r, filename)
	} else {
		fmt.Fprintf(w, webClientUnavailableMessage)
	}
}

// Handler for API commands.  Results come back in JSON.
// We assume all DVID API commands have URLs with prefix /api/...
// See WebAPIHelp for expected calling URLs and HTTP verbs.
func apiHandler(w http.ResponseWriter, r *http.Request) {
	// Break URL request into arguments
	lenPath := len(WebAPIPath)
	url := r.URL.Path[lenPath:]
	parts := strings.Split(url, "/")
	if len(parts) == 0 {
		BadRequest(w, r, "Poorly formed request")
		return
	}

	// Handle the requests
	switch parts[0] {
	case "help":
		helpRequest(w, r)
	case "load":
		loadRequest(w, r)
	case "server":
		serverRequest(w, r)
	case "datasets":
		datasetsRequest(w, r)
	case "dataset":
		datasetRequest(w, r)
	case "node":
		nodeRequest(w, r)
	default:
		BadRequest(w, r, "Request not in API")
	}
}

func helpRequest(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")
	fmt.Fprintf(w, WebAPIHelp)
}

func loadRequest(w http.ResponseWriter, r *http.Request) {
	m, err := json.Marshal(map[string]int{
		"bytes read":      storage.BytesReadPerSec,
		"bytes written":   storage.BytesWrittenPerSec,
		"GET requests":    storage.GetsPerSec,
		"PUT requests":    storage.PutsPerSec,
		"handlers active": int(100 * ActiveHandlers / MaxChunkHandlers),
	})
	if err != nil {
		BadRequest(w, r, err.Error())
		return
	}
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, string(m))
}

func aboutJSON() (jsonStr string, err error) {
	data := map[string]string{
		"Cores":           fmt.Sprintf("%d", dvid.NumCPU),
		"Maximum Cores":   fmt.Sprintf("%d", runtime.NumCPU()),
		"DVID datastore":  datastore.Version,
		"Storage backend": storage.Version,
	}
	m, err := json.Marshal(data)
	if err != nil {
		return
	}
	jsonStr = string(m)
	return
}

func serverRequest(w http.ResponseWriter, r *http.Request) {
	lenPath := len(WebAPIPath + "server/")
	url := r.URL.Path[lenPath:]
	parts := strings.Split(url, "/")

	badRequest := func() {
		BadRequest(w, r, WebAPIPath+"server/ must be followed with 'info' or 'types'")
	}

	if len(parts) != 1 {
		badRequest()
		return
	}

	switch parts[0] {
	case "info":
		jsonStr, err := aboutJSON()
		if err != nil {
			BadRequest(w, r, err.Error())
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, jsonStr)
	case "types":
		jsonStr, err := runningService.TypesJSON()
		if err != nil {
			BadRequest(w, r, err.Error())
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, jsonStr)
	default:
		badRequest()
	}
}

func datasetsRequest(w http.ResponseWriter, r *http.Request) {
	lenPath := len(WebAPIPath + "datasets/")
	url := r.URL.Path[lenPath:]
	parts := strings.Split(url, "/")
	action := strings.ToLower(r.Method)

	badRequest := func() {
		BadRequest(w, r, WebAPIPath+"datasets/ must be followed with 'info', 'list' or 'new'")
	}

	if len(parts) != 1 {
		badRequest()
		return
	}

	switch parts[0] {
	case "list":
		jsonStr, err := runningService.DatasetsListJSON()
		if err != nil {
			BadRequest(w, r, err.Error())
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, jsonStr)
	case "info":
		jsonStr, err := runningService.DatasetsAllJSON()
		if err != nil {
			BadRequest(w, r, err.Error())
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, jsonStr)
	case "new":
		if action != "post" {
			BadRequest(w, r, "Datasets 'new' request must be made with HTTP POST method")
			return
		}
		root, _, err := runningService.NewDataset()
		if err != nil {
			BadRequest(w, r, err.Error())
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, "{%q: %q}", "Root", root)
	default:
		badRequest()
	}
}

func datasetRequest(w http.ResponseWriter, r *http.Request) {
	lenPath := len(WebAPIPath + "dataset/")
	url := r.URL.Path[lenPath:]
	parts := strings.Split(url, "/")
	action := strings.ToLower(r.Method)

	if len(parts) < 2 || len(parts) > 4 {
		BadRequest(w, r, "Bad dataset request made.  Visit /api/help for help.")
		return
	}

	// Get particular dataset for this UUID
	uuid, err := MatchingUUID(parts[0])
	if err != nil {
		BadRequest(w, r, err.Error())
		return
	}

	// Handle query of dataset properties
	if parts[1] == "info" {
		jsonStr, err := runningService.DatasetJSON(uuid)
		if err != nil {
			BadRequest(w, r, err.Error())
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, jsonStr)
		return
	}

	// Handle creation of new data in dataset via POST.
	if parts[1] == "new" {
		if action != "post" {
			BadRequest(w, r, "Dataset 'new' request must be made with HTTP POST method")
			return
		}
		if len(parts) != 4 {
			BadRequest(w, r, "Bad URL: Expecting /api/dataset/<UUID>/new/<datatype name>/<data name>")
			return
		}
		typename := parts[2]
		dataname := parts[3]
		decoder := json.NewDecoder(r.Body)
		var config dvid.Config
		err = decoder.Decode(&config)
		if err != nil {
			BadRequest(w, r, fmt.Sprintf("Error decoding POSTed JSON config for 'new': %s", err.Error()))
			return
		}
		err = runningService.NewData(uuid, typename, dataname, config)
		if err != nil {
			BadRequest(w, r, err.Error())
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, "{%q: 'Added %s [%s] to node %s'}", "result", dataname, typename, uuid)
		return
	}

	// Forward all other commands to the data service.
	dataname := datastore.DataString(parts[1])
	dataservice, err := runningService.DataService(uuid, dataname)
	if err != nil {
		BadRequest(w, r, err.Error())
		return
	}
	err = dataservice.DoHTTP(uuid, w, r)
	if err != nil {
		BadRequest(w, r, err.Error())
	}
}

func nodeRequest(w http.ResponseWriter, r *http.Request) {
	lenPath := len(WebAPIPath + "node/")
	url := r.URL.Path[lenPath:]
	parts := strings.Split(url, "/")

	if len(parts) < 2 {
		BadRequest(w, r, "Bad node request made.  Visit /api/help for help.")
		return
	}

	// Get particular dataset for this UUID
	uuid, err := MatchingUUID(parts[0])
	if err != nil {
		BadRequest(w, r, err.Error())
		return
	}

	// Handle the dataset command.
	switch parts[1] {
	case "lock":
		err := runningService.Lock(uuid)
		if err != nil {
			BadRequest(w, r, err.Error())
		} else {
			w.Header().Set("Content-Type", "text/plain")
			fmt.Fprintln(w, "Lock on node %s successful.", uuid)
		}

	case "branch":
		newuuid, err := runningService.NewVersion(uuid)
		if err != nil {
			BadRequest(w, r, err.Error())
		} else {
			w.Header().Set("Content-Type", "application/json")
			fmt.Fprintf(w, "{%q: %q}", "Branch", newuuid)
		}

	default:
		dataname := datastore.DataString(parts[1])
		dataservice, err := runningService.DataService(uuid, dataname)
		if err != nil {
			BadRequest(w, r, err.Error())
			return
		}
		err = dataservice.DoHTTP(uuid, w, r)
		if err != nil {
			BadRequest(w, r, err.Error())
		}
	}
}
<commit_msg>Add # active goroutines to load response<commit_after>/*
	This file supports the DVID REST API, breaking down URLs into
	commands and massaging attached data into appropriate data types.
*/

package server

import (
	"encoding/json"
	"fmt"
	"net/http"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/janelia-flyem/dvid/datastore"
	"github.com/janelia-flyem/dvid/dvid"
	"github.com/janelia-flyem/dvid/storage"
)

const webClientUnavailableMessage = `
DVID Web Client Unavailable!  To make the web client available, you have two choices:

1) Invoke the DVID server using the full path to the DVID executable to use
   the built-in web client.

2) Specify a path to web pages that implement a web client via the "-webclient=PATH"
   option to dvid.  Example:
   % dvid -webclient=/path/to/html/files -datastore=/path/to/db serve
`

const WebAPIHelp = `
DVID's HTTP API is a Level 2 REST API that roughly translates the dvid terminal
commands into URL form.

Commands that set or create data use POST.  Commands that return data use GET,
and the returned format will be in JSON except for "help" which returns HTML.

    GET /api/help
    GET /api/load

    GET /api/server/info
    GET /api/server/types

    GET /api/datasets/info
    GET /api/datasets/list
    POST /api/datasets/new  (Returns JSON like {"Root": "My Root UUID"})

    GET /api/dataset/<UUID>/info
    POST /api/dataset/<UUID>/new/<datatype name>/<data name>
        Type-specific configuration settings should be sent via JSON.

    GET /api/dataset/<UUID>/<data name>/<type-specific commands>

    POST /api/node/<UUID>/lock
    POST /api/node/<UUID>/branch

    GET /api/node/<UUID>/<data name>/<type-specific commands>
    POST /api/node/<UUID>/<data name>/<type-specific commands>

To examine the data type-specific API commands available, use GET /api/dataset/.../help
shown above.
`

func BadRequest(w http.ResponseWriter, r *http.Request, message string) {
	errorMsg := fmt.Sprintf("ERROR using REST API: %s (%s).", message, r.URL.Path)
	errorMsg += "  Use 'dvid help' to get proper API request format.\n"
	dvid.Error(errorMsg)
	http.Error(w, errorMsg, http.StatusBadRequest)
}

// Index file redirection.
func indexHandler(w http.ResponseWriter, r *http.Request) {
	http.Redirect(w, r, "/index.html", http.StatusMovedPermanently)
}

// Handler for web client
func mainHandler(w http.ResponseWriter, r *http.Request) {
	if runningService.WebClientPath != "" {
		consoleFile := strings.TrimPrefix(r.URL.Path, "/console/")
		filename := filepath.Join(runningService.WebClientPath, consoleFile)
		dvid.Fmt(dvid.Debug, "Web client: %s -> %s\n", r.URL.Path, filename)
		http.ServeFile(w, r, filename)
	} else {
		fmt.Fprintf(w, webClientUnavailableMessage)
	}
}

// Handler for API commands.  Results come back in JSON.
// We assume all DVID API commands have URLs with prefix /api/...
// See WebAPIHelp for expected calling URLs and HTTP verbs.
func apiHandler(w http.ResponseWriter, r *http.Request) {
	// Break URL request into arguments
	lenPath := len(WebAPIPath)
	url := r.URL.Path[lenPath:]
	parts := strings.Split(url, "/")
	if len(parts) == 0 {
		BadRequest(w, r, "Poorly formed request")
		return
	}

	// Handle the requests
	switch parts[0] {
	case "help":
		helpRequest(w, r)
	case "load":
		loadRequest(w, r)
	case "server":
		serverRequest(w, r)
	case "datasets":
		datasetsRequest(w, r)
	case "dataset":
		datasetRequest(w, r)
	case "node":
		nodeRequest(w, r)
	default:
		BadRequest(w, r, "Request not in API")
	}
}

func helpRequest(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")
	fmt.Fprintf(w, WebAPIHelp)
}

func loadRequest(w http.ResponseWriter, r *http.Request) {
	m, err := json.Marshal(map[string]int{
		"bytes read":      storage.BytesReadPerSec,
		"bytes written":   storage.BytesWrittenPerSec,
		"GET requests":    storage.GetsPerSec,
		"PUT requests":    storage.PutsPerSec,
		"handlers active": int(100 * ActiveHandlers / MaxChunkHandlers),
		"goroutines":      runtime.NumGoroutine(),
	})
	if err != nil {
		BadRequest(w, r, err.Error())
		return
	}
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, string(m))
}

func aboutJSON() (jsonStr string, err error) {
	data := map[string]string{
		"Cores":           fmt.Sprintf("%d", dvid.NumCPU),
		"Maximum Cores":   fmt.Sprintf("%d", runtime.NumCPU()),
		"DVID datastore":  datastore.Version,
		"Storage backend": storage.Version,
	}
	m, err := json.Marshal(data)
	if err != nil {
		return
	}
	jsonStr = string(m)
	return
}

func serverRequest(w http.ResponseWriter, r *http.Request) {
	lenPath := len(WebAPIPath + "server/")
	url := r.URL.Path[lenPath:]
	parts := strings.Split(url, "/")

	badRequest := func() {
		BadRequest(w, r, WebAPIPath+"server/ must be followed with 'info' or 'types'")
	}

	if len(parts) != 1 {
		badRequest()
		return
	}

	switch parts[0] {
	case "info":
		jsonStr, err := aboutJSON()
		if err != nil {
			BadRequest(w, r, err.Error())
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, jsonStr)
	case "types":
		jsonStr, err := runningService.TypesJSON()
		if err != nil {
			BadRequest(w, r, err.Error())
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, jsonStr)
	default:
		badRequest()
	}
}

func datasetsRequest(w http.ResponseWriter, r *http.Request) {
	lenPath := len(WebAPIPath + "datasets/")
	url := r.URL.Path[lenPath:]
	parts := strings.Split(url, "/")
	action := strings.ToLower(r.Method)

	badRequest := func() {
		BadRequest(w, r, WebAPIPath+"datasets/ must be followed with 'info', 'list' or 'new'")
	}

	if len(parts) != 1 {
		badRequest()
		return
	}

	switch parts[0] {
	case "list":
		jsonStr, err := runningService.DatasetsListJSON()
		if err != nil {
			BadRequest(w, r, err.Error())
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, jsonStr)
	case "info":
		jsonStr, err := runningService.DatasetsAllJSON()
		if err != nil {
			BadRequest(w, r, err.Error())
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, jsonStr)
	case "new":
		if action != "post" {
			BadRequest(w, r, "Datasets 'new' request must be made with HTTP POST method")
			return
		}
		root, _, err := runningService.NewDataset()
		if err != nil {
			BadRequest(w, r, err.Error())
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, "{%q: %q}", "Root", root)
	default:
		badRequest()
	}
}

func datasetRequest(w http.ResponseWriter, r *http.Request) {
	lenPath := len(WebAPIPath + "dataset/")
	url := r.URL.Path[lenPath:]
	parts := strings.Split(url, "/")
	action := strings.ToLower(r.Method)

	if len(parts) < 2 || len(parts) > 4 {
		BadRequest(w, r, "Bad dataset request made.  Visit /api/help for help.")
		return
	}

	// Get particular dataset for this UUID
	uuid, err := MatchingUUID(parts[0])
	if err != nil {
		BadRequest(w, r, err.Error())
		return
	}

	// Handle query of dataset properties
	if parts[1] == "info" {
		jsonStr, err := runningService.DatasetJSON(uuid)
		if err != nil {
			BadRequest(w, r, err.Error())
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, jsonStr)
		return
	}

	// Handle creation of new data in dataset via POST.
	if parts[1] == "new" {
		if action != "post" {
			BadRequest(w, r, "Dataset 'new' request must be made with HTTP POST method")
			return
		}
		if len(parts) != 4 {
			BadRequest(w, r, "Bad URL: Expecting /api/dataset/<UUID>/new/<datatype name>/<data name>")
			return
		}
		typename := parts[2]
		dataname := parts[3]
		decoder := json.NewDecoder(r.Body)
		var config dvid.Config
		err = decoder.Decode(&config)
		if err != nil {
			BadRequest(w, r, fmt.Sprintf("Error decoding POSTed JSON config for 'new': %s", err.Error()))
			return
		}
		err = runningService.NewData(uuid, typename, dataname, config)
		if err != nil {
			BadRequest(w, r, err.Error())
			return
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprintf(w, "{%q: 'Added %s [%s] to node %s'}", "result", dataname, typename, uuid)
		return
	}

	// Forward all other commands to the data service.
	dataname := datastore.DataString(parts[1])
	dataservice, err := runningService.DataService(uuid, dataname)
	if err != nil {
		BadRequest(w, r, err.Error())
		return
	}
	err = dataservice.DoHTTP(uuid, w, r)
	if err != nil {
		BadRequest(w, r, err.Error())
	}
}

func nodeRequest(w http.ResponseWriter, r *http.Request) {
	lenPath := len(WebAPIPath + "node/")
	url := r.URL.Path[lenPath:]
	parts := strings.Split(url, "/")

	if len(parts) < 2 {
		BadRequest(w, r, "Bad node request made.  Visit /api/help for help.")
		return
	}

	// Get particular dataset for this UUID
	uuid, err := MatchingUUID(parts[0])
	if err != nil {
		BadRequest(w, r, err.Error())
		return
	}

	// Handle the dataset command.
	switch parts[1] {
	case "lock":
		err := runningService.Lock(uuid)
		if err != nil {
			BadRequest(w, r, err.Error())
		} else {
			w.Header().Set("Content-Type", "text/plain")
			fmt.Fprintln(w, "Lock on node %s successful.", uuid)
		}

	case "branch":
		newuuid, err := runningService.NewVersion(uuid)
		if err != nil {
			BadRequest(w, r, err.Error())
		} else {
			w.Header().Set("Content-Type", "application/json")
			fmt.Fprintf(w, "{%q: %q}", "Branch", newuuid)
		}

	default:
		dataname := datastore.DataString(parts[1])
		dataservice, err := runningService.DataService(uuid, dataname)
		if err != nil {
			BadRequest(w, r, err.Error())
			return
		}
		err = dataservice.DoHTTP(uuid, w, r)
		if err != nil {
			BadRequest(w, r, err.Error())
		}
	}
}
<|endoftext|>


<commit_before>package v2

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"testing"

	. "github.com/onsi/gomega"
)

type HoverflyMiddlewareStub struct {
	Binary     string
	Script     string
	Middleware string
	Remote     string
}

func (this HoverflyMiddlewareStub) GetMiddleware() (string, string, string) {
	return this.Binary, this.Script, this.Remote
}

func (this *HoverflyMiddlewareStub) SetMiddleware(binary, script, remote string) error {
	this.Binary = binary
	this.Script = script
	this.Remote = remote
	if script == "error" {
		return fmt.Errorf("error")
	}

	return nil
}

func TestHoverflyMiddlewareHandlerGetReturnsTheCorrectMiddleware(t *testing.T) {
	RegisterTestingT(t)

	stubHoverfly := &HoverflyMiddlewareStub{
		Binary: "test",
		Script: "middleware",
		Remote: "remote",
	}

	unit := HoverflyMiddlewareHandler{Hoverfly: stubHoverfly}

	request, err := http.NewRequest("GET", "", nil)
	Expect(err).To(BeNil())

	response := makeRequestOnHandler(unit.Get, request)

	Expect(response.Code).To(Equal(http.StatusOK))

	middlewareView, err := unmarshalMiddlewareView(response.Body)
	Expect(err).To(BeNil())
	Expect(middlewareView.Binary).To(Equal("test"))
	Expect(middlewareView.Script).To(Equal("middleware"))
	Expect(middlewareView.Remote).To(Equal("remote"))
}

func TestHoverflyMiddlewareHandlerPutSetsTheNewMiddlewarendReplacesTheTestMiddleware(t *testing.T) {
	RegisterTestingT(t)

	stubHoverfly := &HoverflyMiddlewareStub{Binary: "test"}
	unit := HoverflyMiddlewareHandler{Hoverfly: stubHoverfly}

	middlewareView := &MiddlewareView{Binary: "python", Script: "new-middleware"}

	bodyBytes, err := json.Marshal(middlewareView)
	Expect(err).To(BeNil())

	request, err := http.NewRequest("PUT", "", ioutil.NopCloser(bytes.NewBuffer(bodyBytes)))
	Expect(err).To(BeNil())

	response := makeRequestOnHandler(unit.Put, request)
	Expect(response.Code).To(Equal(http.StatusOK))
	Expect(stubHoverfly.Binary).To(Equal("python"))
	Expect(stubHoverfly.Script).To(Equal("new-middleware"))

	middlewareViewResponse, err := unmarshalMiddlewareView(response.Body)
	Expect(err).To(BeNil())

	Expect(middlewareViewResponse.Binary).To(Equal("python"))
	Expect(middlewareViewResponse.Script).To(Equal("new-middleware"))
}

func TestHoverflyMiddlewareHandlerPutWill422ErrorIfHoverflyErrors(t *testing.T) {
	RegisterTestingT(t)

	var stubHoverfly HoverflyMiddlewareStub
	unit := HoverflyMiddlewareHandler{Hoverfly: &stubHoverfly}

	middlewareView := &MiddlewareView{Script: "error"}

	bodyBytes, err := json.Marshal(middlewareView)
	Expect(err).To(BeNil())

	request, err := http.NewRequest("PUT", "", ioutil.NopCloser(bytes.NewBuffer(bodyBytes)))
	Expect(err).To(BeNil())

	response := makeRequestOnHandler(unit.Put, request)
	Expect(response.Code).To(Equal(http.StatusUnprocessableEntity))

	errorViewResponse, err := unmarshalErrorView(response.Body)
	Expect(err).To(BeNil())

	Expect(errorViewResponse.Error).To(Equal("Invalid middleware: error"))
}

func TestHoverflyMiddlewareHandlerPutWill400ErrorIfJsonIsBad(t *testing.T) {
	RegisterTestingT(t)

	var stubHoverfly HoverflyMiddlewareStub
	unit := HoverflyMiddlewareHandler{Hoverfly: &stubHoverfly}

	bodyBytes := []byte("{{}{}}")

	request, err := http.NewRequest("PUT", "/api/v2/hoverfly/mode", ioutil.NopCloser(bytes.NewBuffer(bodyBytes)))
	Expect(err).To(BeNil())

	response := makeRequestOnHandler(unit.Put, request)
	Expect(response.Code).To(Equal(http.StatusBadRequest))

	errorViewResponse, err := unmarshalErrorView(response.Body)
	Expect(err).To(BeNil())

	Expect(errorViewResponse.Error).To(Equal("Malformed JSON"))
}

func unmarshalMiddlewareView(buffer *bytes.Buffer) (MiddlewareView, error) {
	body, err := ioutil.ReadAll(buffer)
	if err != nil {
		return MiddlewareView{}, err
	}

	var middlewareView MiddlewareView

	err = json.Unmarshal(body, &middlewareView)
	if err != nil {
		return MiddlewareView{}, err
	}

	return middlewareView, nil
}
<commit_msg>Removing unused field from test stub<commit_after>package v2

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"testing"

	. "github.com/onsi/gomega"
)

type HoverflyMiddlewareStub struct {
	Binary string
	Script string
	Remote string
}

func (this HoverflyMiddlewareStub) GetMiddleware() (string, string, string) {
	return this.Binary, this.Script, this.Remote
}

func (this *HoverflyMiddlewareStub) SetMiddleware(binary, script, remote string) error {
	this.Binary = binary
	this.Script = script
	this.Remote = remote
	if script == "error" {
		return fmt.Errorf("error")
	}

	return nil
}

func TestHoverflyMiddlewareHandlerGetReturnsTheCorrectMiddleware(t *testing.T) {
	RegisterTestingT(t)

	stubHoverfly := &HoverflyMiddlewareStub{
		Binary: "test",
		Script: "middleware",
		Remote: "remote",
	}

	unit := HoverflyMiddlewareHandler{Hoverfly: stubHoverfly}

	request, err := http.NewRequest("GET", "", nil)
	Expect(err).To(BeNil())

	response := makeRequestOnHandler(unit.Get, request)

	Expect(response.Code).To(Equal(http.StatusOK))

	middlewareView, err := unmarshalMiddlewareView(response.Body)
	Expect(err).To(BeNil())
	Expect(middlewareView.Binary).To(Equal("test"))
	Expect(middlewareView.Script).To(Equal("middleware"))
	Expect(middlewareView.Remote).To(Equal("remote"))
}

func TestHoverflyMiddlewareHandlerPutSetsTheNewMiddlewarendReplacesTheTestMiddleware(t *testing.T) {
	RegisterTestingT(t)

	stubHoverfly := &HoverflyMiddlewareStub{Binary: "test"}
	unit := HoverflyMiddlewareHandler{Hoverfly: stubHoverfly}

	middlewareView := &MiddlewareView{Binary: "python", Script: "new-middleware"}

	bodyBytes, err := json.Marshal(middlewareView)
	Expect(err).To(BeNil())

	request, err := http.NewRequest("PUT", "", ioutil.NopCloser(bytes.NewBuffer(bodyBytes)))
	Expect(err).To(BeNil())

	response := makeRequestOnHandler(unit.Put, request)
	Expect(response.Code).To(Equal(http.StatusOK))
	Expect(stubHoverfly.Binary).To(Equal("python"))
	Expect(stubHoverfly.Script).To(Equal("new-middleware"))

	middlewareViewResponse, err := unmarshalMiddlewareView(response.Body)
	Expect(err).To(BeNil())

	Expect(middlewareViewResponse.Binary).To(Equal("python"))
	Expect(middlewareViewResponse.Script).To(Equal("new-middleware"))
}

func TestHoverflyMiddlewareHandlerPutWill422ErrorIfHoverflyErrors(t *testing.T) {
	RegisterTestingT(t)

	var stubHoverfly HoverflyMiddlewareStub
	unit := HoverflyMiddlewareHandler{Hoverfly: &stubHoverfly}

	middlewareView := &MiddlewareView{Script: "error"}

	bodyBytes, err := json.Marshal(middlewareView)
	Expect(err).To(BeNil())

	request, err := http.NewRequest("PUT", "", ioutil.NopCloser(bytes.NewBuffer(bodyBytes)))
	Expect(err).To(BeNil())

	response := makeRequestOnHandler(unit.Put, request)
	Expect(response.Code).To(Equal(http.StatusUnprocessableEntity))

	errorViewResponse, err := unmarshalErrorView(response.Body)
	Expect(err).To(BeNil())

	Expect(errorViewResponse.Error).To(Equal("Invalid middleware: error"))
}

func TestHoverflyMiddlewareHandlerPutWill400ErrorIfJsonIsBad(t *testing.T) {
	RegisterTestingT(t)

	var stubHoverfly HoverflyMiddlewareStub
	unit := HoverflyMiddlewareHandler{Hoverfly: &stubHoverfly}

	bodyBytes := []byte("{{}{}}")

	request, err := http.NewRequest("PUT", "/api/v2/hoverfly/mode", ioutil.NopCloser(bytes.NewBuffer(bodyBytes)))
	Expect(err).To(BeNil())

	response := makeRequestOnHandler(unit.Put, request)
	Expect(response.Code).To(Equal(http.StatusBadRequest))

	errorViewResponse, err := unmarshalErrorView(response.Body)
	Expect(err).To(BeNil())

	Expect(errorViewResponse.Error).To(Equal("Malformed JSON"))
}

func unmarshalMiddlewareView(buffer *bytes.Buffer) (MiddlewareView, error) {
	body, err := ioutil.ReadAll(buffer)
	if err != nil {
		return MiddlewareView{}, err
	}

	var middlewareView MiddlewareView

	err = json.Unmarshal(body, &middlewareView)
	if err != nil {
		return MiddlewareView{}, err
	}

	return middlewareView, nil
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// TODO(wallyworld) bug http://pad.lv/1408459
// Re-enable tests for i386 when these tests are fixed to work on that architecture.

// +build !386

package apiserver_test

import (
	"time"

	"github.com/juju/loggo"
	gitjujutesting "github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/api"
	"github.com/juju/juju/apiserver"
	"github.com/juju/juju/juju/testing"
	"github.com/juju/juju/rpc"
	coretesting "github.com/juju/juju/testing"
)

type pingerSuite struct {
	testing.JujuConnSuite
}

var _ = gc.Suite(&pingerSuite{})

var testPingPeriod = 100 * time.Millisecond

func (s *pingerSuite) TestConnectionBrokenDetection(c *gc.C) {
	s.PatchValue(&api.PingPeriod, testPingPeriod)

	st, _ := s.OpenAPIAsNewMachine(c)

	// Connection still alive
	select {
	case <-time.After(testPingPeriod):
	case <-st.Broken():
		c.Fatalf("connection should be alive still")
	}

	// Close the connection and see if we detect this
	go st.Close()

	// Check it's detected
	select {
	case <-time.After(testPingPeriod + time.Second):
		c.Fatalf("connection not closed as expected")
	case <-st.Broken():
		return
	}
}

func (s *pingerSuite) TestPing(c *gc.C) {
	tw := &loggo.TestWriter{}
	c.Assert(loggo.RegisterWriter("ping-tester", tw, loggo.DEBUG), gc.IsNil)
	defer loggo.RemoveWriter("ping-tester")

	st, _ := s.OpenAPIAsNewMachine(c)
	err := st.Ping()
	c.Assert(err, jc.ErrorIsNil)
	err = st.Close()
	c.Assert(err, jc.ErrorIsNil)
	err = st.Ping()
	c.Assert(err, gc.Equals, rpc.ErrShutdown)

	// Make sure that ping messages have not been logged.
	for _, m := range tw.Log() {
		c.Logf("checking %q", m.Message)
		c.Check(m.Message, gc.Not(gc.Matches), `.*"Request":"Ping".*`)
	}
}

func (s *pingerSuite) TestClientNoNeedToPing(c *gc.C) {
	s.PatchValue(apiserver.MaxClientPingInterval, time.Duration(0))
	st, err := api.Open(s.APIInfo(c), api.DefaultDialOpts())
	c.Assert(err, jc.ErrorIsNil)
	defer st.Close()
	time.Sleep(coretesting.ShortWait)
	err = st.Ping()
	c.Assert(err, jc.ErrorIsNil)
}

func (s *pingerSuite) TestAgentConnectionShutsDownWithNoPing(c *gc.C) {
	s.PatchValue(apiserver.MaxClientPingInterval, time.Duration(0))
	st, _ := s.OpenAPIAsNewMachine(c)
	time.Sleep(coretesting.ShortWait)
	err := st.Ping()
	c.Assert(err, gc.ErrorMatches, "connection is shut down")
}

func (s *pingerSuite) TestAgentConnectionDelaysShutdownWithPing(c *gc.C) {
	const shortTimeout = 100 * time.Millisecond
	const attemptDelay = shortTimeout / 4
	s.PatchValue(apiserver.MaxClientPingInterval, time.Duration(shortTimeout))

	st, _ := s.OpenAPIAsNewMachine(c)
	err := st.Ping()
	c.Assert(err, jc.ErrorIsNil)

	// As long as we don't wait too long, the connection stays open
	attempt := utils.AttemptStrategy{
		Min:   10,
		Delay: attemptDelay,
	}
	testStart := time.Now()
	c.Logf(
		"pinging %d times with %v delay, starting at %v",
		attempt.Min, attempt.Delay, testStart,
	)
	var lastLoop time.Time
	for a := attempt.Start(); a.Next(); {
		testNow := time.Now()
		loopDelta := testNow.Sub(lastLoop)
		if lastLoop.IsZero() {
			loopDelta = 0
		}
		c.Logf("duration since last iteration: %v", loopDelta)
		err = st.Ping()
		if !c.Check(
			err, jc.ErrorIsNil,
			gc.Commentf(
				"pinger timeout exceeded at %v (%v since the test start)",
				testNow, testNow.Sub(testNow),
			),
		) {
			c.Check(err, gc.ErrorMatches, "connection is shut down")
			return
		}
		lastLoop = time.Now()
	}

	// However, once we stop pinging for too long, the connection dies
	time.Sleep(2 * shortTimeout) // Exceed the timeout.
	err = st.Ping()
	c.Assert(err, gc.ErrorMatches, "connection is shut down")
}

type mongoPingerSuite struct {
	testing.JujuConnSuite
}

var _ = gc.Suite(&mongoPingerSuite{})

func (s *mongoPingerSuite) SetUpSuite(c *gc.C) {
	s.JujuConnSuite.SetUpSuite(c)
	// We need to set the ping interval before the server is started in test setup.
	restore := gitjujutesting.PatchValue(apiserver.MongoPingInterval, coretesting.ShortWait)
	s.AddSuiteCleanup(func(*gc.C) { restore() })
}

func (s *mongoPingerSuite) TestAgentConnectionsShutDownWhenStateDies(c *gc.C) {
	st, _ := s.OpenAPIAsNewMachine(c)
	err := st.Ping()
	c.Assert(err, jc.ErrorIsNil)
	gitjujutesting.MgoServer.Destroy()

	attempt := utils.AttemptStrategy{
		Total: coretesting.LongWait,
		Delay: coretesting.ShortWait,
	}
	for a := attempt.Start(); a.Next(); {
		if err := st.Ping(); err != nil {
			c.Assert(err, gc.ErrorMatches, "connection is shut down")
			return
		}
	}
	c.Fatalf("timed out waiting for API server to die")
}
<commit_msg>Fixed lp:1455627: Ping timeout varies based on machine load<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// TODO(wallyworld) bug http://pad.lv/1408459
// Re-enable tests for i386 when these tests are fixed to work on that architecture.

// +build !386

package apiserver_test

import (
	"time"

	"github.com/juju/loggo"
	gitjujutesting "github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/api"
	"github.com/juju/juju/apiserver"
	"github.com/juju/juju/juju/testing"
	"github.com/juju/juju/rpc"
	coretesting "github.com/juju/juju/testing"
)

type pingerSuite struct {
	testing.JujuConnSuite
}

var _ = gc.Suite(&pingerSuite{})

var testPingPeriod = 100 * time.Millisecond

func (s *pingerSuite) TestConnectionBrokenDetection(c *gc.C) {
	s.PatchValue(&api.PingPeriod, testPingPeriod)

	st, _ := s.OpenAPIAsNewMachine(c)

	// Connection still alive
	select {
	case <-time.After(testPingPeriod):
	case <-st.Broken():
		c.Fatalf("connection should be alive still")
	}

	// Close the connection and see if we detect this
	go st.Close()

	// Check it's detected
	select {
	case <-time.After(testPingPeriod + time.Second):
		c.Fatalf("connection not closed as expected")
	case <-st.Broken():
		return
	}
}

func (s *pingerSuite) TestPing(c *gc.C) {
	tw := &loggo.TestWriter{}
	c.Assert(loggo.RegisterWriter("ping-tester", tw, loggo.DEBUG), gc.IsNil)
	defer loggo.RemoveWriter("ping-tester")

	st, _ := s.OpenAPIAsNewMachine(c)
	err := st.Ping()
	c.Assert(err, jc.ErrorIsNil)
	err = st.Close()
	c.Assert(err, jc.ErrorIsNil)
	err = st.Ping()
	c.Assert(err, gc.Equals, rpc.ErrShutdown)

	// Make sure that ping messages have not been logged.
	for _, m := range tw.Log() {
		c.Logf("checking %q", m.Message)
		c.Check(m.Message, gc.Not(gc.Matches), `.*"Request":"Ping".*`)
	}
}

func (s *pingerSuite) TestClientNoNeedToPing(c *gc.C) {
	s.PatchValue(apiserver.MaxClientPingInterval, time.Duration(0))
	st, err := api.Open(s.APIInfo(c), api.DefaultDialOpts())
	c.Assert(err, jc.ErrorIsNil)
	defer st.Close()
	time.Sleep(coretesting.ShortWait)
	err = st.Ping()
	c.Assert(err, jc.ErrorIsNil)
}

func (s *pingerSuite) TestAgentConnectionShutsDownWithNoPing(c *gc.C) {
	s.PatchValue(apiserver.MaxClientPingInterval, time.Duration(0))
	st, _ := s.OpenAPIAsNewMachine(c)
	time.Sleep(coretesting.ShortWait)
	err := st.Ping()
	c.Assert(err, gc.ErrorMatches, "connection is shut down")
}

func (s *pingerSuite) calculatePingTimeout(c *gc.C) time.Duration {
	openStart := time.Now()
	st, _ := s.OpenAPIAsNewMachine(c)
	defer st.Close()

	err := st.Ping()
	c.Assert(err, jc.ErrorIsNil)
	openDelay := time.Since(openStart)
	pingTimeout := openDelay + openDelay/10
	c.Logf("API open and initial ping took %v; using ping timeout %v", openDelay, pingTimeout)
	return pingTimeout
}

func (s *pingerSuite) TestAgentConnectionDelaysShutdownWithPing(c *gc.C) {
	// To negate the effects of an underpowered or heavily loaded
	// machine running this test, tune the shortTimeout based on the
	// time it takes to open an API connection +10%.
	shortTimeout := s.calculatePingTimeout(c)
	attemptDelay := shortTimeout / 4

	s.PatchValue(apiserver.MaxClientPingInterval, time.Duration(shortTimeout))

	st, _ := s.OpenAPIAsNewMachine(c)
	err := st.Ping()
	c.Assert(err, jc.ErrorIsNil)

	// As long as we don't wait too long, the connection stays open
	attempt := utils.AttemptStrategy{
		Min:   10,
		Delay: attemptDelay,
	}
	testStart := time.Now()
	c.Logf(
		"pinging %d times with %v delay, starting at %v",
		attempt.Min, attempt.Delay, testStart,
	)
	var lastLoop time.Time
	for a := attempt.Start(); a.Next(); {
		testNow := time.Now()
		loopDelta := testNow.Sub(lastLoop)
		if lastLoop.IsZero() {
			loopDelta = 0
		}
		c.Logf("duration since last ping: %v", loopDelta)
		err = st.Ping()
		if !c.Check(
			err, jc.ErrorIsNil,
			gc.Commentf(
				"ping timeout exceeded at %v (%v since the test start)",
				testNow, testNow.Sub(testStart),
			),
		) {
			c.Check(err, gc.ErrorMatches, "connection is shut down")
			return
		}
		lastLoop = time.Now()
	}

	// However, once we stop pinging for too long, the connection dies
	time.Sleep(2 * shortTimeout) // Exceed the timeout.
	err = st.Ping()
	c.Assert(err, gc.ErrorMatches, "connection is shut down")
}

type mongoPingerSuite struct {
	testing.JujuConnSuite
}

var _ = gc.Suite(&mongoPingerSuite{})

func (s *mongoPingerSuite) SetUpSuite(c *gc.C) {
	s.JujuConnSuite.SetUpSuite(c)
	// We need to set the ping interval before the server is started in test setup.
	restore := gitjujutesting.PatchValue(apiserver.MongoPingInterval, coretesting.ShortWait)
	s.AddSuiteCleanup(func(*gc.C) { restore() })
}

func (s *mongoPingerSuite) TestAgentConnectionsShutDownWhenStateDies(c *gc.C) {
	st, _ := s.OpenAPIAsNewMachine(c)
	err := st.Ping()
	c.Assert(err, jc.ErrorIsNil)
	gitjujutesting.MgoServer.Destroy()

	attempt := utils.AttemptStrategy{
		Total: coretesting.LongWait,
		Delay: coretesting.ShortWait,
	}
	for a := attempt.Start(); a.Next(); {
		if err := st.Ping(); err != nil {
			c.Assert(err, gc.ErrorMatches, "connection is shut down")
			return
		}
	}
	c.Fatalf("timed out waiting for API server to die")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/bewuethr/advent-of-code/go/convert"
	"github.com/bewuethr/advent-of-code/go/ioutil"
	"github.com/bewuethr/advent-of-code/go/log"
)

func main() {
	scanner, err := ioutil.GetInputScanner()
	if err != nil {
		log.Die("getting scanner", err)
	}

	scanner.Scan()
	opCodesStr := strings.Split(scanner.Text(), ",")
	if err := scanner.Err(); err != nil {
		log.Die("reading input", err)
	}

	initOpCodes, err := convert.StrSliceToInt(opCodesStr)
	if err != nil {
		log.Die("converting string slice to int", err)
	}

	const target = 19690720

	for noun := 0; noun < 100; noun++ {
		for verb := 0; verb < 100; verb++ {
			opCodes := make([]int, len(initOpCodes))
			copy(opCodes, initOpCodes)
			opCodes[1], opCodes[2] = noun, verb

			opCodes, err = runProgram(opCodes)
			if err != nil {
				log.Die("running op codes", err)
			}

			if opCodes[0] == target {
				fmt.Println(100*noun + verb)
				os.Exit(0)
			}
		}
	}
}

func runProgram(codes []int) ([]int, error) {
	const (
		add  = 1
		mult = 2
		halt = 99
	)

	instrPtr := 0
	for {
		switch codes[instrPtr] {
		case halt:
			return codes, nil

		case add:
			codes[codes[instrPtr+3]] = codes[codes[instrPtr+1]] + codes[codes[instrPtr+2]]

		case mult:
			codes[codes[instrPtr+3]] = codes[codes[instrPtr+1]] * codes[codes[instrPtr+2]]

		default:
			return nil, fmt.Errorf("illegal opcode %d", codes[instrPtr])
		}

		instrPtr += 4
	}
}
<commit_msg>Use intcode package for day 2, second part<commit_after>package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/bewuethr/advent-of-code/go/convert"
	"github.com/bewuethr/advent-of-code/go/intcode"
	"github.com/bewuethr/advent-of-code/go/ioutil"
	"github.com/bewuethr/advent-of-code/go/log"
)

func main() {
	scanner, err := ioutil.GetInputScanner()
	if err != nil {
		log.Die("getting scanner", err)
	}

	scanner.Scan()
	opCodesStr := strings.Split(scanner.Text(), ",")
	if err := scanner.Err(); err != nil {
		log.Die("reading input", err)
	}

	initOpCodes, err := convert.StrSliceToInt(opCodesStr)
	if err != nil {
		log.Die("converting string slice to int", err)
	}

	const target = 19690720

	for noun := 0; noun < 100; noun++ {
		for verb := 0; verb < 100; verb++ {
			opCodes := make([]int, len(initOpCodes))
			copy(opCodes, initOpCodes)
			opCodes[1], opCodes[2] = noun, verb

			comp := intcode.NewComputer(opCodes)
			if err := comp.RunProgram(); err != nil {
				log.Die("running op codes", err)
			}

			if comp.Value(0) == target {
				fmt.Println(100*noun + verb)
				os.Exit(0)
			}
		}
	}
}
<|endoftext|>


<commit_before>package net

import (
	"bytes"
	"context"
	"encoding/json"
	"encoding/xml"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
)

// Net 请求结构体
type Net struct {
	client   *http.Client // 可重复使用的client
	baseURL  *url.URL     // 请求根地址
	isRelase bool         // 是否是生成环境
}

// SuperAgent 请求参数
type SuperAgent struct {
	net         *Net              // 当前请求包实例
	url         string            // 请求地址
	method      string            // 请求方式
	contentType string            // 请求类型
	body        interface{}       // 发送请求的body
	header      map[string]string // 头文件
}

const (
	contentJSON = "application/json;charset=utf-8"
	contentXML  = "application/xml;charset=utf-8"
	contentText = "text/plain;charset=utf-8"
)

// New 初始化一个请求包对象
func New() *Net {
	var release bool
	netMode := os.Getenv("NET_MODE")
	if len(netMode) == 0 {
		release = false
	} else if netMode == "release" {
		release = true
	}

	return &Net{
		client:   http.DefaultClient,
		isRelase: release,
	}
}

// GetClient 获取http client
func (n *Net) GetClient() *http.Client {
	return n.client
}

// NewWithClient 初始化一个请求包对象，自己传入Client
func NewWithClient(client *http.Client) *Net {
	return &Net{
		client: client,
	}
}

// Get 发送 Get 请求
func (n *Net) Get(url string) *SuperAgent {
	return &SuperAgent{net: n, url: url, method: "GET"}
}

// Post 发送 Post 请求
func (n *Net) Post(url string) *SuperAgent {
	return &SuperAgent{net: n, url: url, method: "POST"}
}

// Put 发送 Put 请求
func (n *Net) Put(url string) *SuperAgent {
	return &SuperAgent{net: n, url: url, method: "PUT"}
}

// Delete 发送 Delete 请求
func (n *Net) Delete(url string) *SuperAgent {
	return &SuperAgent{net: n, url: url, method: "DELETE"}
}

// JSON 设置请求数据内容，默认用 Content-Type=application/json; 方式发送json数据
func (s *SuperAgent) JSON(body interface{}) *SuperAgent {
	s.body = body
	s.contentType = contentJSON
	return s
}

// XML 设置请求数据内容，默认用 Content-Type=application/json; 方式发送json数据
func (s *SuperAgent) XML(body interface{}) *SuperAgent {
	s.body = body
	s.contentType = contentXML
	return s
}

// Text 设置请求数据内容，默认用 Content-Type=text/plain; 方式发送string数据
func (s *SuperAgent) Text(body string) *SuperAgent {
	s.body = body
	s.contentType = contentText
	return s
}

// Header 设置请求头内容
func (s *SuperAgent) Header(header map[string]string) *SuperAgent {
	s.header = header
	return s
}

// End 开始http请求
func (s *SuperAgent) End(ctx context.Context, v interface{}) (*http.Response, error) {
	if len(s.contentType) > 0 && s.body == nil {
		s.body = ""
	}
	var req *http.Request
	var err error
	buf := new(bytes.Buffer)
	switch s.contentType {
	case contentJSON:
		err = json.NewEncoder(buf).Encode(s.body)
	case contentXML:
		err = xml.NewEncoder(buf).Encode(s.body)
	case contentText:
		_, err = buf.WriteString(s.body.(string))
	}
	if err != nil {
		return nil, err
	}

	// 转换 url
	rel, err := url.Parse(s.url)
	if err != nil {
		return nil, err
	}
	u := s.net.baseURL.ResolveReference(rel)

	req, err = http.NewRequest(s.method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	if len(s.contentType) > 0 {
		req.Header.Set("Content-Type", s.contentType)
	}
	for key, value := range s.header {
		req.Header.Set(key, value)
	}

	if ctx != nil {
		req = req.WithContext(ctx)
	}

	// 执行网络请求
	resp, err := s.net.client.Do(req)
	if err != nil {

		if ctx != nil {
			// If we got an error, and the context has been canceled, the context's error is probably more useful.
			select {
			case <-ctx.Done():
				return nil, ctx.Err()
			default:
			}
		}

		// If the error type is *url.Error, sanitize its URL before returning.
		if e, ok := err.(*url.Error); ok {
			if url, err := url.Parse(e.URL); err == nil {
				e.URL = url.String()
				return nil, e
			}
		}
		return nil, err
	}

	defer func() {
		// Drain up to 512 bytes and close the body to let the Transport reuse the connection
		io.CopyN(ioutil.Discard, resp.Body, 512)
		resp.Body.Close()
	}()

	if v != nil {
		if w, ok := v.(io.Writer); ok {
			io.Copy(w, resp.Body)
		} else {
			body, err := ioutil.ReadAll(resp.Body)
			if !strings.Contains(string(body), "ip_list") {
				if s.net.isRelase {
					log.Printf("url %s body %s", req.URL.Path, string(body))
				}
			}

			// 默认认为 contentType 不为xml的情况下，所有返回都用json解析
			if strings.EqualFold(s.contentType, contentXML) {
				err = xml.Unmarshal(body, v)
			} else {
				err = json.Unmarshal(body, v)
			}

			if err == io.EOF {
				err = nil // ignore EOF errors caused by empty response body
			}
		}
	}

	return resp, err
}
<commit_msg>去除ip_list返回内容判断，修改log是否打印判断<commit_after>package net

import (
	"bytes"
	"context"
	"encoding/json"
	"encoding/xml"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
)

// Net 请求结构体
type Net struct {
	client   *http.Client // 可重复使用的client
	baseURL  *url.URL     // 请求根地址
	isRelase bool         // 是否是生成环境
}

// SuperAgent 请求参数
type SuperAgent struct {
	net         *Net              // 当前请求包实例
	url         string            // 请求地址
	method      string            // 请求方式
	contentType string            // 请求类型
	body        interface{}       // 发送请求的body
	header      map[string]string // 头文件
}

const (
	contentJSON = "application/json;charset=utf-8"
	contentXML  = "application/xml;charset=utf-8"
	contentText = "text/plain;charset=utf-8"
)

// New 初始化一个请求包对象
func New() *Net {
	var release bool
	netMode := os.Getenv("NET_MODE")
	if len(netMode) == 0 {
		release = false
	} else if netMode == "release" {
		release = true
	}

	return &Net{
		client:   http.DefaultClient,
		isRelase: release,
	}
}

// GetClient 获取http client
func (n *Net) GetClient() *http.Client {
	return n.client
}

// NewWithClient 初始化一个请求包对象，自己传入Client
func NewWithClient(client *http.Client) *Net {
	return &Net{
		client: client,
	}
}

// Get 发送 Get 请求
func (n *Net) Get(url string) *SuperAgent {
	return &SuperAgent{net: n, url: url, method: "GET"}
}

// Post 发送 Post 请求
func (n *Net) Post(url string) *SuperAgent {
	return &SuperAgent{net: n, url: url, method: "POST"}
}

// Put 发送 Put 请求
func (n *Net) Put(url string) *SuperAgent {
	return &SuperAgent{net: n, url: url, method: "PUT"}
}

// Delete 发送 Delete 请求
func (n *Net) Delete(url string) *SuperAgent {
	return &SuperAgent{net: n, url: url, method: "DELETE"}
}

// JSON 设置请求数据内容，默认用 Content-Type=application/json; 方式发送json数据
func (s *SuperAgent) JSON(body interface{}) *SuperAgent {
	s.body = body
	s.contentType = contentJSON
	return s
}

// XML 设置请求数据内容，默认用 Content-Type=application/json; 方式发送json数据
func (s *SuperAgent) XML(body interface{}) *SuperAgent {
	s.body = body
	s.contentType = contentXML
	return s
}

// Text 设置请求数据内容，默认用 Content-Type=text/plain; 方式发送string数据
func (s *SuperAgent) Text(body string) *SuperAgent {
	s.body = body
	s.contentType = contentText
	return s
}

// Header 设置请求头内容
func (s *SuperAgent) Header(header map[string]string) *SuperAgent {
	s.header = header
	return s
}

// End 开始http请求
func (s *SuperAgent) End(ctx context.Context, v interface{}) (*http.Response, error) {
	if len(s.contentType) > 0 && s.body == nil {
		s.body = ""
	}
	var req *http.Request
	var err error
	buf := new(bytes.Buffer)
	switch s.contentType {
	case contentJSON:
		err = json.NewEncoder(buf).Encode(s.body)
	case contentXML:
		err = xml.NewEncoder(buf).Encode(s.body)
	case contentText:
		_, err = buf.WriteString(s.body.(string))
	}
	if err != nil {
		return nil, err
	}

	// 转换 url
	rel, err := url.Parse(s.url)
	if err != nil {
		return nil, err
	}
	u := s.net.baseURL.ResolveReference(rel)

	req, err = http.NewRequest(s.method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	if len(s.contentType) > 0 {
		req.Header.Set("Content-Type", s.contentType)
	}
	for key, value := range s.header {
		req.Header.Set(key, value)
	}

	if ctx != nil {
		req = req.WithContext(ctx)
	}

	// 执行网络请求
	resp, err := s.net.client.Do(req)
	if err != nil {

		if ctx != nil {
			// If we got an error, and the context has been canceled, the context's error is probably more useful.
			select {
			case <-ctx.Done():
				return nil, ctx.Err()
			default:
			}
		}

		// If the error type is *url.Error, sanitize its URL before returning.
		if e, ok := err.(*url.Error); ok {
			if url, err := url.Parse(e.URL); err == nil {
				e.URL = url.String()
				return nil, e
			}
		}
		return nil, err
	}

	defer func() {
		// Drain up to 512 bytes and close the body to let the Transport reuse the connection
		io.CopyN(ioutil.Discard, resp.Body, 512)
		resp.Body.Close()
	}()

	if v != nil {
		if w, ok := v.(io.Writer); ok {
			io.Copy(w, resp.Body)
		} else {
			body, err := ioutil.ReadAll(resp.Body)
			if !s.net.isRelase {
				log.Printf("url: %s , response body: %s", s.url, string(body))
			}

			// 默认认为 contentType 不为xml的情况下，所有返回都用json解析
			if strings.EqualFold(s.contentType, contentXML) {
				err = xml.Unmarshal(body, v)
			} else {
				err = json.Unmarshal(body, v)
			}

			if err == io.EOF {
				err = nil // ignore EOF errors caused by empty response body
			}
		}
	}

	return resp, err
}
<|endoftext|>


<commit_before>/*
This package implements a TCP network service which allows clients to queue and
deque data. The client has 2 verbs "ENQUE" and "DEQUE" and the server can send 3
reponse status words "OK", "ERROR" and "ITEM".

All messages have the following format:

    COMMAND DATA

Where DATA is Base64 encoded and COMMAND is a single ASCII word with no spaces.

The client can send ENQUE or DEQUE at any time. If the queue is empty the server
will respond with:

    ERROR cXVldWUgaXMgZW1wdHk=

Which decodes too:

    ERROR queue is empty

Otherwise it repondes with

    ITEM XXXXXXXXXXXXXXX

When enqueing data the client sends

    ENQUE XXXXXXXXXXXXXXXXX

The server will respond with

    OK

Unless there is some internal error.

*/

package net

/* queued
 * Author: Tim Henderson
 * Email: tadh@case.edu
 * Copyright 2013 All Right Reserved
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *  * Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 *  * Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 *  * Neither the name of the queued nor the names of its contributors may be
 *    used to endorse or promote products derived from this software without
 *    specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

import (
  "fmt"
  "os"
  "net"
  "encoding/base64"
  "bytes"
  logpkg "log"
)

import (
  netutils "github.com/timtadh/netutils"
)

var log *logpkg.Logger

func init() {
    log = logpkg.New(os.Stderr, "queued/net> ", logpkg.Ltime | logpkg.Lshortfile)
}

type Server struct {
    ln *net.TCPListener
    queue Queue
}

func NewServer(queue Queue) *Server {
    return &Server{queue:queue}
}

/*
Starts a server. This is a blocking call it will run until the server shuts
down. If you want to run this in a seperate thread simply call it in its own
goroutine. Additionally under certain conditions this function may panic.

These include:
    1. There is already a link established.
    2. The server is unable to bind to the port.

The expectation is for these errors to either cause a hard crash or be caught
logged and then crashed.  */
func (self *Server) Start(port int) {
    if self.ln != nil {
        panic("Server already started")
    }
    ln, err := net.ListenTCP(
      "tcp",
      &net.TCPAddr{IP:net.ParseIP("0.0.0.0"), Port:port},
    )
    if err != nil {
        panic(err)
    }
    self.ln = ln
    self.listen()
}

/*
Stop a started server. If there is some problem stopping the server an error
will be returned.  */
func (self *Server) Stop() error {
    if self.ln == nil {
        return fmt.Errorf("Can't close non-existent link")
    }
    return self.ln.Close()
}

func (self *Server) listen() {
    var EOF bool
    for !EOF {
        con, err := self.ln.AcceptTCP()
        if netutils.IsEOF(err) {
            EOF = true
        } else if err != nil {
            log.Panic(err)
        } else {
            send := netutils.TCPWriter(con)
            recv := netutils.TCPReader(con)
            go self.connection(send, recv)
        }
    }
}

/*
Decode messages to/from the server. The server reads and sends line oriented
commands with message bodies base64 encoded. This function handles decoding
those lines for you.  */
func DecodeMessage(line []byte) (string, []byte) {
    split := bytes.SplitN(line, []byte(" "), 2)
    command := string(bytes.TrimSpace(split[0]))
    var rest []byte
    if len(split) > 1 {
        b64 := base64.StdEncoding
        encoded := bytes.TrimSpace(split[1])
        rest = make([]byte, b64.DecodedLen(len(encoded)))
        if n, err := b64.Decode(rest, encoded); err != nil {
            panic(err)
        } else {
            rest = rest[:n]
        }
    }
    return command, rest
}

/*
Encode a message for the server. Handles base64ing the msg. It is expected that
the paramter `cmd` will not have a space in it. If it does problems will occur
on the reciever side.  */
func EncodeMessage(cmd string, msg []byte) []byte {
    if msg == nil {
        return []byte(cmd + "\n")
    }
    b64 := base64.StdEncoding
    bcmd := []byte(cmd)
    cmdlen := len(bcmd)
    msg64len := cmdlen + b64.EncodedLen(len(msg)) + 2
    msg64 := make([]byte, msg64len)
    copy(msg64[:cmdlen], bcmd)
    msg64[cmdlen] = ' '
    b64.Encode(msg64[cmdlen+1:msg64len-1], msg)
    msg64[len(msg64)-1] = '\n'
    return msg64
}

func (self *Server) connection(send chan<- []byte, recv <-chan byte) {
    defer func() { <-recv }()
    defer log.Println("client closed")
    defer close(send)

    log.Println("new client")

    responder := func(f func([]byte)(string, []byte, error)) (g func([]byte)) {
        return func(rest []byte) {
            cmd, data, err := f(rest)
            if err != nil {
                log.Println(err)
                send<-EncodeMessage("ERROR", []byte(err.Error()))
            } else if cmd != "" {
                send<-EncodeMessage(cmd, data)
            } else {
                send<-EncodeMessage("OK", nil)
            }
        }
    }

    bad := responder(func(rest []byte) (string, []byte, error) {
        return "", nil, fmt.Errorf("bad command recieved")
    })

    enque := responder(func(rest []byte) (string, []byte, error) {
        // log.Println("got", rest)
        if rest == nil {
            return "", nil, fmt.Errorf("no data sent to queue")
        } else {
            return "", nil, self.queue.Enque(rest)
        }
    })

    deque := responder(func(rest []byte) (string, []byte, error) {
        if rest != nil {
            return "", nil, fmt.Errorf("recieved msg data when none was expected")
        }
        if self.queue.Empty() {
            return "", nil, fmt.Errorf("queue is empty")
        }
        data, err := self.queue.Deque()
        if err != nil {
            return "", nil, err
        }
        return "ITEM", data, nil
    })

    defer func() {
        if e := recover(); e != nil {
            send<-EncodeMessage("error", []byte(fmt.Sprintf("%v", e)))
        }
    }()

    for line := range netutils.Readlines(recv) {
        command, rest := DecodeMessage(line)
        switch command {
        case "ENQUE": enque(rest)
        case "DEQUE": deque(rest)
        default:
            log.Printf("'%s'\n", command)
            bad(rest)
        }
    }
}

<commit_msg>made it quieter<commit_after>/*
This package implements a TCP network service which allows clients to queue and
deque data. The client has 2 verbs "ENQUE" and "DEQUE" and the server can send 3
reponse status words "OK", "ERROR" and "ITEM".

All messages have the following format:

    COMMAND DATA

Where DATA is Base64 encoded and COMMAND is a single ASCII word with no spaces.

The client can send ENQUE or DEQUE at any time. If the queue is empty the server
will respond with:

    ERROR cXVldWUgaXMgZW1wdHk=

Which decodes too:

    ERROR queue is empty

Otherwise it repondes with

    ITEM XXXXXXXXXXXXXXX

When enqueing data the client sends

    ENQUE XXXXXXXXXXXXXXXXX

The server will respond with

    OK

Unless there is some internal error.

*/

package net

/* queued
 * Author: Tim Henderson
 * Email: tadh@case.edu
 * Copyright 2013 All Right Reserved
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *  * Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 *  * Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 *  * Neither the name of the queued nor the names of its contributors may be
 *    used to endorse or promote products derived from this software without
 *    specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

import (
  "fmt"
  "os"
  "net"
  "encoding/base64"
  "bytes"
  logpkg "log"
)

import (
  netutils "github.com/timtadh/netutils"
)

var log *logpkg.Logger

func init() {
    log = logpkg.New(os.Stderr, "queued/net> ", logpkg.Ltime | logpkg.Lshortfile)
}

type Server struct {
    ln *net.TCPListener
    queue Queue
}

func NewServer(queue Queue) *Server {
    return &Server{queue:queue}
}

/*
Starts a server. This is a blocking call it will run until the server shuts
down. If you want to run this in a seperate thread simply call it in its own
goroutine. Additionally under certain conditions this function may panic.

These include:
    1. There is already a link established.
    2. The server is unable to bind to the port.

The expectation is for these errors to either cause a hard crash or be caught
logged and then crashed.  */
func (self *Server) Start(port int) {
    if self.ln != nil {
        panic("Server already started")
    }
    ln, err := net.ListenTCP(
      "tcp",
      &net.TCPAddr{IP:net.ParseIP("0.0.0.0"), Port:port},
    )
    if err != nil {
        panic(err)
    }
    self.ln = ln
    self.listen()
}

/*
Stop a started server. If there is some problem stopping the server an error
will be returned.  */
func (self *Server) Stop() error {
    if self.ln == nil {
        return fmt.Errorf("Can't close non-existent link")
    }
    return self.ln.Close()
}

func (self *Server) listen() {
    var EOF bool
    for !EOF {
        con, err := self.ln.AcceptTCP()
        if netutils.IsEOF(err) {
            EOF = true
        } else if err != nil {
            log.Panic(err)
        } else {
            send := netutils.TCPWriter(con)
            recv := netutils.TCPReader(con)
            go self.connection(send, recv)
        }
    }
}

/*
Decode messages to/from the server. The server reads and sends line oriented
commands with message bodies base64 encoded. This function handles decoding
those lines for you.  */
func DecodeMessage(line []byte) (string, []byte) {
    split := bytes.SplitN(line, []byte(" "), 2)
    command := string(bytes.TrimSpace(split[0]))
    var rest []byte
    if len(split) > 1 {
        b64 := base64.StdEncoding
        encoded := bytes.TrimSpace(split[1])
        rest = make([]byte, b64.DecodedLen(len(encoded)))
        if n, err := b64.Decode(rest, encoded); err != nil {
            panic(err)
        } else {
            rest = rest[:n]
        }
    }
    return command, rest
}

/*
Encode a message for the server. Handles base64ing the msg. It is expected that
the paramter `cmd` will not have a space in it. If it does problems will occur
on the reciever side.  */
func EncodeMessage(cmd string, msg []byte) []byte {
    if msg == nil {
        return []byte(cmd + "\n")
    }
    b64 := base64.StdEncoding
    bcmd := []byte(cmd)
    cmdlen := len(bcmd)
    msg64len := cmdlen + b64.EncodedLen(len(msg)) + 2
    msg64 := make([]byte, msg64len)
    copy(msg64[:cmdlen], bcmd)
    msg64[cmdlen] = ' '
    b64.Encode(msg64[cmdlen+1:msg64len-1], msg)
    msg64[len(msg64)-1] = '\n'
    return msg64
}

func (self *Server) connection(send chan<- []byte, recv <-chan byte) {
    defer func() { <-recv }()
    // defer log.Println("client closed")
    defer close(send)

    // log.Println("new client")

    responder := func(f func([]byte)(string, []byte, error)) (g func([]byte)) {
        return func(rest []byte) {
            cmd, data, err := f(rest)
            if err != nil {
                if err.Error() != "queue is empty" {
                    log.Println(err)
                }
                send<-EncodeMessage("ERROR", []byte(err.Error()))
            } else if cmd != "" {
                send<-EncodeMessage(cmd, data)
            } else {
                send<-EncodeMessage("OK", nil)
            }
        }
    }

    bad := responder(func(rest []byte) (string, []byte, error) {
        return "", nil, fmt.Errorf("bad command recieved")
    })

    enque := responder(func(rest []byte) (string, []byte, error) {
        // log.Println("got", rest)
        if rest == nil {
            return "", nil, fmt.Errorf("no data sent to queue")
        } else {
            return "", nil, self.queue.Enque(rest)
        }
    })

    deque := responder(func(rest []byte) (string, []byte, error) {
        if rest != nil {
            return "", nil, fmt.Errorf("recieved msg data when none was expected")
        }
        if self.queue.Empty() {
            return "", nil, fmt.Errorf("queue is empty")
        }
        data, err := self.queue.Deque()
        if err != nil {
            return "", nil, err
        }
        return "ITEM", data, nil
    })

    defer func() {
        if e := recover(); e != nil {
            send<-EncodeMessage("error", []byte(fmt.Sprintf("%v", e)))
        }
    }()

    for line := range netutils.Readlines(recv) {
        command, rest := DecodeMessage(line)
        switch command {
        case "ENQUE": enque(rest)
        case "DEQUE": deque(rest)
        default:
            log.Printf("'%s'\n", command)
            bad(rest)
        }
    }
}

<|endoftext|>