<commit_before>/*
The MIT License (MIT)

transfer.sh was originally written by and
Copyright (c) 2014 DutchCoders [https://github.com/dutchcoders/]

Some modifications
Copyright (c) 2015 John Ko

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package main

import (
	"fmt"
	"github.com/gorilla/mux"
	"github.com/kennygrant/sanitize"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"
)

func healthHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "All systems go.")
}

func notFoundHandler(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "404 Not Found.", 404)
}

func putHandler(w http.ResponseWriter, r *http.Request) {
	if allowedPut() {
		var contentLength uint64
		var err error
		vars := mux.Vars(r)
		filename := sanitize.Path(filepath.Base(vars["filename"]))
		contentType := mime.TypeByExtension(filepath.Ext(filename))
		if contentType == "" {
			contentType = "application/octet-stream"
		}
		file, err := ioutil.TempFile(config.Temp, "transfer-")
		if err != nil {
			log.Printf("%s", err.Error())
			http.Error(w, "Internal server error.", 500)
			return
		} else {
			defer file.Close()
			_, err = io.Copy(file, r.Body)
			if err != nil {
				os.Remove(file.Name())
				log.Printf("%s", err.Error())
				http.Error(w, "Internal server error.", 500)
				return
			} else {
				w.Header().Set("Content-Type", "text/plain")
				var hash string
				if hash, contentLength, err = storage.HardLinkSha512Path(file.Name(), filename); err != nil {
					log.Printf("%s", err.Error())
				} else if err == nil {
					log.Printf("Hashed %s as %s", filename, hash)
					fmt.Fprintf(w, "{\"sha512\":\"%s\",\"filename\":\"%s\",\"length\":%d,\"content_type\":\"%s\",\"stub\":true}", hash, filename, contentLength, contentType)
				}
			}
		}
	} else {
		http.Error(w, "403 Forbidden. Uploading is disabled.", 403)
	}
}

func getHandler(w http.ResponseWriter, r *http.Request) {
	var filename string
	var reader io.ReadSeeker
	var modTime time.Time
	var err error
	// look for custom user agent
	gouseragent := regexp.MustCompile("dtfc.*")
	if allowedGet() {
		vars := mux.Vars(r)
		hash := vars["hash"]
		filename, reader, _, modTime, err = storage.Seeker(hash)
		if err != nil {
			if strings.Index(err.Error(), "no such file or directory") >= 0 {
				log.Printf("%s", err.Error())
				// try from peer
				var found = false
				// TODO this useragent test fails too often. too many recursive GETs, maybe try GET /hash/norecursion?
				if !gouseragent.MatchString(r.UserAgent()) {
					// dtfc specific
					found, filename, reader, modTime, err = getFromPeers(hash)
					if err != nil {
						if strings.Index(err.Error(), "Already peerloading") >= 0 {
							// https://tools.ietf.org/html/rfc2616#section-10.5.4
							log.Printf("%s", err.Error())
							// Retry in 2 minutes (120 seconds)
							w.Header().Set("Retry-After", fmt.Sprintf("120"))
							http.Error(w, "Service Unavailable. Try again in 2 minutes.", 503)
							return
						} else {
							log.Printf("Error while getFromPeers. %s", err.Error())
							http.Error(w, "Internal server error.", 500)
							return
						}
					}
				}
				// end try from peer
				if found == false {
					notFoundHandler(w, r)
					return
				}
			} else {
				log.Printf("%s", err.Error())
				http.Error(w, "Could not retrieve file.", 500)
				return
			}
		}
		w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
		http.ServeContent(w, r, filename, modTime, reader)
		// refresh peers a percentage of the time
		if rand.Intn(10) == 0 {
			refreshPeerList()
		}
	} else {
		http.Error(w, "403 Forbidden. Downloading is disabled.", 403)
	}
}

func RedirectHandler(h http.Handler) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/health.html" {
		} else if ipAddrFromRemoteAddr(r.Host) == "127.0.0.1" {
		} else if ipAddrFromRemoteAddr(r.Host) == "::1" {
		} else if r.Header.Get("X-Forwarded-Proto") != "https" && r.Method == "GET" {
			http.Redirect(w, r, "https://"+r.Host+r.RequestURI, 301)
			return
		}
		h.ServeHTTP(w, r)
	}
}
<commit_msg>set header only before print<commit_after>/*
The MIT License (MIT)

transfer.sh was originally written by and
Copyright (c) 2014 DutchCoders [https://github.com/dutchcoders/]

Some modifications
Copyright (c) 2015 John Ko

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package main

import (
	"fmt"
	"github.com/gorilla/mux"
	"github.com/kennygrant/sanitize"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"
)

func healthHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "All systems go.")
}

func notFoundHandler(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "404 Not Found.", 404)
}

func putHandler(w http.ResponseWriter, r *http.Request) {
	if allowedPut() {
		var contentLength uint64
		var err error
		vars := mux.Vars(r)
		filename := sanitize.Path(filepath.Base(vars["filename"]))
		contentType := mime.TypeByExtension(filepath.Ext(filename))
		if contentType == "" {
			contentType = "application/octet-stream"
		}
		file, err := ioutil.TempFile(config.Temp, "transfer-")
		if err != nil {
			log.Printf("%s", err.Error())
			http.Error(w, "Internal server error.", 500)
			return
		} else {
			defer file.Close()
			_, err = io.Copy(file, r.Body)
			if err != nil {
				os.Remove(file.Name())
				log.Printf("%s", err.Error())
				http.Error(w, "Internal server error.", 500)
				return
			} else {
				var hash string
				if hash, contentLength, err = storage.HardLinkSha512Path(file.Name(), filename); err != nil {
					log.Printf("%s", err.Error())
				} else if err == nil {
					log.Printf("Hashed %s as %s", filename, hash)
					w.Header().Set("Content-Type", "text/plain")
					fmt.Fprintf(w, "{\"sha512\":\"%s\",\"filename\":\"%s\",\"length\":%d,\"content_type\":\"%s\",\"stub\":true}", hash, filename, contentLength, contentType)
				}
			}
		}
	} else {
		http.Error(w, "403 Forbidden. Uploading is disabled.", 403)
	}
}

func getHandler(w http.ResponseWriter, r *http.Request) {
	var filename string
	var reader io.ReadSeeker
	var modTime time.Time
	var err error
	// look for custom user agent
	gouseragent := regexp.MustCompile("dtfc.*")
	if allowedGet() {
		vars := mux.Vars(r)
		hash := vars["hash"]
		filename, reader, _, modTime, err = storage.Seeker(hash)
		if err != nil {
			if strings.Index(err.Error(), "no such file or directory") >= 0 {
				log.Printf("%s", err.Error())
				// try from peer
				var found = false
				// TODO this useragent test fails too often. too many recursive GETs, maybe try GET /hash/norecursion?
				if !gouseragent.MatchString(r.UserAgent()) {
					// dtfc specific
					found, filename, reader, modTime, err = getFromPeers(hash)
					if err != nil {
						if strings.Index(err.Error(), "Already peerloading") >= 0 {
							// https://tools.ietf.org/html/rfc2616#section-10.5.4
							log.Printf("%s", err.Error())
							// Retry in 2 minutes (120 seconds)
							w.Header().Set("Retry-After", fmt.Sprintf("120"))
							http.Error(w, "Service Unavailable. Try again in 2 minutes.", 503)
							return
						} else {
							log.Printf("Error while getFromPeers. %s", err.Error())
							http.Error(w, "Internal server error.", 500)
							return
						}
					}
				}
				// end try from peer
				if found == false {
					notFoundHandler(w, r)
					return
				}
			} else {
				log.Printf("%s", err.Error())
				http.Error(w, "Could not retrieve file.", 500)
				return
			}
		}
		w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
		http.ServeContent(w, r, filename, modTime, reader)
		// refresh peers a percentage of the time
		if rand.Intn(10) == 0 {
			refreshPeerList()
		}
	} else {
		http.Error(w, "403 Forbidden. Downloading is disabled.", 403)
	}
}

func RedirectHandler(h http.Handler) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/health.html" {
		} else if ipAddrFromRemoteAddr(r.Host) == "127.0.0.1" {
		} else if ipAddrFromRemoteAddr(r.Host) == "::1" {
		} else if r.Header.Get("X-Forwarded-Proto") != "https" && r.Method == "GET" {
			http.Redirect(w, r, "https://"+r.Host+r.RequestURI, 301)
			return
		}
		h.ServeHTTP(w, r)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"net/http"
	"fmt"
	"github.com/gorilla/mux"
	log "github.com/Sirupsen/logrus"
	tid "github.com/Financial-Times/transactionid-utils-go"
	"golang.org/x/net/context"
)

const mapiPath = "/eom-file/"
const matPath ="/content-transform/"

var logger = log.New()

type Handlers struct {
	mapiAuth      string
	matHostHeader string
	mapiUri       string
	matUri        string
}

func pingHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "pong")
}

func (h Handlers) buildInfoHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "build-info")
}

func (h Handlers) contentPreviewHandler(w http.ResponseWriter, r *http.Request) {
	ctx := tid.TransactionAwareContext(context.Background(), r.Header.Get(tid.TransactionIDHeader))

	logger.Formatter = new(log.JSONFormatter)
	logger.WithFields(log.Fields{
		"requestUri" : r.RequestURI,
		"transactionId" : tid.GetTransactionIDFromRequest(r),
	}).Info("request started");

	vars := mux.Vars(r)
	uuid := vars["uuid"]

	success, mapiResp := h.getNativeContent(ctx, uuid, w)
	if !success { return }
	success, matResp := h.getTransformedContent(ctx, uuid, *mapiResp, w)
	if(!success) {
		mapiResp.Body.Close()
		return
	}
	io.Copy(w, matResp.Body)
	matResp.Body.Close()
}

func ( h Handlers) getNativeContent(ctx context.Context, uuid string, w http.ResponseWriter) (ok bool, resp *http.Response) {
	logger.Formatter = new(log.JSONFormatter)
	requestUrl := fmt.Sprintf("%s%s%s", h.mapiUri, mapiPath, uuid)

	transactionId, _ := tid.GetTransactionIDFromContext(ctx)
	logger.WithFields(log.Fields{
		"requestUri" : requestUrl,
		"transactionId" : transactionId,
	}).Info("Request to MAPI");

	req, err := http.NewRequest("GET", requestUrl, nil)

	req.Header.Set(tid.TransactionIDHeader, transactionId)
	req.Header.Set("Authorization", "Basic " + h.mapiAuth)
	req.Header.Set("Content-Type", "application/json")
	resp, err = client.Do(req)

	//this happens when hostname cannot be resolved or host is not accessible
	if err !=nil {
		log.WithFields(log.Fields{"error" : err, "transactionId" : req.Header.Get(tid.TransactionIDHeader)}).Warnf("Cannot reach MAPI host")
		w.WriteHeader(http.StatusServiceUnavailable)
		return false, nil
	}
	if resp.StatusCode != http.StatusOK {
		w.WriteHeader(http.StatusNotFound);
		log.WithFields(log.Fields{"MAPI.Status" : resp.StatusCode, "transactionId" : resp.Header.Get(tid.TransactionIDHeader)}).Warnf("Request to MAPI failed")
		return false, nil
	}
	logger.WithFields(log.Fields{"status" : resp.Status, "transactionId" : resp.Header.Get(tid.TransactionIDHeader)}).Info("Response from MAPI")
	return true, resp
}

func ( h Handlers) getTransformedContent(ctx context.Context, uuid string, mapiResp http.Response, w http.ResponseWriter) (ok bool, resp *http.Response) {
	logger.Formatter = new(log.JSONFormatter)
	requestUrl := fmt.Sprintf("%s%s%s", h.matUri, matPath, uuid)
	transactionId, _ := tid.GetTransactionIDFromContext(ctx)
	//TODO we need to assert that  mapiResp.Header.Get(tid.TransactionIDHeader) ==  transactionId
	//to ensure that we are logging exactly what is actually passed around in the headers
	logger.WithFields(log.Fields{
		"requestUri" : requestUrl,
		"transactionId" : transactionId,
	}).Info("Request to MAT");

	req, err := http.NewRequest("POST", requestUrl, mapiResp.Body)
	req.Host = h.matHostHeader
	req.Header.Set(tid.TransactionIDHeader, transactionId)
	req.Header.Set("Content-Type", "application/json")
	resp, err = client.Do(req)

	//this happens when hostname cannot be resolved or host is not accessible
	if err !=nil {
		log.WithFields(log.Fields{"error" : err, "transactionId" : transactionId }).Warnf("Cannot reach MAT host")
		w.WriteHeader(http.StatusServiceUnavailable)
		return false, nil
	}

	if resp.StatusCode != http.StatusOK {
		w.WriteHeader(http.StatusNotFound);
		log.WithFields(log.Fields{"MAT.Status" : resp.StatusCode, "transactionId" : transactionId }).Warnf("Request to MAT failed")
		return false, nil
	}

	logger.WithFields(log.Fields{"status" : resp.Status, "transactionId" : transactionId}).Info("Response from MAT")
	return true, resp
}
<commit_msg>added uuid to context just as well<commit_after>package main

import (
	"io"
	"net/http"
	"fmt"
	"github.com/gorilla/mux"
	log "github.com/Sirupsen/logrus"
	tid "github.com/Financial-Times/transactionid-utils-go"
	"golang.org/x/net/context"
)

const mapiPath = "/eom-file/"
const matPath ="/content-transform/"
const uuidKey = "uuid"

var logger = log.New()

type Handlers struct {
	mapiAuth      string
	matHostHeader string
	mapiUri       string
	matUri        string
}

func pingHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "pong")
}

func (h Handlers) buildInfoHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "build-info")
}

func (h Handlers) contentPreviewHandler(w http.ResponseWriter, r *http.Request) {
	logger.Formatter = new(log.JSONFormatter)
	logger.WithFields(log.Fields{
		"requestUri" : r.RequestURI,
		"transactionId" : tid.GetTransactionIDFromRequest(r),
	}).Info("request started");

	vars := mux.Vars(r)
	uuid := vars["uuid"]

	ctx := tid.TransactionAwareContext(context.Background(), r.Header.Get(tid.TransactionIDHeader))
	ctx = context.WithValue(ctx, uuidKey, uuid)

	success, mapiResp := h.getNativeContent(ctx, w)
	if !success { return }
	success, matResp := h.getTransformedContent(ctx, *mapiResp, w)
	if(!success) {
		mapiResp.Body.Close()
		return
	}
	io.Copy(w, matResp.Body)
	matResp.Body.Close()
}

func ( h Handlers) getNativeContent(ctx context.Context, w http.ResponseWriter) (ok bool, resp *http.Response) {
	logger.Formatter = new(log.JSONFormatter)
	uuid := ctx.Value(uuidKey).(string)
	requestUrl := fmt.Sprintf("%s%s%s", h.mapiUri, mapiPath, uuid)

	transactionId, _ := tid.GetTransactionIDFromContext(ctx)
	logger.WithFields(log.Fields{
		"requestUri" : requestUrl,
		"transactionId" : transactionId,
	}).Info("Request to MAPI");

	req, err := http.NewRequest("GET", requestUrl, nil)

	req.Header.Set(tid.TransactionIDHeader, transactionId)
	req.Header.Set("Authorization", "Basic " + h.mapiAuth)
	req.Header.Set("Content-Type", "application/json")
	resp, err = client.Do(req)

	//this happens when hostname cannot be resolved or host is not accessible
	if err !=nil {
		log.WithFields(log.Fields{"error" : err, "transactionId" : req.Header.Get(tid.TransactionIDHeader)}).Warnf("Cannot reach MAPI host")
		w.WriteHeader(http.StatusServiceUnavailable)
		return false, nil
	}
	if resp.StatusCode != http.StatusOK {
		w.WriteHeader(http.StatusNotFound);
		log.WithFields(log.Fields{"MAPI.Status" : resp.StatusCode, "transactionId" : resp.Header.Get(tid.TransactionIDHeader)}).Warnf("Request to MAPI failed")
		return false, nil
	}
	logger.WithFields(log.Fields{"status" : resp.Status, "transactionId" : resp.Header.Get(tid.TransactionIDHeader)}).Info("Response from MAPI")
	return true, resp
}

func ( h Handlers) getTransformedContent(ctx context.Context, mapiResp http.Response, w http.ResponseWriter) (ok bool, resp *http.Response) {
	logger.Formatter = new(log.JSONFormatter)
	uuid := ctx.Value(uuidKey).(string)
	requestUrl := fmt.Sprintf("%s%s%s", h.matUri, matPath, uuid)
	transactionId, _ := tid.GetTransactionIDFromContext(ctx)
	//TODO we need to assert that  mapiResp.Header.Get(tid.TransactionIDHeader) ==  transactionId
	//to ensure that we are logging exactly what is actually passed around in the headers
	logger.WithFields(log.Fields{
		"requestUri" : requestUrl,
		"transactionId" : transactionId,
	}).Info("Request to MAT");

	req, err := http.NewRequest("POST", requestUrl, mapiResp.Body)
	req.Host = h.matHostHeader
	req.Header.Set(tid.TransactionIDHeader, transactionId)
	req.Header.Set("Content-Type", "application/json")
	resp, err = client.Do(req)

	//this happens when hostname cannot be resolved or host is not accessible
	if err !=nil {
		log.WithFields(log.Fields{"error" : err, "transactionId" : transactionId }).Warnf("Cannot reach MAT host")
		w.WriteHeader(http.StatusServiceUnavailable)
		return false, nil
	}

	if resp.StatusCode != http.StatusOK {
		w.WriteHeader(http.StatusNotFound);
		log.WithFields(log.Fields{"MAT.Status" : resp.StatusCode, "transactionId" : transactionId }).Warnf("Request to MAT failed")
		return false, nil
	}

	logger.WithFields(log.Fields{"status" : resp.Status, "transactionId" : transactionId}).Info("Response from MAT")
	return true, resp
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	_ "net/url"
	_ "os"

	_ "github.com/codegangsta/negroni"
	_ "github.com/garyburd/redigo/redis"
	_ "gopkg.in/mgo.v2"
	_ "gopkg.in/mgo.v2/bson"
)

const (
	Kb = 1024
)

func SetupMux() *http.ServeMux {
	server_mux := http.NewServeMux()
	server_mux.HandleFunc("/", IndexHandler)
	server_mux.HandleFunc("/login", LoginHandler)
	server_mux.HandleFunc("/api/v1/user/exercises", SubmissionHandler)
	return server_mux
}

func IndexHandler(res http.ResponseWriter, req *http.Request) {
	fmt.Fprintf(res, "Hello There")
}

func LoginHandler(res http.ResponseWriter, req *http.Request) {
	fmt.Fprintf(res, "hi")
}

type submitResponse struct {
	Status string `json::"status"`
	Error  string `json::"error"`
}

type submitRequest struct {
	Email string `json::"email"`
	Key   string `json::"key"`
	Code  string `json::"code"`
}

func SubmissionHandler(res http.ResponseWriter, req *http.Request) {
	switch req.Method {
	case "POST":
		PostSubmissionHandler(res, req)
	default:
		res.WriteHeader(http.StatusBadRequest)
	}
}

func PostSubmissionHandler(res http.ResponseWriter, req *http.Request) {
	var sreq *submitRequest
	decoder := json.NewDecoder(req.Body)
	if err := decoder.Decode(&sreq); err != nil {
		log.Printf("Error parsing response: %v", err)
		res.WriteHeader(http.StatusBadRequest)
		return
	}

	u, err := Users.FindUserByEmail(sreq.Email)
	if err != nil || sreq.Key != u.APIKey {
		res.WriteHeader(http.StatusForbidden)
		return
	}

	code := sreq.Code

	compressed_code, err := compressCode(code)
	if err != nil {
		log.Printf("Compressing code failure: %v\n", err)
		res.WriteHeader(http.StatusInternalServerError)
	}
	uncompressed_code, err := uncompressCode(compressed_code)
	if err != nil {
		log.Printf("Uncompressing code failure: %v\n", err)
		res.WriteHeader(http.StatusInternalServerError)
	}

	fmt.Printf("Compressed Code: %v\n", compressed_code)
	fmt.Printf("Uncompressed Code: %v\n", uncompressed_code)

	if len(compressed_code) > 20*Kb {
		log.Println("File is too big!")
		res.WriteHeader(http.StatusBadRequest)
		return
	}

	_, err = Users.SubmitCode(u.Email, compressed_code)
	if err != nil {
		res.WriteHeader(http.StatusInternalServerError)
		return
	}

	sres := submitResponse{Status: "OK", Error: "None"}

	res.Header().Set("Access-Control-Allow-Origin", "*")
	res.Header().Set("Access-Control-Allow-Headers", "X-Requested-With")
	res.Header().Set("Content-Type", "application/json")
	res.WriteHeader(http.StatusCreated)
	encoder := json.NewEncoder(res)
	if err := encoder.Encode(sres); err != nil {
		log.Printf("Error parsing response: %v", err)
	}
}

func compressCode(code string) (string, error) {
	var b bytes.Buffer
	gz := gzip.NewWriter(&b)
	defer gz.Close()

	_, err := gz.Write([]byte(code))
	if err != nil {
		return b.String(), err
	}

	return b.String(), nil
}

func uncompressCode(code string) (string, error) {
	b := bytes.NewBufferString(code)
	var uncompresed []byte
	gunz, err := gzip.NewReader(&b)
	if err != nil {
		return "", err
	}
	defer gunz.Close()

	if _, err := gunz.Read(uncompresed); err != nil {
		return "", err
	}

	return "", nil
}
<commit_msg>gziping code<commit_after>package main

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	_ "net/url"
	_ "os"

	_ "github.com/codegangsta/negroni"
	_ "github.com/garyburd/redigo/redis"
	_ "gopkg.in/mgo.v2"
	_ "gopkg.in/mgo.v2/bson"
)

const (
	Kb = 1024
)

func SetupMux() *http.ServeMux {
	server_mux := http.NewServeMux()
	server_mux.HandleFunc("/", IndexHandler)
	server_mux.HandleFunc("/login", LoginHandler)
	server_mux.HandleFunc("/api/v1/user/exercises", SubmissionHandler)
	return server_mux
}

func IndexHandler(res http.ResponseWriter, req *http.Request) {
	fmt.Fprintf(res, "Hello There")
}

func LoginHandler(res http.ResponseWriter, req *http.Request) {
	fmt.Fprintf(res, "hi")
}

type submitResponse struct {
	Status string `json::"status"`
	Error  string `json::"error"`
}

type submitRequest struct {
	Email string `json::"email"`
	Key   string `json::"key"`
	Code  string `json::"code"`
}

func SubmissionHandler(res http.ResponseWriter, req *http.Request) {
	switch req.Method {
	case "POST":
		PostSubmissionHandler(res, req)
	default:
		res.WriteHeader(http.StatusBadRequest)
	}
}

func PostSubmissionHandler(res http.ResponseWriter, req *http.Request) {
	var sreq *submitRequest
	decoder := json.NewDecoder(req.Body)
	if err := decoder.Decode(&sreq); err != nil {
		log.Printf("Error parsing response: %v", err)
		res.WriteHeader(http.StatusBadRequest)
		return
	}

	u, err := Users.FindUserByEmail(sreq.Email)
	if err != nil || sreq.Key != u.APIKey {
		res.WriteHeader(http.StatusForbidden)
		return
	}

	code := sreq.Code

	compressed_code, err := compressCode(code)
	if err != nil {
		log.Printf("Compressing code failure: %v\n", err)
		res.WriteHeader(http.StatusInternalServerError)
	}
	uncompressed_code, err := uncompressCode(compressed_code)
	if err != nil {
		log.Printf("Uncompressing code failure: %v\n", err)
		res.WriteHeader(http.StatusInternalServerError)
	}

	fmt.Printf("Compressed Code: %v\n", compressed_code)
	fmt.Printf("Uncompressed Code: %v\n", uncompressed_code)

	if len(compressed_code) > 20*Kb {
		log.Println("File is too big!")
		res.WriteHeader(http.StatusBadRequest)
		return
	}

	_, err = Users.SubmitCode(u.Email, compressed_code)
	if err != nil {
		res.WriteHeader(http.StatusInternalServerError)
		return
	}

	sres := submitResponse{Status: "OK", Error: "None"}

	res.Header().Set("Access-Control-Allow-Origin", "*")
	res.Header().Set("Access-Control-Allow-Headers", "X-Requested-With")
	res.Header().Set("Content-Type", "application/json")
	res.WriteHeader(http.StatusCreated)
	encoder := json.NewEncoder(res)
	if err := encoder.Encode(sres); err != nil {
		log.Printf("Error parsing response: %v", err)
	}
}

func compressCode(code string) (string, error) {
	var b bytes.Buffer
	gz := gzip.NewWriter(&b)
	defer gz.Close()

	_, err := gz.Write([]byte(code))
	if err != nil {
		return b.String(), err
	}

	return b.String(), nil
}

func uncompressCode(code string) (string, error) {
	b := bytes.NewBufferString(code)
	var uncompresed []byte
	gunz, err := gzip.NewReader(&b)
	if err != nil {
		return "", err
	}
	defer gunz.Close()

	if _, err := gunz.Read(uncompresed); err != nil {
		return string(uncompresed), err
	}

	return string(uncompresed), nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Qiang Xue. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package routing

import (
	"net/http"
	"strings"
	"fmt"
	"time"
	"io"
	"sync"
)

// HTTPHandlerFunc adapts a http.HandlerFunc into a routing.Handler.
func HTTPHandlerFunc(h http.HandlerFunc) Handler {
	return func(c *Context) {
		h(c.Response, c.Request)
	}
}

// HTTPHandler adapts a http.Handler into a routing.Handler.
func HTTPHandler(h http.Handler) Handler {
	return func(c *Context) {
		h.ServeHTTP(c.Response, c.Request)
	}
}

// Logger specifies the logger interface needed by ErrorHandler to log an error.
type Logger interface {
	// Error records an error message using the format and optional arguments.
	// The usage of format and arguments is similar to those for fmt.Printf().
	Error(format string, a ...interface{})
}

// ErrorHandler returns a handler that handles the error recorded in Context.Error.
//
// If Context.Error is an HTTPError, ErrorHandler will set the response status code
// as the status code specified by Context.Error; Otherwise, ErrorHandler will
// set the status code to be 500 (http.StatusInternalServerError) and record
// the error using logger (when it is not nil).
//
// This handler is usually used as one of the last handlers for a router.
func ErrorHandler(logger Logger) Handler {
	return func(c *Context) HTTPError {
		if err, ok := c.Error.(HTTPError); ok {
			c.Response.WriteHeader(err.Code())
			return err
		}
		if logger != nil {
			logger.Error("%v", c.Error)
		}
		c.Response.WriteHeader(http.StatusInternalServerError)
		return NewHTTPError(http.StatusInternalServerError)
	}
}

// NotFoundHandler returns a handler that triggers an HTTPError with the status http.StatusNotFound.
//
// This handler is usually used as one of the last handlers for a router.
func NotFoundHandler() Handler {
	return func() {
		panic(NewHTTPError(http.StatusNotFound))
	}
}

// TrailingSlashRemover returns a handler that removes trailing slashes from the requested URL.
// The handler will redirect the browser to the new URL without trailing slashes.
// The status parameter should be either http.StatusMovedPermanently (301) or http.StatusFound (302).
// If the original URL has no trailing slashes, nothing will be done by this handler.
func TrailingSlashRemover(status int) Handler {
	return func(c *Context) {
		if c.Request.URL.Path != "/" && strings.HasSuffix(c.Request.URL.Path, "/") {
			http.Redirect(c.Response, c.Request, strings.TrimRight(c.Request.URL.Path, "/"), status)
		} else {
			c.Next()
		}
	}
}

// AccessLogger returns a handler that logs an entry for every request.
// The access log entries will be written using the specified writer and the Apache httpd access log format.
func AccessLogger(writer io.Writer) Handler {
	var mu sync.Mutex
	return func(c *Context) {
		startTime := time.Now()

		req := c.Request
		rw := &logResponseWriter{c.Response, http.StatusOK, 0}
		c.Response = rw

		c.Next()

		clientIP := getClientIP(req)
		start := startTime.Format("02/Jan/2006 15:04:05 -0700")
		elapsed := time.Now().Sub(startTime).Seconds() * 1000
		requestLine := fmt.Sprintf("%s %s %s", req.Method, req.RequestURI, req.Proto)
		mu.Lock()
		defer mu.Unlock()
		fmt.Fprintf(writer, "%s - - [%s] \"%s %d %d\" %.3fms\n",
			clientIP, start, requestLine,
			rw.status, rw.bytesWritten, elapsed)
	}
}

func getClientIP(req *http.Request) string {
	ip := req.Header.Get("X-Real-IP")
	if ip == "" {
		ip = req.Header.Get("X-Forwarded-For")
		if ip == "" {
			ip = req.RemoteAddr
		}
	}
	if colon := strings.LastIndex(ip, ":"); colon != -1 {
		ip = ip[:colon]
	}
	return ip
}

type logResponseWriter struct {
	http.ResponseWriter
	status       int
	bytesWritten int64
}

func (r *logResponseWriter) Write(p []byte) (int, error) {
	written, err := r.ResponseWriter.Write(p)
	r.bytesWritten += int64(written)
	return written, err
}

func (r *logResponseWriter) WriteHeader(status int) {
	r.status = status
	r.ResponseWriter.WriteHeader(status)
}
<commit_msg>renamed Logger to ErrorLogger<commit_after>// Copyright 2015 Qiang Xue. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package routing

import (
	"net/http"
	"strings"
	"fmt"
	"time"
	"io"
	"sync"
)

// HTTPHandlerFunc adapts a http.HandlerFunc into a routing.Handler.
func HTTPHandlerFunc(h http.HandlerFunc) Handler {
	return func(c *Context) {
		h(c.Response, c.Request)
	}
}

// HTTPHandler adapts a http.Handler into a routing.Handler.
func HTTPHandler(h http.Handler) Handler {
	return func(c *Context) {
		h.ServeHTTP(c.Response, c.Request)
	}
}

// ErrorLogger specifies the logger interface needed by ErrorHandler to log an error.
type ErrorLogger interface {
	// Error records an error message using the format and optional arguments.
	// The usage of format and arguments is similar to those for fmt.Printf().
	Error(format string, a ...interface{})
}

// ErrorHandler returns a handler that handles the error recorded in Context.Error.
//
// If Context.Error is an HTTPError, ErrorHandler will set the response status code
// as the status code specified by Context.Error; Otherwise, ErrorHandler will
// set the status code to be 500 (http.StatusInternalServerError) and record
// the error using logger (when it is not nil).
//
// This handler is usually used as one of the last handlers for a router.
func ErrorHandler(logger ErrorLogger) Handler {
	return func(c *Context) HTTPError {
		if err, ok := c.Error.(HTTPError); ok {
			c.Response.WriteHeader(err.Code())
			return err
		}
		if logger != nil {
			logger.Error("%v", c.Error)
		}
		c.Response.WriteHeader(http.StatusInternalServerError)
		return NewHTTPError(http.StatusInternalServerError)
	}
}

// NotFoundHandler returns a handler that triggers an HTTPError with the status http.StatusNotFound.
//
// This handler is usually used as one of the last handlers for a router.
func NotFoundHandler() Handler {
	return func() {
		panic(NewHTTPError(http.StatusNotFound))
	}
}

// TrailingSlashRemover returns a handler that removes trailing slashes from the requested URL.
// The handler will redirect the browser to the new URL without trailing slashes.
// The status parameter should be either http.StatusMovedPermanently (301) or http.StatusFound (302).
// If the original URL has no trailing slashes, nothing will be done by this handler.
func TrailingSlashRemover(status int) Handler {
	return func(c *Context) {
		if c.Request.URL.Path != "/" && strings.HasSuffix(c.Request.URL.Path, "/") {
			http.Redirect(c.Response, c.Request, strings.TrimRight(c.Request.URL.Path, "/"), status)
		} else {
			c.Next()
		}
	}
}

// AccessLogger returns a handler that logs an entry for every request.
// The access log entries will be written using the specified writer and the Apache httpd access log format.
func AccessLogger(writer io.Writer) Handler {
	var mu sync.Mutex
	return func(c *Context) {
		startTime := time.Now()

		req := c.Request
		rw := &logResponseWriter{c.Response, http.StatusOK, 0}
		c.Response = rw

		c.Next()

		clientIP := getClientIP(req)
		start := startTime.Format("02/Jan/2006 15:04:05 -0700")
		elapsed := time.Now().Sub(startTime).Seconds() * 1000
		requestLine := fmt.Sprintf("%s %s %s", req.Method, req.RequestURI, req.Proto)
		mu.Lock()
		defer mu.Unlock()
		fmt.Fprintf(writer, "%s - - [%s] \"%s %d %d\" %.3fms\n",
			clientIP, start, requestLine,
			rw.status, rw.bytesWritten, elapsed)
	}
}

func getClientIP(req *http.Request) string {
	ip := req.Header.Get("X-Real-IP")
	if ip == "" {
		ip = req.Header.Get("X-Forwarded-For")
		if ip == "" {
			ip = req.RemoteAddr
		}
	}
	if colon := strings.LastIndex(ip, ":"); colon != -1 {
		ip = ip[:colon]
	}
	return ip
}

type logResponseWriter struct {
	http.ResponseWriter
	status       int
	bytesWritten int64
}

func (r *logResponseWriter) Write(p []byte) (int, error) {
	written, err := r.ResponseWriter.Write(p)
	r.bytesWritten += int64(written)
	return written, err
}

func (r *logResponseWriter) WriteHeader(status int) {
	r.status = status
	r.ResponseWriter.WriteHeader(status)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"text/template"

	"github.com/johansundell/cocapi"
)

type page struct {
	Title       string
	Name        string
	Description string
	MembersJson string
	Image       string
}

type alert struct {
	page
	OldMembersJson string
}

func handlePages(w http.ResponseWriter, req *http.Request) {
	t, err := template.New("clan-info.html").Delims("*{{", "}}*").ParseFiles(basePath + "tmpl/clan-info.html")
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
		return
	}

	p := page{}
	clan, err := getMembers(myClanTag, "rank")
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
		return
	}

	b, err := json.Marshal(clan.MemberList)
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
		return
	}
	p.Name = clan.Name
	p.Description = clan.Description
	p.MembersJson = string(b)
	p.Image = clan.BadgeUrls.Small

	err = t.Execute(w, p)
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
	}
}

func handleIndexPage(w http.ResponseWriter, req *http.Request) {
	t, err := template.New("index.html").Delims("*{{", "}}*").ParseFiles(basePath + "pages/index.html")

	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
		return
	}

	p := page{}
	p.Title = "COC Playground"

	t.Delims("*{{", "}}*")
	err = t.Execute(w, p)
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
	}
}

func getAlerts(w http.ResponseWriter, req *http.Request) {
	t, err := template.New("alert.html").Delims("*{{", "}}*").ParseFiles(basePath + "pages/alert.html")
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
		return
	}
	p := page{}
	p.Title = "COC Errors"

	t.Delims("*{{", "}}*")
	err = t.Execute(w, p)
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
	}
}

func handleAlerts(w http.ResponseWriter, req *http.Request) {
	t, err := template.New("clan-errors.html").Delims("*{{", "}}*").ParseFiles(basePath + "tmpl/clan-errors.html")
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
		return
	}

	p := alert{}

	var players = make([]cocapi.Player, 0)
	rows, err := db.Query("SELECT tag FROM members WHERE active = 1 AND exited > 0")
	if err != nil {
		log.Println(err)
		return
	}
	for rows.Next() {
		var tag string
		rows.Scan(&tag)
		p, err := cocapi.GetPlayerInfo(tag)
		if err == nil {
			players = append(players, p)
		}

	}

	b, err := json.Marshal(players)
	p.OldMembersJson = string(b)
	rows, err = db.Query("SELECT GROUP_CONCAT(name) as usernames, tag, count(*) AS c FROM members GROUP BY tag HAVING c > 1")
	if err != nil {
		log.Println(err)
	}
	players = make([]cocapi.Player, 0)
	for rows.Next() {
		var tag string
		var usernames string
		var count int
		err = rows.Scan(&usernames, &tag, &count)
		if err != nil {
			log.Println(err)
		}
		log.Println("Found:", tag)
		p, err := cocapi.GetPlayerInfo(tag)
		p.Name = usernames
		if err == nil {
			players = append(players, p)
		}
	}
	log.Println(players)
	b, err = json.Marshal(players)
	p.MembersJson = string(b)

	err = t.Execute(w, p)
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
	}
}
<commit_msg>Fixes<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"text/template"

	"github.com/johansundell/cocapi"
)

type page struct {
	Title       string
	Name        string
	Description string
	MembersJson string
	Image       string
}

type alert struct {
	page
	OldMembersJson string
}

func handlePages(w http.ResponseWriter, req *http.Request) {
	t, err := template.New("clan-info.html").Delims("*{{", "}}*").ParseFiles(basePath + "tmpl/clan-info.html")
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
		return
	}

	p := page{}
	clan, err := getMembers(myClanTag, "rank")
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
		return
	}

	b, err := json.Marshal(clan.MemberList)
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
		return
	}
	p.Name = clan.Name
	p.Description = clan.Description
	p.MembersJson = string(b)
	p.Image = clan.BadgeUrls.Small

	err = t.Execute(w, p)
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
	}
}

func handleIndexPage(w http.ResponseWriter, req *http.Request) {
	t, err := template.New("index.html").Delims("*{{", "}}*").ParseFiles(basePath + "pages/index.html")

	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err, basePath+"pages/index.html")
		return
	}

	p := page{}
	p.Title = "COC Playground"

	t.Delims("*{{", "}}*")
	err = t.Execute(w, p)
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
	}
}

func getAlerts(w http.ResponseWriter, req *http.Request) {
	t, err := template.New("alert.html").Delims("*{{", "}}*").ParseFiles(basePath + "pages/alert.html")
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err, basePath+"pages/alert.html")
		return
	}
	p := page{}
	p.Title = "COC Errors"

	t.Delims("*{{", "}}*")
	err = t.Execute(w, p)
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
	}
}

func handleAlerts(w http.ResponseWriter, req *http.Request) {
	t, err := template.New("clan-errors.html").Delims("*{{", "}}*").ParseFiles(basePath + "tmpl/clan-errors.html")
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
		return
	}

	p := alert{}

	var players = make([]cocapi.Player, 0)
	rows, err := db.Query("SELECT tag FROM members WHERE active = 1 AND exited > 0")
	if err != nil {
		log.Println(err)
		return
	}
	for rows.Next() {
		var tag string
		rows.Scan(&tag)
		p, err := cocapi.GetPlayerInfo(tag)
		if err == nil {
			players = append(players, p)
		}

	}

	b, err := json.Marshal(players)
	p.OldMembersJson = string(b)
	rows, err = db.Query("SELECT GROUP_CONCAT(name) as usernames, tag, count(*) AS c FROM members GROUP BY tag HAVING c > 1")
	if err != nil {
		log.Println(err)
	}
	players = make([]cocapi.Player, 0)
	for rows.Next() {
		var tag string
		var usernames string
		var count int
		err = rows.Scan(&usernames, &tag, &count)
		if err != nil {
			log.Println(err)
		}
		log.Println("Found:", tag)
		p, err := cocapi.GetPlayerInfo(tag)
		p.Name = usernames
		if err == nil {
			players = append(players, p)
		}
	}
	log.Println(players)
	b, err = json.Marshal(players)
	p.MembersJson = string(b)

	err = t.Execute(w, p)
	if err != nil {
		http.Error(w, http.StatusText(500), 500)
		fmt.Println(err)
	}
}
<|endoftext|>


<commit_before>//
// Copyright (C) 2015-present  Laszlo Csontos
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>. */
//

package youtube

import (
	"net/http"
	"time"

	"google.golang.org/api/googleapi/transport"
	"google.golang.org/api/youtube/v3"

	"github.com/lcsontos/uwatch/catalog"
	"github.com/lcsontos/uwatch/config"
)

const _PART = "id,snippet"

type Service struct {
	httpClient     *http.Client
	youTubeService *youtube.Service
}

var developerKey string

func New() (*Service, error) {
	service, err := NewWithRoundTripper(nil)

	return service, err
}

func NewWithRoundTripper(roundTripper http.RoundTripper) (*Service, error) {
	transport := &transport.APIKey{
		Key:       developerKey,
		Transport: roundTripper,
	}

	client := &http.Client{
		Transport: transport,
	}

	service, err := youtube.New(client)

	if err != nil {
		return nil, err
	}

	return &Service{httpClient: client, youTubeService: service}, nil
}

func (service Service) SearchByID(videoId string) (*catalog.VideoRecord, error) {
	call := service.getVideosListCall(videoId)

	response, err := call.Do()

	if err != nil {
		return nil, err
	}

	if cap(response.Items) == 0 {
		return nil, &catalog.NoSuchVideoError{videoId}
	}

	videoId = response.Items[0].Id
	videoKey := &catalog.VideoKey{videoId, catalog.YouTube}

	publishedTime := parsePublishedAt(response.Items[0].Snippet.PublishedAt)

	title := response.Items[0].Snippet.Title

	videoRecord := catalog.NewVideoRecord(videoKey, publishedTime, title)

	return videoRecord, nil
}

func (service Service) SearchByTitle(title string, maxResults int64) ([]catalog.VideoRecord, error) {
	call := service.getSearchListCall(title, maxResults)

	response, err := call.Do()

	if err != nil {
		return nil, err
	}

	items := response.Items
	itemCount := cap(items)

	var videoRecords []catalog.VideoRecord

	if itemCount == 0 {
		videoRecords = []catalog.VideoRecord{}
	} else {
		videoRecords = make([]catalog.VideoRecord, itemCount)

		for index, item := range items {
			videoKey := &catalog.VideoKey{item.Id.VideoId, catalog.YouTube}

			publishedTime := parsePublishedAt(item.Snippet.PublishedAt)
			title := item.Snippet.Title

			videoRecord := catalog.NewVideoRecord(videoKey, publishedTime, title)

			videoRecords[index] = *videoRecord
		}
	}

	return videoRecords, nil
}

func (service Service) getSearchListCall(searchTerm string, maxResults int64) *youtube.SearchListCall {
	call := service.youTubeService.Search.List(_PART)

	call.Q(searchTerm)
	call.MaxResults(maxResults)

	return call
}

func (service Service) getVideosListCall(videoId string) *youtube.VideosListCall {
	call := service.youTubeService.Videos.List(_PART)

	return call.Id(videoId)
}

func init() {
	developerKey = config.GetValue("YouTubeDevKey")
}

func parsePublishedAt(publishedAt string) time.Time {
	if publishedTime, err := time.Parse(time.RFC3339Nano, publishedAt); err == nil {
		return publishedTime
	} else {
		return time.Unix(0, 0)
	}
}

func setDeveloperKey(devKey string) {
	developerKey = devKey
}
<commit_msg>Issue #10: Fix dev key initialization bug<commit_after>//
// Copyright (C) 2015-present  Laszlo Csontos
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>. */
//

package youtube

import (
	"net/http"
	"time"

	"google.golang.org/api/googleapi/transport"
	"google.golang.org/api/youtube/v3"

	"github.com/lcsontos/uwatch/catalog"
	"github.com/lcsontos/uwatch/config"
)

const _PART = "id,snippet"

type Service struct {
	httpClient     *http.Client
	youTubeService *youtube.Service
}

var developerKey string

func New() (*Service, error) {
	service, err := NewWithRoundTripper(nil)

	return service, err
}

func NewWithRoundTripper(roundTripper http.RoundTripper) (*Service, error) {
	if developerKey == "" {
		developerKey = config.GetValue("YouTubeDevKey")
	}

	transport := &transport.APIKey{
		Key:       developerKey,
		Transport: roundTripper,
	}

	client := &http.Client{
		Transport: transport,
	}

	service, err := youtube.New(client)

	if err != nil {
		return nil, err
	}

	return &Service{httpClient: client, youTubeService: service}, nil
}

func (service Service) SearchByID(videoId string) (*catalog.VideoRecord, error) {
	call := service.getVideosListCall(videoId)

	response, err := call.Do()

	if err != nil {
		return nil, err
	}

	if cap(response.Items) == 0 {
		return nil, &catalog.NoSuchVideoError{videoId}
	}

	videoId = response.Items[0].Id
	videoKey := &catalog.VideoKey{videoId, catalog.YouTube}

	publishedTime := parsePublishedAt(response.Items[0].Snippet.PublishedAt)

	title := response.Items[0].Snippet.Title

	videoRecord := catalog.NewVideoRecord(videoKey, publishedTime, title)

	return videoRecord, nil
}

func (service Service) SearchByTitle(title string, maxResults int64) ([]catalog.VideoRecord, error) {
	call := service.getSearchListCall(title, maxResults)

	response, err := call.Do()

	if err != nil {
		return nil, err
	}

	items := response.Items
	itemCount := cap(items)

	var videoRecords []catalog.VideoRecord

	if itemCount == 0 {
		videoRecords = []catalog.VideoRecord{}
	} else {
		videoRecords = make([]catalog.VideoRecord, itemCount)

		for index, item := range items {
			videoKey := &catalog.VideoKey{item.Id.VideoId, catalog.YouTube}

			publishedTime := parsePublishedAt(item.Snippet.PublishedAt)
			title := item.Snippet.Title

			videoRecord := catalog.NewVideoRecord(videoKey, publishedTime, title)

			videoRecords[index] = *videoRecord
		}
	}

	return videoRecords, nil
}

func (service Service) getSearchListCall(searchTerm string, maxResults int64) *youtube.SearchListCall {
	call := service.youTubeService.Search.List(_PART)

	call.Q(searchTerm)
	call.MaxResults(maxResults)

	return call
}

func (service Service) getVideosListCall(videoId string) *youtube.VideosListCall {
	call := service.youTubeService.Videos.List(_PART)

	return call.Id(videoId)
}

func parsePublishedAt(publishedAt string) time.Time {
	if publishedTime, err := time.Parse(time.RFC3339Nano, publishedAt); err == nil {
		return publishedTime
	} else {
		return time.Unix(0, 0)
	}
}

func setDeveloperKey(devKey string) {
	developerKey = devKey
}
<|endoftext|>


<commit_before>package extract

// The following code was sourced and modified from the
// https://github.com/andrew-d/goscrape package governed by MIT license.

import (
	"bytes"
	"errors"
	"fmt"
	"regexp"

	"github.com/slotix/dataflowkit/log"

	"net/url"

	"github.com/PuerkitoBio/goquery"
	"github.com/sirupsen/logrus"
	"golang.org/x/net/html"
)

var logger *logrus.Logger

func init() {
	logger = log.NewLogger()
}

// The Extractor interface represents something that can extract data from
// a selection.
type Extractor interface {
	// Extract some data from the given Selection and return it.  The returned
	// data should be encodable - i.e. passing it to json.Marshal should succeed.
	// If the returned data is nil, then the output from this part will not be
	// included.
	//
	// If this function returns an error, then the scrape is aborted.
	Extract(*goquery.Selection) (interface{}, error)
}

// Const is an Extractor that returns a constant value.
type Const struct {
	// The value to return when the Extract() function is called.
	Val interface{}
}

// Extract returns Const value.
func (e Const) Extract(sel *goquery.Selection) (interface{}, error) {
	return e.Val, nil
}

var _ Extractor = Const{}

// Text is an Extractor that returns the combined text contents of
// the given selection.
type Text struct {
	// If text is empty in the selection, then return the empty string from Extract,
	// instead of 'nil'.  This signals that the result of this Part
	// should be included to the results, as opposed to omitting the
	// empty string.
	IncludeIfEmpty bool
}

// Extract returns Text value from specified selection.
func (e Text) Extract(sel *goquery.Selection) (interface{}, error) {
	results := []string{}

	sel.Each(func(i int, s *goquery.Selection) {
		results = append(results, s.Text())
	})

	if len(results) == 0 && !e.IncludeIfEmpty {
		return nil, nil
	}

	if len(results) == 1 {
		return results[0], nil
	}

	return results, nil
}

var _ Extractor = Text{}

// Html extracts and returns the HTML from inside each part of the
// given selection, as a string.
//
// Note that this results in what is effectively the innerHTML of the element -
// i.e. if our selection consists of ["
ONE
", "
TWO
"]
// then the output will be: "ONETWO".
//
// The return type is a string of all the inner HTML joined together.
type Html struct{}

// Extract returns HTML from specified selection.
func (e Html) Extract(sel *goquery.Selection) (interface{}, error) {
	var ret, h string
	var err error

	sel.EachWithBreak(func(i int, s *goquery.Selection) bool {
		h, err = s.Html()
		if err != nil {
			return false
		}

		ret += h
		return true
	})

	if err != nil {
		return nil, err
	}
	return ret, nil
}

var _ Extractor = Html{}

// OuterHtml extracts and returns the HTML of each part of the
// given selection, as a string.
//
// To illustrate, if our selection consists of
// ["
ONE
", "
TWO
"] then the output will be:
// "
ONE

TWO
".
//
// The return type is a string of all the outer HTML joined together.
type OuterHtml struct{}

// Extract returns OuterHtml from specified selection.
func (e OuterHtml) Extract(sel *goquery.Selection) (interface{}, error) {
	output := bytes.NewBufferString("")
	for _, node := range sel.Nodes {
		if err := html.Render(output, node); err != nil {
			return nil, err
		}
	}

	return output.String(), nil
}

var _ Extractor = OuterHtml{}

// Regex runs the given regex over the contents of each part in the
// given selection, and, for each match, extracts the given subexpression.
// The return type of the extractor is a list of string matches (i.e. []string).
type Regex struct {
	// The regular expression to match.  This regular expression must define
	// exactly one parenthesized subexpression (sometimes known as a "capturing
	// group"), which will be extracted.
	Regex *regexp.Regexp
	// The subexpression of the regex to match.  If this value is not set, and if
	// the given regex has more than one subexpression, an error will be thrown.
	Subexpression int

	// When OnlyText is true, only run the given regex over the text contents of
	// each part in the selection, as opposed to the HTML contents.
	OnlyText bool

	// By default, if there is only a single match, Regex will return
	// the match itself (as opposed to an array containing the single match).
	// Set AlwaysReturnList to true to disable this behaviour, ensuring that the
	// Extract function always returns an array.
	AlwaysReturnList bool

	// If no matches of the provided regex could be extracted, then return the empty list
	// from Extract, instead of 'nil'.  This signals that the result of
	// this Part should be included to the results, as opposed to
	// omitting the empty list.
	IncludeIfEmpty bool
}

// Extract returns Regex'ed  value from specified selection.
func (e Regex) Extract(sel *goquery.Selection) (interface{}, error) {
	if e.Regex == nil {
		return nil, errors.New("no regex given")
	}
	if e.Regex.NumSubexp() == 0 {
		return nil, errors.New("regex has no subexpressions")
	}

	var subexp int
	if e.Subexpression == 0 {
		if e.Regex.NumSubexp() != 1 {
			e := fmt.Errorf(
				"regex has more than one subexpression (%d), but which to "+
					"extract was not specified",
				e.Regex.NumSubexp())
			return nil, e
		}

		subexp = 1
	} else {
		subexp = e.Subexpression
	}

	results := []string{}

	// For each part in the selector...
	var err error
	sel.EachWithBreak(func(i int, s *goquery.Selection) bool {
		var contents string
		if e.OnlyText {
			contents = s.Text()
		} else {
			contents, err = s.Html()
			if err != nil {
				return false
			}
		}

		ret := e.Regex.FindAllStringSubmatch(contents, -1)

		// A return value of nil == no match
		if ret == nil {
			return true
		}

		// For each regex match...
		for _, submatches := range ret {
			// The 0th entry will be the match of the entire string.  The 1st
			// entry will be the first capturing group, which is what we want to
			// extract.
			if len(submatches) > 1 {
				results = append(results, submatches[subexp])
			}
		}

		return true
	})

	if err != nil {
		return nil, err
	}
	if len(results) == 0 && !e.IncludeIfEmpty {
		return nil, nil
	}
	if len(results) == 1 && !e.AlwaysReturnList {
		return results[0], nil
	}

	return results, nil
}

/*
func (e Regex) FillParams(m map[string]interface{}) error {
	err := FillStruct(m, &e)
	if err != nil {
		return err
	}
	regExp := m["regexp"]
	e.Regex = regexp.MustCompile(regExp.(string))
	logger.Println(e)
	return nil
}
*/
var _ Extractor = Regex{}

// Attr extracts the value of a given HTML attribute from each part
// in the selection, and returns them as a list.
// The return type of the extractor is a list of attribute values (i.e. []string).
type Attr struct {
	// The HTML attribute to extract from each part.
	Attr string
	//BaseURL specifies the base URL to use for all relative URLs contained within a document.
	BaseURL string
	// By default, if there is only a single attribute extracted, AttrExtractor
	// will return the match itself (as opposed to an array containing the single
	// match). Set AlwaysReturnList to true to disable this behaviour, ensuring
	// that the Extract function always returns an array.
	AlwaysReturnList bool

	// If no parts with this attribute are found, then return the empty list from
	// Extract, instead of  'nil'.  This signals that the result of this
	// Part should be included to the results, as opposed to omitting
	// the empty list.
	IncludeIfEmpty bool
}

// Extract returns Attr value from specified selection.
//Absolute URL will be returned for href and src attributes if relative URLs provided
func (e Attr) Extract(sel *goquery.Selection) (interface{}, error) {
	if len(e.Attr) == 0 {
		return nil, errors.New("no attribute provided")
	}
	results := []string{}
	sel.Each(func(i int, s *goquery.Selection) {
		if val, found := s.Attr(e.Attr); found {
			if e.Attr == "href" || e.Attr == "src" {
				u, err := url.Parse(val)
				if err != nil {
					logger.Error(err)
				}
				if u.Host == "" {
					val = e.BaseURL + "/" + val
				}
				results = append(results, val)
			}
		}
	})

	if len(results) == 0 && !e.IncludeIfEmpty {
		return nil, nil
	}
	if len(results) == 1 && !e.AlwaysReturnList {
		return results[0], nil
	}

	return results, nil
}

var _ Extractor = Attr{}

// Count extracts the count of parts that are matched and returns it.
type Count struct {
	// If no parts with this attribute are found, then return a number from
	// Extract, instead of 'nil'.  This signals that the result of this
	// Part should be included to the results, as opposed to omitting
	// the empty list.
	IncludeIfEmpty bool
}

// Extract returns length of elements in selection.
func (e Count) Extract(sel *goquery.Selection) (interface{}, error) {
	l := sel.Length()
	if l == 0 && !e.IncludeIfEmpty {
		return nil, nil
	}

	return l, nil
}

var _ Extractor = Count{}

// Link is an Extractor that returns the combined text contents and  Attr{Attr: "href"} of
// the given selection
type Link struct {
	// If no parts with this attribute are found, then return the empty list from
	// Extract, instead of  'nil'.  This signals that the result of this
	// Part should be included to the results, as opposed to omitting
	// the empty list.
	IncludeIfEmpty bool
	// By default, if there is only a single attribute extracted, AttrExtractor
	// will return the match itself (as opposed to an array containing the single
	// match). Set AlwaysReturnList to true to disable this behaviour, ensuring
	// that the Extract function always returns an array.
	AlwaysReturnList bool
	Text
	Href Attr
}

//Extract returns maps of links including text and href attributes.
//It is not used as we need these values separated as a result. Own Extract methods for Text and Attr are used instead.
/* func (e Link) Extract(sel *goquery.Selection) (interface{}, error) {
	//we need to keep an order of links.
	//map structure doesn't guarantee an order of items when iterating through map
	links := []map[string]string{}
	sel.Each(func(i int, s *goquery.Selection) {
		if val, found := s.Attr("href"); found {
			link := map[string]string{}
			link[s.Text()] = val
			links = append(links, link)
		}

	})

	if len(links) == 0 && !e.IncludeIfEmpty {
		return nil, nil
	}

	if len(links) == 1 {
		return links[0], nil
	}

	if len(links) == 1 && !e.AlwaysReturnList {
		return links[0], nil
	}

	return links, nil
}

var _ Extractor = Link{} */

// Image is an Extractor that returns the combined Src and Alt attributes of the given Image
type Image struct {
	// If no parts with this attribute are found, then return the empty list from
	// Extract, instead of  'nil'.  This signals that the result of this
	// Part should be included to the results, as opposed to omitting
	// the empty list.
	IncludeIfEmpty bool
	// By default, if there is only a single attribute extracted, AttrExtractor
	// will return the match itself (as opposed to an array containing the single
	// match). Set AlwaysReturnList to true to disable this behaviour, ensuring
	// that the Extract function always returns an array.
	AlwaysReturnList bool
	Src              Attr
	Alt              Attr
}
<commit_msg>fixed double slash issue between base URL and the rest<commit_after>package extract

// The following code was sourced and modified from the
// https://github.com/andrew-d/goscrape package governed by MIT license.

import (
	"bytes"
	"errors"
	"fmt"
	"regexp"
	"strings"

	"github.com/slotix/dataflowkit/log"

	"net/url"

	"github.com/PuerkitoBio/goquery"
	"github.com/sirupsen/logrus"
	"golang.org/x/net/html"
)

var logger *logrus.Logger

func init() {
	logger = log.NewLogger()
}

// The Extractor interface represents something that can extract data from
// a selection.
type Extractor interface {
	// Extract some data from the given Selection and return it.  The returned
	// data should be encodable - i.e. passing it to json.Marshal should succeed.
	// If the returned data is nil, then the output from this part will not be
	// included.
	//
	// If this function returns an error, then the scrape is aborted.
	Extract(*goquery.Selection) (interface{}, error)
}

// Const is an Extractor that returns a constant value.
type Const struct {
	// The value to return when the Extract() function is called.
	Val interface{}
}

// Extract returns Const value.
func (e Const) Extract(sel *goquery.Selection) (interface{}, error) {
	return e.Val, nil
}

var _ Extractor = Const{}

// Text is an Extractor that returns the combined text contents of
// the given selection.
type Text struct {
	// If text is empty in the selection, then return the empty string from Extract,
	// instead of 'nil'.  This signals that the result of this Part
	// should be included to the results, as opposed to omitting the
	// empty string.
	IncludeIfEmpty bool
}

// Extract returns Text value from specified selection.
func (e Text) Extract(sel *goquery.Selection) (interface{}, error) {
	results := []string{}

	sel.Each(func(i int, s *goquery.Selection) {
		results = append(results, s.Text())
	})

	if len(results) == 0 && !e.IncludeIfEmpty {
		return nil, nil
	}

	if len(results) == 1 {
		return results[0], nil
	}

	return results, nil
}

var _ Extractor = Text{}

// Html extracts and returns the HTML from inside each part of the
// given selection, as a string.
//
// Note that this results in what is effectively the innerHTML of the element -
// i.e. if our selection consists of ["
ONE
", "
TWO
"]
// then the output will be: "ONETWO".
//
// The return type is a string of all the inner HTML joined together.
type Html struct{}

// Extract returns HTML from specified selection.
func (e Html) Extract(sel *goquery.Selection) (interface{}, error) {
	var ret, h string
	var err error

	sel.EachWithBreak(func(i int, s *goquery.Selection) bool {
		h, err = s.Html()
		if err != nil {
			return false
		}

		ret += h
		return true
	})

	if err != nil {
		return nil, err
	}
	return ret, nil
}

var _ Extractor = Html{}

// OuterHtml extracts and returns the HTML of each part of the
// given selection, as a string.
//
// To illustrate, if our selection consists of
// ["
ONE
", "
TWO
"] then the output will be:
// "
ONE

TWO
".
//
// The return type is a string of all the outer HTML joined together.
type OuterHtml struct{}

// Extract returns OuterHtml from specified selection.
func (e OuterHtml) Extract(sel *goquery.Selection) (interface{}, error) {
	output := bytes.NewBufferString("")
	for _, node := range sel.Nodes {
		if err := html.Render(output, node); err != nil {
			return nil, err
		}
	}

	return output.String(), nil
}

var _ Extractor = OuterHtml{}

// Regex runs the given regex over the contents of each part in the
// given selection, and, for each match, extracts the given subexpression.
// The return type of the extractor is a list of string matches (i.e. []string).
type Regex struct {
	// The regular expression to match.  This regular expression must define
	// exactly one parenthesized subexpression (sometimes known as a "capturing
	// group"), which will be extracted.
	Regex *regexp.Regexp
	// The subexpression of the regex to match.  If this value is not set, and if
	// the given regex has more than one subexpression, an error will be thrown.
	Subexpression int

	// When OnlyText is true, only run the given regex over the text contents of
	// each part in the selection, as opposed to the HTML contents.
	OnlyText bool

	// By default, if there is only a single match, Regex will return
	// the match itself (as opposed to an array containing the single match).
	// Set AlwaysReturnList to true to disable this behaviour, ensuring that the
	// Extract function always returns an array.
	AlwaysReturnList bool

	// If no matches of the provided regex could be extracted, then return the empty list
	// from Extract, instead of 'nil'.  This signals that the result of
	// this Part should be included to the results, as opposed to
	// omitting the empty list.
	IncludeIfEmpty bool
}

// Extract returns Regex'ed  value from specified selection.
func (e Regex) Extract(sel *goquery.Selection) (interface{}, error) {
	if e.Regex == nil {
		return nil, errors.New("no regex given")
	}
	if e.Regex.NumSubexp() == 0 {
		return nil, errors.New("regex has no subexpressions")
	}

	var subexp int
	if e.Subexpression == 0 {
		if e.Regex.NumSubexp() != 1 {
			e := fmt.Errorf(
				"regex has more than one subexpression (%d), but which to "+
					"extract was not specified",
				e.Regex.NumSubexp())
			return nil, e
		}

		subexp = 1
	} else {
		subexp = e.Subexpression
	}

	results := []string{}

	// For each part in the selector...
	var err error
	sel.EachWithBreak(func(i int, s *goquery.Selection) bool {
		var contents string
		if e.OnlyText {
			contents = s.Text()
		} else {
			contents, err = s.Html()
			if err != nil {
				return false
			}
		}

		ret := e.Regex.FindAllStringSubmatch(contents, -1)

		// A return value of nil == no match
		if ret == nil {
			return true
		}

		// For each regex match...
		for _, submatches := range ret {
			// The 0th entry will be the match of the entire string.  The 1st
			// entry will be the first capturing group, which is what we want to
			// extract.
			if len(submatches) > 1 {
				results = append(results, submatches[subexp])
			}
		}

		return true
	})

	if err != nil {
		return nil, err
	}
	if len(results) == 0 && !e.IncludeIfEmpty {
		return nil, nil
	}
	if len(results) == 1 && !e.AlwaysReturnList {
		return results[0], nil
	}

	return results, nil
}

/*
func (e Regex) FillParams(m map[string]interface{}) error {
	err := FillStruct(m, &e)
	if err != nil {
		return err
	}
	regExp := m["regexp"]
	e.Regex = regexp.MustCompile(regExp.(string))
	logger.Println(e)
	return nil
}
*/
var _ Extractor = Regex{}

// Attr extracts the value of a given HTML attribute from each part
// in the selection, and returns them as a list.
// The return type of the extractor is a list of attribute values (i.e. []string).
type Attr struct {
	// The HTML attribute to extract from each part.
	Attr string
	//BaseURL specifies the base URL to use for all relative URLs contained within a document.
	BaseURL string
	// By default, if there is only a single attribute extracted, AttrExtractor
	// will return the match itself (as opposed to an array containing the single
	// match). Set AlwaysReturnList to true to disable this behaviour, ensuring
	// that the Extract function always returns an array.
	AlwaysReturnList bool

	// If no parts with this attribute are found, then return the empty list from
	// Extract, instead of  'nil'.  This signals that the result of this
	// Part should be included to the results, as opposed to omitting
	// the empty list.
	IncludeIfEmpty bool
}

// Extract returns Attr value from specified selection.
//Absolute URL will be returned for href and src attributes if relative URLs provided
func (e Attr) Extract(sel *goquery.Selection) (interface{}, error) {
	if len(e.Attr) == 0 {
		return nil, errors.New("no attribute provided")
	}
	results := []string{}
	sel.Each(func(i int, s *goquery.Selection) {
		if val, found := s.Attr(e.Attr); found {
			if e.Attr == "href" || e.Attr == "src" {
				u, err := url.Parse(val)
				if err != nil {
					logger.Error(err)
				}
				if u.Host == "" {
					val = strings.TrimSuffix(e.BaseURL, "/") + "/" + val
				}
				results = append(results, val)
			}
		}
	})

	if len(results) == 0 && !e.IncludeIfEmpty {
		return nil, nil
	}
	if len(results) == 1 && !e.AlwaysReturnList {
		return results[0], nil
	}

	return results, nil
}

var _ Extractor = Attr{}

// Count extracts the count of parts that are matched and returns it.
type Count struct {
	// If no parts with this attribute are found, then return a number from
	// Extract, instead of 'nil'.  This signals that the result of this
	// Part should be included to the results, as opposed to omitting
	// the empty list.
	IncludeIfEmpty bool
}

// Extract returns length of elements in selection.
func (e Count) Extract(sel *goquery.Selection) (interface{}, error) {
	l := sel.Length()
	if l == 0 && !e.IncludeIfEmpty {
		return nil, nil
	}

	return l, nil
}

var _ Extractor = Count{}

// Link is an Extractor that returns the combined text contents and  Attr{Attr: "href"} of
// the given selection
type Link struct {
	// If no parts with this attribute are found, then return the empty list from
	// Extract, instead of  'nil'.  This signals that the result of this
	// Part should be included to the results, as opposed to omitting
	// the empty list.
	IncludeIfEmpty bool
	// By default, if there is only a single attribute extracted, AttrExtractor
	// will return the match itself (as opposed to an array containing the single
	// match). Set AlwaysReturnList to true to disable this behaviour, ensuring
	// that the Extract function always returns an array.
	AlwaysReturnList bool
	Text
	Href Attr
}

//Extract returns maps of links including text and href attributes.
//It is not used as we need these values separated as a result. Own Extract methods for Text and Attr are used instead.
/* func (e Link) Extract(sel *goquery.Selection) (interface{}, error) {
	//we need to keep an order of links.
	//map structure doesn't guarantee an order of items when iterating through map
	links := []map[string]string{}
	sel.Each(func(i int, s *goquery.Selection) {
		if val, found := s.Attr("href"); found {
			link := map[string]string{}
			link[s.Text()] = val
			links = append(links, link)
		}

	})

	if len(links) == 0 && !e.IncludeIfEmpty {
		return nil, nil
	}

	if len(links) == 1 {
		return links[0], nil
	}

	if len(links) == 1 && !e.AlwaysReturnList {
		return links[0], nil
	}

	return links, nil
}

var _ Extractor = Link{} */

// Image is an Extractor that returns the combined Src and Alt attributes of the given Image
type Image struct {
	// If no parts with this attribute are found, then return the empty list from
	// Extract, instead of  'nil'.  This signals that the result of this
	// Part should be included to the results, as opposed to omitting
	// the empty list.
	IncludeIfEmpty bool
	// By default, if there is only a single attribute extracted, AttrExtractor
	// will return the match itself (as opposed to an array containing the single
	// match). Set AlwaysReturnList to true to disable this behaviour, ensuring
	// that the Extract function always returns an array.
	AlwaysReturnList bool
	Src              Attr
	Alt              Attr
}
<|endoftext|>


<commit_before>package genutil

import (
	"bytes"
	"code.google.com/p/go.tools/go/types"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"honnef.co/go/importer"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

// Package represents a parsed package with all its
// dependencies. Use NewPackage to create a Package.
type Package struct {
	*types.Package
	dir string
}

// Dir returns the package source directory.
func (p *Package) Dir() string {
	return p.dir
}

type _package struct {
	Path     string
	fset     *token.FileSet
	astFiles []*ast.File
	files    map[string]*file
}

type file struct {
	fset  *token.FileSet
	name  string
	ast   *ast.File
	lines [][]byte
}

// NewPackage returns a new Package struct, which can be
// used to generate code related to the package. The package
// might be given as either an absolute path or an import path.
// If the package can't be found or the package is not compilable,
// this function returns an error.
func NewPackage(path string) (*Package, error) {
	p := &_package{Path: path, fset: token.NewFileSet()}
	pkg, err := findPackage(path)
	if err != nil {
		return nil, fmt.Errorf("could not find package: %s", err)
	}
	fileNames := packageFiles(pkg)
	if len(fileNames) == 0 {
		return nil, fmt.Errorf("no go files")
	}
	p.astFiles = make([]*ast.File, len(fileNames))
	p.files = make(map[string]*file, len(fileNames))

	for ii, v := range fileNames {
		f, err := parseFile(p.fset, v)
		if err != nil {
			return nil, fmt.Errorf("could not parse %s: %s", v, err)
		}
		p.files[v] = f
		p.astFiles[ii] = f.ast
	}
	imp := importer.New()
	imp.Config.UseGcFallback = true
	context := types.Config{
		Import: imp.Import,
	}
	ipath := pkg.ImportPath
	if ipath == "." {
		// Check won't accept a "." import
		abs, err := filepath.Abs(pkg.Dir)
		if err != nil {
			return nil, err
		}
		for _, v := range strings.Split(build.Default.GOPATH, ":") {
			src := filepath.Join(v, "src")
			if strings.HasPrefix(abs, src) {
				ipath = abs[len(src)+1:]
				break
			}
		}
	}
	tpkg, err := context.Check(ipath, p.fset, p.astFiles, nil)
	if err != nil {
		return nil, fmt.Errorf("error checking package: %s", err)
	}
	return &Package{
		Package: tpkg,
		dir:     pkg.Dir,
	}, nil
}

func findPackage(path string) (*build.Package, error) {
	ctx := build.Default
	ctx.CgoEnabled = true
	p, err := ctx.Import(path, ".", 0)
	if err == nil {
		return p, err
	}
	return ctx.ImportDir(path, 0)
}

func packageFiles(pkg *build.Package) []string {
	var files []string
	for _, v := range pkg.GoFiles {
		files = append(files, filepath.Join(pkg.Dir, v))
	}
	for _, v := range pkg.CgoFiles {
		files = append(files, filepath.Join(pkg.Dir, v))
	}
	return files
}

func parseFile(fset *token.FileSet, fileName string) (f *file, err error) {
	rd, err := os.Open(fileName)
	if err != nil {
		return f, err
	}
	defer rd.Close()

	data, err := ioutil.ReadAll(rd)
	if err != nil {
		return f, err
	}

	astFile, err := parser.ParseFile(fset, fileName, bytes.NewReader(data), parser.ParseComments)
	if err != nil {
		return f, fmt.Errorf("could not parse: %s", err)
	}

	lines := bytes.Split(data, []byte("\n"))
	f = &file{fset: fset, name: fileName, ast: astFile, lines: lines}
	return f, nil
}
<commit_msg>Add methods for listing the named types in a Package<commit_after>package genutil

import (
	"bytes"
	"code.google.com/p/go.tools/go/types"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"honnef.co/go/importer"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

// Package represents a parsed package with all its
// dependencies. Use NewPackage to create a Package.
type Package struct {
	*types.Package
	dir string
}

// Dir returns the package source directory.
func (p *Package) Dir() string {
	return p.dir
}

func (p *Package) types(exported bool, include *regexp.Regexp, exclude *regexp.Regexp) []*types.Named {
	var packageTypes []*types.Named
	scope := p.Scope()
	prefix := p.Name() + "."
	for _, v := range scope.Names() {
		obj := scope.Lookup(v)
		if exported && !obj.IsExported() {
			continue
		}
		if _, ok := obj.(*types.Const); ok {
			continue
		}
		if _, ok := obj.(*types.Var); ok {
			continue
		}
		if named, ok := obj.Type().(*types.Named); ok {
			name := named.String()
			if strings.HasPrefix(name, prefix) {
				local := name[len(prefix):]
				if exclude != nil && exclude.MatchString(local) {
					continue
				}
				if include != nil && !include.MatchString(local) {
					continue
				}
				packageTypes = append(packageTypes, named)
			}
		}
	}
	return packageTypes
}

// Types returns the types declared in the package which match the required constraints.
// If excluded != nil, any type matching it gets excluded. If include != nil, only types
// matching it are returned.
func (p *Package) Types(include *regexp.Regexp, exclude *regexp.Regexp) []*types.Named {
	return p.types(false, include, exclude)
}

// ExportedTypes works like Types, but only returns types that are exported.
func (p *Package) ExportedTypes(include *regexp.Regexp, exclude *regexp.Regexp) []*types.Named {
	return p.types(true, include, exclude)
}

type _package struct {
	Path     string
	fset     *token.FileSet
	astFiles []*ast.File
	files    map[string]*file
}

type file struct {
	fset  *token.FileSet
	name  string
	ast   *ast.File
	lines [][]byte
}

// NewPackage returns a new Package struct, which can be
// used to generate code related to the package. The package
// might be given as either an absolute path or an import path.
// If the package can't be found or the package is not compilable,
// this function returns an error.
func NewPackage(path string) (*Package, error) {
	p := &_package{Path: path, fset: token.NewFileSet()}
	pkg, err := findPackage(path)
	if err != nil {
		return nil, fmt.Errorf("could not find package: %s", err)
	}
	fileNames := packageFiles(pkg)
	if len(fileNames) == 0 {
		return nil, fmt.Errorf("no go files")
	}
	p.astFiles = make([]*ast.File, len(fileNames))
	p.files = make(map[string]*file, len(fileNames))

	for ii, v := range fileNames {
		f, err := parseFile(p.fset, v)
		if err != nil {
			return nil, fmt.Errorf("could not parse %s: %s", v, err)
		}
		p.files[v] = f
		p.astFiles[ii] = f.ast
	}
	imp := importer.New()
	imp.Config.UseGcFallback = true
	context := types.Config{
		Import: imp.Import,
	}
	ipath := pkg.ImportPath
	if ipath == "." {
		// Check won't accept a "." import
		abs, err := filepath.Abs(pkg.Dir)
		if err != nil {
			return nil, err
		}
		for _, v := range strings.Split(build.Default.GOPATH, ":") {
			src := filepath.Join(v, "src")
			if strings.HasPrefix(abs, src) {
				ipath = abs[len(src)+1:]
				break
			}
		}
	}
	tpkg, err := context.Check(ipath, p.fset, p.astFiles, nil)
	if err != nil {
		return nil, fmt.Errorf("error checking package: %s", err)
	}
	return &Package{
		Package: tpkg,
		dir:     pkg.Dir,
	}, nil
}

func findPackage(path string) (*build.Package, error) {
	ctx := build.Default
	ctx.CgoEnabled = true
	p, err := ctx.Import(path, ".", 0)
	if err == nil {
		return p, err
	}
	return ctx.ImportDir(path, 0)
}

func packageFiles(pkg *build.Package) []string {
	var files []string
	for _, v := range pkg.GoFiles {
		files = append(files, filepath.Join(pkg.Dir, v))
	}
	for _, v := range pkg.CgoFiles {
		files = append(files, filepath.Join(pkg.Dir, v))
	}
	return files
}

func parseFile(fset *token.FileSet, fileName string) (f *file, err error) {
	rd, err := os.Open(fileName)
	if err != nil {
		return f, err
	}
	defer rd.Close()

	data, err := ioutil.ReadAll(rd)
	if err != nil {
		return f, err
	}

	astFile, err := parser.ParseFile(fset, fileName, bytes.NewReader(data), parser.ParseComments)
	if err != nil {
		return f, fmt.Errorf("could not parse: %s", err)
	}

	lines := bytes.Split(data, []byte("\n"))
	f = &file{fset: fset, name: fileName, ast: astFile, lines: lines}
	return f, nil
}
<|endoftext|>


<commit_before>package gnosis

// package file will contain MarkdownHandler and RawHandler to handle incoming requests
// Whole thing needs to be written

import (
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"

	"github.com/JackKnifed/blackfriday"
)

const (
	bodyHtmlFlags = 0 |
		blackfriday.HTML_USE_XHTML |
		blackfriday.HTML_USE_SMARTYPANTS |
		blackfriday.HTML_SMARTYPANTS_FRACTIONS |
		blackfriday.HTML_SMARTYPANTS_LATEX_DASHES

	bodyExtensions = 0 |
		blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
		blackfriday.EXTENSION_TABLES |
		blackfriday.EXTENSION_FENCED_CODE |
		blackfriday.EXTENSION_AUTOLINK |
		blackfriday.EXTENSION_STRIKETHROUGH |
		blackfriday.EXTENSION_SPACE_HEADERS |
		blackfriday.EXTENSION_AUTO_HEADER_IDS |
		blackfriday.EXTENSION_TITLEBLOCK

	tocHtmlFlags = 0 |
		blackfriday.HTML_USE_XHTML |
		blackfriday.HTML_SMARTYPANTS_FRACTIONS |
		blackfriday.HTML_SMARTYPANTS_LATEX_DASHES |
		blackfriday.HTML_TOC |
		blackfriday.HTML_OMIT_CONTENTS

	tocExtensions = 0 |
		blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
		blackfriday.EXTENSION_TABLES |
		blackfriday.EXTENSION_FENCED_CODE |
		blackfriday.EXTENSION_AUTOLINK |
		blackfriday.EXTENSION_STRIKETHROUGH |
		blackfriday.EXTENSION_SPACE_HEADERS |
		blackfriday.EXTENSION_AUTO_HEADER_IDS |
		blackfriday.EXTENSION_TITLEBLOCK
)

type WikiPage struct {
	Title string
	ToC   template.HTML
	Body  template.HTML
}

var wikiFilter = regexp.MustCompile("^(/([a-zA-Z0-9_ /]+/)?)([a-zA-Z0-9_ ]+)(\.md)?$")

func bodyParseMarkdown(input []byte) []byte {
	// set up the HTML renderer
	renderer := blackfriday.HtmlRenderer(bodyHtmlFlags, "", "")
	return blackfriday.Markdown(input, renderer, bodyExtensions)
}

func tocParseMarkdown(input []byte) []byte {
	// set up the HTML renderer
	renderer := blackfriday.HtmlRenderer(tocHtmlFlags, "", "")
	return blackfriday.Markdown(input, renderer, tocExtensions)
}

// Of note - this markdown handler is not a direct handler
func MarkdownHandler(responsePipe http.ResponseWriter, request *http.Request, serverConfig ServerSection) {

	var err error

	// break up the request parameters - for reference, regex is listed below
	filteredRequest := wikiFilter.FindStringSubmatch(request.URL.Path)

	// if there are no matches, the regex ovbiously didn't match up
	if filteredRequest == nil {
		log.Printf("null request [ %s ] improperly routed to wiki handler [ %s ]", request.URL.Path, serverConfig.Prefix)
		http.Error(responsePipe, "Request not allowed", 403)
	} else {
		if filteredRequest[1] != serverConfig.Prefix {
			log.Printf("request %s was improperly routed to wiki handler %s", request.URL.Path, serverConfig.Prefix)
			http.Error(responsePipe, err.Error(), 500)
		}

		pdata := new(PateMetadata)
		err = pdata.LoadPage(serverConfig.Path + filteredRequest[3] +".md")
		contents, err := ioutil.ReadFile(serverConfig.Path + filteredRequest[3] + ".md")
		// ## TODO ## need to add better error reporting for pages
		if err != nil {
			log.Printf("request [ %s ] points to an bad file target [ %s ]sent to server %s", request.URL.Path, filteredRequest[3], serverConfig.Prefix)
			http.Error(responsePipe, err.Error(), 404)
		}

		// parse any markdown in the input
		body := template.HTML(bodyParseMarkdown(pdata.Page))
		toc := template.HTML(tocParseMarkdown(pdata.Page))
		keywords := pdata.PrintKeywords()
		// ##TODO## need to move the topic URL to the config
		topics := pdata.PrintTopics("/topic/")

		// ##TODO## before you can use a template, you have to get the template lock to make sure you don't mess with someone else reading it
		response := WikiPage{Title: filteredRequest[3], ToC: toc, Body: body, Keywords: keywords, Topics: topics}
		err = allTemplates.ExecuteTemplate(responsePipe, serverConfig.Template, response)
		if err != nil {
			http.Error(responsePipe, err.Error(), 500)
		}
	}
}

var fileFilter = regexp.MustCompile("^(/([a-zA-Z0-9_ /]+/)?)([a-zA-Z0-9_ ]+)?(\\.)?([a-zA-Z0-9_ ]+)?")

func RawHandler(responsePipe http.ResponseWriter, request *http.Request, serverConfig ServerSection) {

	var err error

	// break up the request parameters - for reference, regex is listed below
	filteredRequest := fileFilter.FindStringSubmatch(request.URL.Path)

	// if there are no matches, the regex ovbiously didn't match up
	if filteredRequest == nil {
		log.Printf("null request [ %s ] improperly routed to file handler [ %s ]", request.URL.Path, serverConfig.Prefix)
		http.Error(responsePipe, "Request not allowed", 403)
	} else {
		if filteredRequest[1] != serverConfig.Prefix {
			log.Printf("request %s was improperly routed to wiki handler %s", request.URL.Path, serverConfig.Prefix)
			http.Error(responsePipe, err.Error(), 500)
		}

		contents, err := ioutil.ReadFile(serverConfig.Path + filteredRequest[3] + ".md")
		if err != nil {
			log.Printf("request [ %s ] points to an bad file target [ %s ]sent to server %s", request.URL.Path, filteredRequest[3], serverConfig.Prefix)
			http.Error(responsePipe, err.Error(), 404)
		}

		_, err = responsePipe.Write([]byte(contents))
		if err != nil {
			http.Error(responsePipe, err.Error(), 500)
		}
	}
}

func MakeHandler(handlerConfig ServerSection) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		switch handlerConfig.ServerType {
		case "markdown":
			MarkdownHandler(w, r, handlerConfig)
		case "raw":
			RawHandler(w, r, handlerConfig)
		default:
			log.Printf("Bad server type [%s]", handlerConfig.ServerType)
		}
	}
}
<commit_msg>added stuff to the markdown handler to limit the requests based on the tags<commit_after>package gnosis

// package file will contain MarkdownHandler and RawHandler to handle incoming requests
// Whole thing needs to be written

import (
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"

	"github.com/JackKnifed/blackfriday"
)

const (
	bodyHtmlFlags = 0 |
		blackfriday.HTML_USE_XHTML |
		blackfriday.HTML_USE_SMARTYPANTS |
		blackfriday.HTML_SMARTYPANTS_FRACTIONS |
		blackfriday.HTML_SMARTYPANTS_LATEX_DASHES

	bodyExtensions = 0 |
		blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
		blackfriday.EXTENSION_TABLES |
		blackfriday.EXTENSION_FENCED_CODE |
		blackfriday.EXTENSION_AUTOLINK |
		blackfriday.EXTENSION_STRIKETHROUGH |
		blackfriday.EXTENSION_SPACE_HEADERS |
		blackfriday.EXTENSION_AUTO_HEADER_IDS |
		blackfriday.EXTENSION_TITLEBLOCK

	tocHtmlFlags = 0 |
		blackfriday.HTML_USE_XHTML |
		blackfriday.HTML_SMARTYPANTS_FRACTIONS |
		blackfriday.HTML_SMARTYPANTS_LATEX_DASHES |
		blackfriday.HTML_TOC |
		blackfriday.HTML_OMIT_CONTENTS

	tocExtensions = 0 |
		blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
		blackfriday.EXTENSION_TABLES |
		blackfriday.EXTENSION_FENCED_CODE |
		blackfriday.EXTENSION_AUTOLINK |
		blackfriday.EXTENSION_STRIKETHROUGH |
		blackfriday.EXTENSION_SPACE_HEADERS |
		blackfriday.EXTENSION_AUTO_HEADER_IDS |
		blackfriday.EXTENSION_TITLEBLOCK
)

type WikiPage struct {
	Title string
	ToC   template.HTML
	Body  template.HTML
}

var wikiFilter = regexp.MustCompile("^(/([a-zA-Z0-9_ /]+/)?)([a-zA-Z0-9_ ]+)(\.md)?$")

func bodyParseMarkdown(input []byte) []byte {
	// set up the HTML renderer
	renderer := blackfriday.HtmlRenderer(bodyHtmlFlags, "", "")
	return blackfriday.Markdown(input, renderer, bodyExtensions)
}

func tocParseMarkdown(input []byte) []byte {
	// set up the HTML renderer
	renderer := blackfriday.HtmlRenderer(tocHtmlFlags, "", "")
	return blackfriday.Markdown(input, renderer, tocExtensions)
}

// Of note - this markdown handler is not a direct handler
func MarkdownHandler(responsePipe http.ResponseWriter, request *http.Request, serverConfig ServerSection) {

	var err error

	// break up the request parameters - for reference, regex is listed below
	filteredRequest := wikiFilter.FindStringSubmatch(request.URL.Path)

	// if there are no matches, the regex ovbiously didn't match up
	if filteredRequest == nil {
		log.Printf("null request [ %s ] improperly routed to wiki handler [ %s ]", request.URL.Path, serverConfig.Prefix)
		http.Error(responsePipe, "Request not allowed", 403)
	} else {
		if filteredRequest[1] != serverConfig.Prefix {
			log.Printf("request %s was improperly routed to wiki handler %s", request.URL.Path, serverConfig.Prefix)
			http.Error(responsePipe, err.Error(), 500)
		}

		pdata := new(PateMetadata)
		err = pdata.LoadPage(serverConfig.Path + filteredRequest[3] +".md")
		contents, err := ioutil.ReadFile(serverConfig.Path + filteredRequest[3] + ".md")
		// ## TODO ## need to add better error reporting for pages
		if err != nil {
			log.Printf("request [ %s ] points to an bad file target [ %s ]sent to server %s", request.URL.Path, filteredRequest[3], serverConfig.Prefix)
			http.Error(responsePipe, err.Error(), 404)
		}

		if pdata.MatchedTag(serverConfig.Restricted) {
			log.Printf("request [ %s ] was against a page with a restricted tag", request.URL.Path)
			http.Error(responsePipe, err.Error(), 403)
		}

		// parse any markdown in the input
		body := template.HTML(bodyParseMarkdown(pdata.Page))
		toc := template.HTML(tocParseMarkdown(pdata.Page))
		keywords := pdata.PrintKeywords()
		// ##TODO## need to move the topic URL to the config
		topics := pdata.PrintTopics("/topic/")

		// ##TODO## before you can use a template, you have to get the template lock to make sure you don't mess with someone else reading it
		response := WikiPage{Title: filteredRequest[3], ToC: toc, Body: body, Keywords: keywords, Topics: topics}
		err = allTemplates.ExecuteTemplate(responsePipe, serverConfig.Template, response)
		if err != nil {
			http.Error(responsePipe, err.Error(), 500)
		}
	}
}

var fileFilter = regexp.MustCompile("^(/([a-zA-Z0-9_ /]+/)?)([a-zA-Z0-9_ ]+)?(\\.)?([a-zA-Z0-9_ ]+)?")

func RawHandler(responsePipe http.ResponseWriter, request *http.Request, serverConfig ServerSection) {

	var err error

	// break up the request parameters - for reference, regex is listed below
	filteredRequest := fileFilter.FindStringSubmatch(request.URL.Path)

	// if there are no matches, the regex ovbiously didn't match up
	if filteredRequest == nil {
		log.Printf("null request [ %s ] improperly routed to file handler [ %s ]", request.URL.Path, serverConfig.Prefix)
		http.Error(responsePipe, "Request not allowed", 403)
	} else {
		if filteredRequest[1] != serverConfig.Prefix {
			log.Printf("request %s was improperly routed to wiki handler %s", request.URL.Path, serverConfig.Prefix)
			http.Error(responsePipe, err.Error(), 500)
		}

		contents, err := ioutil.ReadFile(serverConfig.Path + filteredRequest[3] + ".md")
		if err != nil {
			log.Printf("request [ %s ] points to an bad file target [ %s ]sent to server %s", request.URL.Path, filteredRequest[3], serverConfig.Prefix)
			http.Error(responsePipe, err.Error(), 404)
		}

		_, err = responsePipe.Write([]byte(contents))
		if err != nil {
			http.Error(responsePipe, err.Error(), 500)
		}
	}
}

func MakeHandler(handlerConfig ServerSection) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		switch handlerConfig.ServerType {
		case "markdown":
			MarkdownHandler(w, r, handlerConfig)
		case "raw":
			RawHandler(w, r, handlerConfig)
		default:
			log.Printf("Bad server type [%s]", handlerConfig.ServerType)
		}
	}
}
<|endoftext|>