<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"sync"

	"github.com/gin-gonic/gin"
)

const VERSION = "0.2.0"

type Response struct {
	Success bool   `json:"success"`
	Message string `json:"message"`
}

type StatusResponse struct {
	Running  bool   `json:"running"`            // True if player is running
	File     string `json:"file"`               // Path to current media file
	Name     string `json:"name"`               // Titleized filename
	Position string `json:"position,omitempty"` // Current position in the movie
	Duration string `json:"duration,omitempty"` // Movie duration
}

type FileEntry struct {
	Filename string `json:"filename"`
	IsDir    bool   `json:"directory"`
}

var (
	// Regular expression to match all supported video files
	RegexFormats = regexp.MustCompile(`.(avi|mpg|mov|flv|wmv|asf|mpeg|m4v|divx|mp4|mkv)$`)

	// Regular expression to convert filenames to titles
	RegexBrackets = regexp.MustCompile(`[\(\[\]\)]`)
	RegexYear     = regexp.MustCompile(`((19|20)[\d]{2})`)
	RegexEpisode  = regexp.MustCompile(`(?i)S[\d]+E[\d]+`)
	RegexJunk     = regexp.MustCompile(`(?i)(1080p|720p|3d|brrip|bluray|webrip|x264|aac)`)
	RegexSpace    = regexp.MustCompile(`\s{2,}`)

	// OMXPlayer control commands, these are piped via STDIN to omxplayer process
	Commands = map[string]string{
		"pause":             "p",            // Pause/continue playback
		"stop":              "q",            // Stop playback and exit
		"volume_up":         "+",            // Change volume by +3dB
		"volume_down":       "-",            // Change volume by -3dB
		"subtitles":         "s",            // Enable/disable subtitles
		"seek_back":         "\x1b\x5b\x44", // Seek -30 seconds
		"seek_back_fast":    "\x1b\x5b\x42", // Seek -600 second
		"seek_forward":      "\x1b\x5b\x43", // Seek +30 second
		"seek_forward_fast": "\x1b\x5b\x41", // Seek +600 seconds
	}

	MediaPath   string         // Path where all media files are stored
	OmxPath     string         // Path to omxplayer executable
	Omx         *exec.Cmd      // Child process for spawning omxplayer
	OmxIn       io.WriteCloser // Child process STDIN pipe to send commands
	Command     chan string    // Channel to pass along commands to the player routine
	CurrentFile string         // Currently playing media file name
	Zeroconf    bool           // Enable Zeroconf discovery
	Frontend    bool           // Serve frontend app
	stream      *Stream        // Current stream
)

func httpBrowse(c *gin.Context) {
	path := c.Request.FormValue("path")

	if path != "" {
		path = fmt.Sprintf("%s/%s", MediaPath, path)
	} else {
		path = MediaPath
	}

	c.JSON(200, scanPath(path))
}

func httpCommand(c *gin.Context) {
	val := c.Params.ByName("command")

	if _, ok := Commands[val]; !ok {
		c.JSON(400, Response{false, "Invalid command"})
		return
	}

	fmt.Println("Received command:", val)

	// Handle requested commmand
	Command <- val

	c.JSON(200, Response{true, "OK"})
}

func httpServe(c *gin.Context) {
	file := c.Request.URL.Query().Get("file")
	if file == "" {
		return
	}

	file = fmt.Sprintf("%s/%s", MediaPath, file)
	if !fileExists(file) {
		c.String(404, "Not found")
		return
	}

	if !omxCanPlay(file) {
		c.String(400, "Invalid format")
		return
	}

	http.ServeFile(c.Writer, c.Request, file)
}

func httpPlay(c *gin.Context) {
	if omxIsActive() {
		c.JSON(400, Response{false, "Player is already running"})
		return
	}

	file := c.Request.FormValue("file")
	if file == "" {
		c.JSON(400, Response{false, "File is required"})
		return
	}

	file = fmt.Sprintf("%s/%s", MediaPath, file)

	if !fileExists(file) {
		c.JSON(400, Response{false, "File does not exist"})
		return
	}

	if !omxCanPlay(file) {
		c.JSON(400, Response{false, "File cannot be played"})
		return
	}

	go omxPlay(file)

	c.JSON(200, Response{true, "OK"})
}

func httpRemoveFile(c *gin.Context) {
	file := strings.TrimSpace(c.Request.FormValue("file"))
	if file == "" {
		c.JSON(400, Response{false, "File is required"})
		return
	}

	fullPath := fmt.Sprintf("%s/%s", MediaPath, file)
	if !fileExists(fullPath) {
		c.JSON(400, Response{false, "File does not exist"})
		return
	}

	if err := os.RemoveAll(fullPath); err != nil {
		c.JSON(400, Response{false, err.Error()})
		return
	}

	c.JSON(200, Response{true, "OK"})
}

func httpStatus(c *gin.Context) {
	resp := StatusResponse{
		Running: omxIsActive(),
		File:    CurrentFile,
		Name:    fileToTitle(filepath.Base(CurrentFile)),
	}

	if stream != nil {
		resp.Duration = durationFromSeconds(stream.duration)
		resp.Position = stream.pos.String()
	}

	c.JSON(200, resp)
}

func httpIndex(c *gin.Context) {
	data, err := Asset("static/index.html")

	if err != nil {
		c.String(400, err.Error())
		return
	}

	c.Data(200, "text/html; charset=utf-8", data)
}

// Retrieve information about the host: uptime, storage, etc
func httpHost(c *gin.Context) {
	output := map[string]string{}
	commands := map[string]string{
		"os":      "uname -a",
		"uptime":  "uptime",
		"storage": "df -m",
		"memory":  "free -m",
	}

	lock := &sync.Mutex{}
	wg := &sync.WaitGroup{}

	wg.Add(len(commands))

	for k, v := range commands {
		go func(key, name string) {
			defer wg.Done()

			args := strings.Split(name, " ")

			out := bytes.NewBuffer(nil)
			cmd := exec.Command(args[0], args[1:]...)
			cmd.Stdout = out
			cmd.Stderr = out

			if err := cmd.Run(); err != nil {
				log.Println("Failed to execute command", k, err)

				lock.Lock()
				output[key] = "N/A"
				lock.Unlock()

				return
			}

			lock.Lock()
			output[key] = strings.TrimSpace(out.String())
			lock.Unlock()

		}(k, v)
	}

	wg.Wait()

	c.JSON(200, map[string]interface{}{
		"uptime":  output["uptime"],
		"os":      output["os"],
		"storage": parseStorageInfo(output["storage"]),
		"memory":  parseMemoryInfo(output["memory"]),
	})
}

func terminate(message string, code int) {
	fmt.Println(message)
	os.Exit(code)
}

func usage() {
	terminate("Usage: omxremote path/to/media/dir", 0)
}

func init() {
	var printVersion bool

	flag.StringVar(&MediaPath, "media", "./", "Path to media files")
	flag.BoolVar(&Frontend, "frontend", true, "Enable frontend applicaiton")
	flag.BoolVar(&Zeroconf, "zeroconf", true, "Enable service advertisement with Zeroconf")
	flag.BoolVar(&printVersion, "v", false, "Print version")
	flag.Parse()

	if printVersion {
		fmt.Printf("omxremote v%v\n", VERSION)
		os.Exit(0)
	}
}

func main() {
	// Expand media path if needed
	MediaPath = strings.Replace(MediaPath, "~", os.Getenv("HOME"), 1)

	// Get path from arguments and remove trailing slash
	MediaPath = strings.TrimRight(MediaPath, "/")

	if !fileExists(MediaPath) {
		terminate(fmt.Sprintf("Directory does not exist: %s", MediaPath), 1)
	}

	// Check if player is installed
	if omxDetect() != nil {
		terminate("omxplayer is not installed", 1)
	}

	// Make sure nothing is running
	omxCleanup()

	// Start a remote command listener
	go omxListen()

	// Start zeroconf service advertisement
	if Zeroconf {
		stopZeroconf := make(chan bool)
		go startZeroConfAdvertisement(stopZeroconf)
	}

	// Disable debugging mode
	gin.SetMode("release")

	// Setup HTTP server
	router := gin.Default()

	// Handle CORS
	router.Use(func(c *gin.Context) {
		c.Header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
		c.Header("Access-Control-Allow-Origin", "*")
		c.Header("Access-Control-Expose-Headers", "*")
	})

	// Server frontend application only if its enabled
	if Frontend == true {
		router.GET("/", httpIndex)
	}

	router.GET("/status", httpStatus)
	router.GET("/browse", httpBrowse)
	router.GET("/play", httpPlay)
	router.GET("/serve", httpServe)
	router.POST("/remove", httpRemoveFile)
	router.GET("/command/:command", httpCommand)
	router.GET("/host", httpHost)

	port := os.Getenv("PORT")
	if port == "" {
		port = "8080"
	}

	host := os.Getenv("HOST")
	if host == "" {
		host = "0.0.0.0"
	}

	fmt.Println("Starting server on " + host + ":" + port)
	router.Run(host + ":" + port)
}
<commit_msg>Add endpoint to reboot the system<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"sync"

	"github.com/gin-gonic/gin"
)

const VERSION = "0.2.0"

type Response struct {
	Success bool   `json:"success"`
	Message string `json:"message"`
}

type StatusResponse struct {
	Running  bool   `json:"running"`            // True if player is running
	File     string `json:"file"`               // Path to current media file
	Name     string `json:"name"`               // Titleized filename
	Position string `json:"position,omitempty"` // Current position in the movie
	Duration string `json:"duration,omitempty"` // Movie duration
}

type FileEntry struct {
	Filename string `json:"filename"`
	IsDir    bool   `json:"directory"`
}

var (
	// Regular expression to match all supported video files
	RegexFormats = regexp.MustCompile(`.(avi|mpg|mov|flv|wmv|asf|mpeg|m4v|divx|mp4|mkv)$`)

	// Regular expression to convert filenames to titles
	RegexBrackets = regexp.MustCompile(`[\(\[\]\)]`)
	RegexYear     = regexp.MustCompile(`((19|20)[\d]{2})`)
	RegexEpisode  = regexp.MustCompile(`(?i)S[\d]+E[\d]+`)
	RegexJunk     = regexp.MustCompile(`(?i)(1080p|720p|3d|brrip|bluray|webrip|x264|aac)`)
	RegexSpace    = regexp.MustCompile(`\s{2,}`)

	// OMXPlayer control commands, these are piped via STDIN to omxplayer process
	Commands = map[string]string{
		"pause":             "p",            // Pause/continue playback
		"stop":              "q",            // Stop playback and exit
		"volume_up":         "+",            // Change volume by +3dB
		"volume_down":       "-",            // Change volume by -3dB
		"subtitles":         "s",            // Enable/disable subtitles
		"seek_back":         "\x1b\x5b\x44", // Seek -30 seconds
		"seek_back_fast":    "\x1b\x5b\x42", // Seek -600 second
		"seek_forward":      "\x1b\x5b\x43", // Seek +30 second
		"seek_forward_fast": "\x1b\x5b\x41", // Seek +600 seconds
	}

	MediaPath   string         // Path where all media files are stored
	OmxPath     string         // Path to omxplayer executable
	Omx         *exec.Cmd      // Child process for spawning omxplayer
	OmxIn       io.WriteCloser // Child process STDIN pipe to send commands
	Command     chan string    // Channel to pass along commands to the player routine
	CurrentFile string         // Currently playing media file name
	Zeroconf    bool           // Enable Zeroconf discovery
	Frontend    bool           // Serve frontend app
	stream      *Stream        // Current stream
)

func httpBrowse(c *gin.Context) {
	path := c.Request.FormValue("path")

	if path != "" {
		path = fmt.Sprintf("%s/%s", MediaPath, path)
	} else {
		path = MediaPath
	}

	c.JSON(200, scanPath(path))
}

func httpCommand(c *gin.Context) {
	val := c.Params.ByName("command")

	if _, ok := Commands[val]; !ok {
		c.JSON(400, Response{false, "Invalid command"})
		return
	}

	fmt.Println("Received command:", val)

	// Handle requested commmand
	Command <- val

	c.JSON(200, Response{true, "OK"})
}

func httpServe(c *gin.Context) {
	file := c.Request.URL.Query().Get("file")
	if file == "" {
		return
	}

	file = fmt.Sprintf("%s/%s", MediaPath, file)
	if !fileExists(file) {
		c.String(404, "Not found")
		return
	}

	if !omxCanPlay(file) {
		c.String(400, "Invalid format")
		return
	}

	http.ServeFile(c.Writer, c.Request, file)
}

func httpPlay(c *gin.Context) {
	if omxIsActive() {
		c.JSON(400, Response{false, "Player is already running"})
		return
	}

	file := c.Request.FormValue("file")
	if file == "" {
		c.JSON(400, Response{false, "File is required"})
		return
	}

	file = fmt.Sprintf("%s/%s", MediaPath, file)

	if !fileExists(file) {
		c.JSON(400, Response{false, "File does not exist"})
		return
	}

	if !omxCanPlay(file) {
		c.JSON(400, Response{false, "File cannot be played"})
		return
	}

	go omxPlay(file)

	c.JSON(200, Response{true, "OK"})
}

func httpRemoveFile(c *gin.Context) {
	file := strings.TrimSpace(c.Request.FormValue("file"))
	if file == "" {
		c.JSON(400, Response{false, "File is required"})
		return
	}

	fullPath := fmt.Sprintf("%s/%s", MediaPath, file)
	if !fileExists(fullPath) {
		c.JSON(400, Response{false, "File does not exist"})
		return
	}

	if err := os.RemoveAll(fullPath); err != nil {
		c.JSON(400, Response{false, err.Error()})
		return
	}

	c.JSON(200, Response{true, "OK"})
}

func httpStatus(c *gin.Context) {
	resp := StatusResponse{
		Running: omxIsActive(),
		File:    CurrentFile,
		Name:    fileToTitle(filepath.Base(CurrentFile)),
	}

	if stream != nil {
		resp.Duration = durationFromSeconds(stream.duration)
		resp.Position = stream.pos.String()
	}

	c.JSON(200, resp)
}

func httpIndex(c *gin.Context) {
	data, err := Asset("static/index.html")

	if err != nil {
		c.String(400, err.Error())
		return
	}

	c.Data(200, "text/html; charset=utf-8", data)
}

// Retrieve information about the host: uptime, storage, etc
func httpHost(c *gin.Context) {
	output := map[string]string{}
	commands := map[string]string{
		"os":      "uname -a",
		"uptime":  "uptime",
		"storage": "df -m",
		"memory":  "free -m",
	}

	lock := &sync.Mutex{}
	wg := &sync.WaitGroup{}

	wg.Add(len(commands))

	for k, v := range commands {
		go func(key, name string) {
			defer wg.Done()

			args := strings.Split(name, " ")

			out := bytes.NewBuffer(nil)
			cmd := exec.Command(args[0], args[1:]...)
			cmd.Stdout = out
			cmd.Stderr = out

			if err := cmd.Run(); err != nil {
				log.Println("Failed to execute command", k, err)

				lock.Lock()
				output[key] = "N/A"
				lock.Unlock()

				return
			}

			lock.Lock()
			output[key] = strings.TrimSpace(out.String())
			lock.Unlock()

		}(k, v)
	}

	wg.Wait()

	c.JSON(200, map[string]interface{}{
		"uptime":  output["uptime"],
		"os":      output["os"],
		"storage": parseStorageInfo(output["storage"]),
		"memory":  parseMemoryInfo(output["memory"]),
	})
}

// Reboot the operating system
// POST /reboot
func httpReboot(c *gin.Context) {
	if err := exec.Command("sudo", "reboot").Run(); err != nil {
		c.JSON(400, Response{Success: false, Message: err.Error()})
		return
	}
	c.JSON(200, Response{Success: true})
}

func terminate(message string, code int) {
	fmt.Println(message)
	os.Exit(code)
}

func usage() {
	terminate("Usage: omxremote path/to/media/dir", 0)
}

func init() {
	var printVersion bool

	flag.StringVar(&MediaPath, "media", "./", "Path to media files")
	flag.BoolVar(&Frontend, "frontend", true, "Enable frontend applicaiton")
	flag.BoolVar(&Zeroconf, "zeroconf", true, "Enable service advertisement with Zeroconf")
	flag.BoolVar(&printVersion, "v", false, "Print version")
	flag.Parse()

	if printVersion {
		fmt.Printf("omxremote v%v\n", VERSION)
		os.Exit(0)
	}
}

func main() {
	// Expand media path if needed
	MediaPath = strings.Replace(MediaPath, "~", os.Getenv("HOME"), 1)

	// Get path from arguments and remove trailing slash
	MediaPath = strings.TrimRight(MediaPath, "/")

	if !fileExists(MediaPath) {
		terminate(fmt.Sprintf("Directory does not exist: %s", MediaPath), 1)
	}

	// Check if player is installed
	if omxDetect() != nil {
		terminate("omxplayer is not installed", 1)
	}

	// Make sure nothing is running
	omxCleanup()

	// Start a remote command listener
	go omxListen()

	// Start zeroconf service advertisement
	if Zeroconf {
		stopZeroconf := make(chan bool)
		go startZeroConfAdvertisement(stopZeroconf)
	}

	// Disable debugging mode
	gin.SetMode("release")

	// Setup HTTP server
	router := gin.Default()

	// Handle CORS
	router.Use(func(c *gin.Context) {
		c.Header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
		c.Header("Access-Control-Allow-Origin", "*")
		c.Header("Access-Control-Expose-Headers", "*")
	})

	// Server frontend application only if its enabled
	if Frontend == true {
		router.GET("/", httpIndex)
	}

	router.GET("/status", httpStatus)
	router.GET("/browse", httpBrowse)
	router.GET("/play", httpPlay)
	router.GET("/serve", httpServe)
	router.POST("/remove", httpRemoveFile)
	router.GET("/command/:command", httpCommand)
	router.GET("/host", httpHost)
	router.POST("/reboot", httpReboot)

	port := os.Getenv("PORT")
	if port == "" {
		port = "8080"
	}

	host := os.Getenv("HOST")
	if host == "" {
		host = "0.0.0.0"
	}

	fmt.Println("Starting server on " + host + ":" + port)
	router.Run(host + ":" + port)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/s-rah/onionscan/report"
	"github.com/s-rah/onionscan/scans"
	"h12.me/socks"
	"io/ioutil"
	"log"
	"net/http"
)

type OnionScan struct {
	TorProxyAddress string
	Client          *http.Client
}

func Configure(torProxyAddress string) *OnionScan {
	onionScan := new(OnionScan)
	onionScan.TorProxyAddress = torProxyAddress
	dialSocksProxy := socks.DialSocksProxy(socks.SOCKS5, onionScan.TorProxyAddress)
	transportConfig := &http.Transport{
		Dial: dialSocksProxy,
	}
	onionScan.Client = &http.Client{Transport: transportConfig}
	return onionScan
}

func (os *OnionScan) Scan(hiddenService string) (*report.OnionScanReport, error) {

	report := report.NewOnionScanReport(hiddenService)

	response, err := os.Client.Get("http://" + hiddenService)

	if err != nil {
		return report, err
	}

	// Initial Attempt at Resolving Server Type
	log.Printf("Attempting to Derive Server Type from Headers..\n")
	report.ServerVersion = response.Header.Get("Server")
	log.Printf("\tServer Version: %s\n", report.ServerVersion)

	// Initial attempt at grabbing X-Powered-By header response
	log.Printf("Attempting to grab additional header information..\n")
	report.ServerPoweredBy = response.Header.Get("X-Powered-By")
	log.Printf("\tX-Powered-By: %s\n", report.ServerPoweredBy)

	// Apache mod-status Check
	os.ScanPage(hiddenService, "/server-status", report, scans.ApacheModStatus)
	os.ScanPage(hiddenService, "/", report, scans.StandardPageScan)

	os.ScanPage(hiddenService, "/style", report, scans.CheckDirectoryListing)
	os.ScanPage(hiddenService, "/styles", report, scans.CheckDirectoryListing)
	os.ScanPage(hiddenService, "/css", report, scans.CheckDirectoryListing)
	os.ScanPage(hiddenService, "/uploads", report, scans.CheckDirectoryListing)
	os.ScanPage(hiddenService, "/images", report, scans.CheckDirectoryListing)
	os.ScanPage(hiddenService, "/img", report, scans.CheckDirectoryListing)
	os.ScanPage(hiddenService, "/static", report, scans.CheckDirectoryListing)

	// Lots of Wordpress installs which don't lock down directory listings
	os.ScanPage(hiddenService, "/wp-content/uploads", report, scans.CheckDirectoryListing)

	// Common with torshops created onions
	os.ScanPage(hiddenService, "/products", report, scans.CheckDirectoryListing)
	os.ScanPage(hiddenService, "/products/cat", report, scans.CheckDirectoryListing)

	return report, nil
}

func (os *OnionScan) ScanPage(hiddenService string, page string, report *report.OnionScanReport, f func(scans.Scanner, string, int, string, *report.OnionScanReport)) {
	response, err := os.Client.Get("http://" + hiddenService + page)
	if err != nil {
		log.Printf("Error connecting to %s%s %s\n", hiddenService, page, err)
		return
	}
	defer response.Body.Close()
	contents, _ := ioutil.ReadAll(response.Body)
	f(os, page, response.StatusCode, string(contents), report)
}
<commit_msg>Adding framework for different port scans<commit_after>package main

import (
	"github.com/s-rah/onionscan/report"
	"github.com/s-rah/onionscan/scans"
	"h12.me/socks"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

type OnionScan struct {
	TorProxyAddress string
	Client          *http.Client
}

func Configure(torProxyAddress string) *OnionScan {
	onionScan := new(OnionScan)
	onionScan.TorProxyAddress = torProxyAddress
	dialSocksProxy := socks.DialSocksProxy(socks.SOCKS5, onionScan.TorProxyAddress)
	transportConfig := &http.Transport{
		Dial: dialSocksProxy,
	}
	onionScan.Client = &http.Client{Transport: transportConfig}
	return onionScan
}

func (os *OnionScan) Scan(hiddenService string) (*report.OnionScanReport, error) {

	// Remove Extra Prefix
	// TODO: Add support for HTTPS?
	if strings.HasPrefix(hiddenService, "http://") {
		hiddenService = hiddenService[7:]
	}

	if strings.HasSuffix(hiddenService, "/") {
		hiddenService = hiddenService[0 : len(hiddenService)-1]
	}

	report := report.NewOnionScanReport(hiddenService)

	// It's Port Scanning Time.
	log.Printf("Checking %s ssh(22)\n", hiddenService)
	_, err := socks.DialSocksProxy(socks.SOCKS5, os.TorProxyAddress)("", hiddenService+":22")
	if err != nil {
		log.Printf("Failed to connect to service on port 22\n")
	} else {
		// TODO SSH Checking
	}

	log.Printf("Checking %s http(80)\n", hiddenService)
	// It's Port Scanning Time.
	_, err = socks.DialSocksProxy(socks.SOCKS5, os.TorProxyAddress)("", hiddenService+":80")
	if err != nil {
		log.Printf("Failed to connect to service on port 80\n")
	} else {
		// FIXME This should probably be moved to it's own file now.
		response, err := os.Client.Get("http://" + hiddenService)

		if err != nil {
			return report, err
		}

		// Initial Attempt at Resolving Server Type
		log.Printf("Attempting to Derive Server Type from Headers..\n")
		report.ServerVersion = response.Header.Get("Server")
		log.Printf("\tServer Version: %s\n", report.ServerVersion)

		// Apache mod-status Check
		os.ScanPage(hiddenService, "/server-status", report, scans.ApacheModStatus)
		os.ScanPage(hiddenService, "/", report, scans.StandardPageScan)

		os.ScanPage(hiddenService, "/style", report, scans.CheckDirectoryListing)
		os.ScanPage(hiddenService, "/styles", report, scans.CheckDirectoryListing)
		os.ScanPage(hiddenService, "/css", report, scans.CheckDirectoryListing)
		os.ScanPage(hiddenService, "/uploads", report, scans.CheckDirectoryListing)
		os.ScanPage(hiddenService, "/images", report, scans.CheckDirectoryListing)
		os.ScanPage(hiddenService, "/img", report, scans.CheckDirectoryListing)
		os.ScanPage(hiddenService, "/static", report, scans.CheckDirectoryListing)

		// Lots of Wordpress installs which don't lock down directory listings
		os.ScanPage(hiddenService, "/wp-content/uploads", report, scans.CheckDirectoryListing)

		// Common with torshops created onions
		os.ScanPage(hiddenService, "/products", report, scans.CheckDirectoryListing)
		os.ScanPage(hiddenService, "/products/cat", report, scans.CheckDirectoryListing)
	}

	return report, nil
}

func (os *OnionScan) ScanPage(hiddenService string, page string, report *report.OnionScanReport, f func(scans.Scanner, string, int, string, *report.OnionScanReport)) {
	response, err := os.Client.Get("http://" + hiddenService + page)
	if err != nil {
		log.Printf("Error connecting to %s%s %s\n", hiddenService, page, err)
		return
	}
	defer response.Body.Close()
	contents, _ := ioutil.ReadAll(response.Body)
	f(os, page, response.StatusCode, string(contents), report)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
	"sync/atomic"
	"time"
)

// A Post is reflective of the JSON used in the tumblr API.
// It contains a PhotoURL, and, optionally, an array of photos.
// If Photos isn't empty, it typically contains at least one URL which matches PhotoURL.
type Post struct {
	Type          string
	PhotoURL      string `json:"photo-url-1280"`
	Photos        []Post `json:"photos,omitempty"`
	UnixTimestamp int64  `json:"unix-timestamp"`

	// for regular posts
	RegularBody string `json:"regular-body"`

	// for answer posts
	Answer string

	// for videos
	Video string `json:"video-player"`
}

// A Blog is the outer container for Posts. It is necessary for easier JSON deserialization,
// even though it's useless in and of itself.
type Blog struct {
	Posts []Post `json:"posts"`
}

var (
	inlineSearch   = regexp.MustCompile(`(http:\/\/\d{2}\.media\.tumblr\.com\/\w{32}\/tumblr_inline_\w+\.\w+)`) // FIXME: Possibly buggy/unoptimized.
	videoSearch    = regexp.MustCompile(`"hdUrl":"(.*\/tumblr_\w+)"`)                                           // fuck it
	altVideoSearch = regexp.MustCompile(`source src="(.*)" type`)
)

// `\n<video  id='embed-568af048eb003889902731' class='crt-video crt-skin-default' width='400' height='225' poster='http://media.tumblr.com/tumblr_nzrij8lMwd1slstjg_frame1.jpg' preload='none' data-crt-video data-crt-options='{\"autoheight\":null,\"duration\":80,\"hdUrl\":\"http:\\/\\/honourcall.tumblr.com\\/video_file\\/136626115139\\/tumblr_nzrij8lMwd1slstjg\",\"filmstrip\":{\"url\":\"http:\\/\\/38.media.tumblr.com\\/previews\\/tumblr_nzrij8lMwd1slstjg_filmstrip.jpg\",\"width\":\"200\",\"height\":\"112\"}}' >\n    <source src=\"http://honourcall.tumblr.com/video_file/136626115139/tumblr_nzrij8lMwd1slstjg/480\" type=\"video/mp4\">\n</video>\n`

func scrape(user blog, limiter <-chan time.Time) <-chan Image {
	imageChannel := make(chan Image)
	fmt.Println(user)
	go func() {
		defer close(imageChannel)

		for i := 1; ; i++ {
			<-limiter

			base := fmt.Sprintf("http://%s.tumblr.com/api/read/json", user.name)

			tumblrURL, err := url.Parse(base)
			if err != nil {
				log.Fatal(err)
			}

			vals := url.Values{}
			vals.Set("num", "50")
			vals.Add("start", strconv.Itoa((i-1)*50))
			// vals.Add("type", "photo")

			if user.tag != "" {
				vals.Add("tagged", user.tag)
			}

			tumblrURL.RawQuery = vals.Encode()

			// fmt.Println(tumblrURL)

			fmt.Println(user.name, "is on page", i)
			resp, err := http.Get(tumblrURL.String())

			// XXX: Ugly as shit. This could probably be done better.
			if err != nil {
				i--
				log.Println(user, err)
				continue
			}
			defer resp.Body.Close()

			contents, _ := ioutil.ReadAll(resp.Body)

			// This is returned as pure javascript. We need to filter out the variable and the ending semicolon.
			contents = []byte(strings.Replace(string(contents), "var tumblr_api_read = ", "", 1))
			contents = []byte(strings.Replace(string(contents), ";", "", -1))

			var blog Blog
			json.Unmarshal(contents, &blog)

			if len(blog.Posts) == 0 {
				break
			}

			for _, post := range blog.Posts {

				var URLs []string

				switch post.Type {
				case "photo":
					if len(post.Photos) == 0 {
						URLs = append(URLs, post.PhotoURL)
					} else {
						for _, photo := range post.Photos {
							URLs = append(URLs, photo.PhotoURL)
						}
					}

				case "answer":
					URLs = inlineSearch.FindAllString(post.Answer, -1)
				case "regular":
					URLs = inlineSearch.FindAllString(post.RegularBody, -1)
				case "video":
					regextest := videoSearch.FindStringSubmatch(post.Video)
					if regextest == nil { // hdUrl is false. We have to get the other URL.
						regextest = altVideoSearch.FindStringSubmatch(post.Video)
					}

					// If it's still nil, it means it's another embedded video type, like Youtube, Vine or Pornhub.
					// In that case, ignore it and move on. Not my problem.
					if regextest == nil {
						continue
					}
					videoURL := strings.Replace(regextest[1], `\`, ``, -1)

					// If there are problems with downloading video, the below part may be the cause.
					videoURL = strings.Replace(videoURL, `/480`, ``, -1)
					videoURL += ".mp4"

					URLs = append(URLs, videoURL)
				default:
					continue
				}

				// fmt.Println(URLs)

				for _, URL := range URLs {
					i := Image{
						User:          user.name,
						URL:           URL,
						UnixTimestamp: post.UnixTimestamp,
					}

					filename := path.Base(i.URL)
					pathname := fmt.Sprintf("downloads/%s/%s", user.name, filename)

					// If there is a file that exists, we skip adding it and move on to the next one.
					// Or, if update mode is enabled, then we can simply stop searching.
					_, err := os.Stat(pathname)
					if err == nil {
						if updateMode {
							return
						}
						atomic.AddUint64(&alreadyExists, 1)
						continue

					}

					atomic.AddUint64(&totalFound, 1)
					imageChannel <- i
				}

			}

		}

		fmt.Println("Done scraping for", user.name)

	}()
	return imageChannel
}
<commit_msg>Added moar concurrency. MOAR<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

// A Post is reflective of the JSON used in the tumblr API.
// It contains a PhotoURL, and, optionally, an array of photos.
// If Photos isn't empty, it typically contains at least one URL which matches PhotoURL.
type Post struct {
	Type          string
	PhotoURL      string `json:"photo-url-1280"`
	Photos        []Post `json:"photos,omitempty"`
	UnixTimestamp int64  `json:"unix-timestamp"`

	// for regular posts
	RegularBody string `json:"regular-body"`

	// for answer posts
	Answer string

	// for videos
	Video string `json:"video-player"`
}

// A Blog is the outer container for Posts. It is necessary for easier JSON deserialization,
// even though it's useless in and of itself.
type Blog struct {
	Posts []Post `json:"posts"`
}

var (
	inlineSearch   = regexp.MustCompile(`(http:\/\/\d{2}\.media\.tumblr\.com\/\w{32}\/tumblr_inline_\w+\.\w+)`) // FIXME: Possibly buggy/unoptimized.
	videoSearch    = regexp.MustCompile(`"hdUrl":"(.*\/tumblr_\w+)"`)                                           // fuck it
	altVideoSearch = regexp.MustCompile(`source src="(.*)\/\d+" type`)
)

// `\n<video  id='embed-568af048eb003889902731' class='crt-video crt-skin-default' width='400' height='225' poster='http://media.tumblr.com/tumblr_nzrij8lMwd1slstjg_frame1.jpg' preload='none' data-crt-video data-crt-options='{\"autoheight\":null,\"duration\":80,\"hdUrl\":\"http:\\/\\/honourcall.tumblr.com\\/video_file\\/136626115139\\/tumblr_nzrij8lMwd1slstjg\",\"filmstrip\":{\"url\":\"http:\\/\\/38.media.tumblr.com\\/previews\\/tumblr_nzrij8lMwd1slstjg_filmstrip.jpg\",\"width\":\"200\",\"height\":\"112\"}}' >\n    <source src=\"http://honourcall.tumblr.com/video_file/136626115139/tumblr_nzrij8lMwd1slstjg/480\" type=\"video/mp4\">\n</video>\n`

func scrape(user blog, limiter <-chan time.Time) <-chan Image {
	var wg sync.WaitGroup
	imageChannel := make(chan Image)
	fmt.Println(user)

	go func() {
		defer close(imageChannel)

		for i := 1; ; i++ {
			<-limiter

			base := fmt.Sprintf("http://%s.tumblr.com/api/read/json", user.name)

			tumblrURL, err := url.Parse(base)
			if err != nil {
				log.Fatal(err)
			}

			vals := url.Values{}
			vals.Set("num", "50")
			vals.Add("start", strconv.Itoa((i-1)*50))
			// vals.Add("type", "photo")

			if user.tag != "" {
				vals.Add("tagged", user.tag)
			}

			tumblrURL.RawQuery = vals.Encode()

			// fmt.Println(tumblrURL)

			fmt.Println(user.name, "is on page", i)
			resp, err := http.Get(tumblrURL.String())

			// XXX: Ugly as shit. This could probably be done better.
			if err != nil {
				i--
				log.Println(user, err)
				continue
			}
			defer resp.Body.Close()

			contents, _ := ioutil.ReadAll(resp.Body)

			// This is returned as pure javascript. We need to filter out the variable and the ending semicolon.
			contents = []byte(strings.Replace(string(contents), "var tumblr_api_read = ", "", 1))
			contents = []byte(strings.Replace(string(contents), ";", "", -1))

			var blog Blog
			json.Unmarshal(contents, &blog)

			if len(blog.Posts) == 0 {
				break
			}

			wg.Add(1)
			go func() {
				for _, post := range blog.Posts {

					var URLs []string

					switch post.Type {
					case "photo":
						if len(post.Photos) == 0 {
							URLs = append(URLs, post.PhotoURL)
						} else {
							for _, photo := range post.Photos {
								URLs = append(URLs, photo.PhotoURL)
							}
						}

					case "answer":
						URLs = inlineSearch.FindAllString(post.Answer, -1)
					case "regular":
						URLs = inlineSearch.FindAllString(post.RegularBody, -1)
					case "video":
						regextest := videoSearch.FindStringSubmatch(post.Video)
						if regextest == nil { // hdUrl is false. We have to get the other URL.
							regextest = altVideoSearch.FindStringSubmatch(post.Video)
						}

						// If it's still nil, it means it's another embedded video type, like Youtube, Vine or Pornhub.
						// In that case, ignore it and move on. Not my problem.
						if regextest == nil {
							continue
						}
						videoURL := strings.Replace(regextest[1], `\`, ``, -1)

						// If there are problems with downloading video, the below part may be the cause.
						// videoURL = strings.Replace(videoURL, `/480`, ``, -1)
						videoURL += ".mp4"

						URLs = append(URLs, videoURL)
					default:
						continue
					}

					// fmt.Println(URLs)

					for _, URL := range URLs {
						i := Image{
							User:          user.name,
							URL:           URL,
							UnixTimestamp: post.UnixTimestamp,
						}

						filename := path.Base(i.URL)
						pathname := fmt.Sprintf("downloads/%s/%s", user.name, filename)

						// If there is a file that exists, we skip adding it and move on to the next one.
						// Or, if update mode is enabled, then we can simply stop searching.
						_, err := os.Stat(pathname)
						if err == nil {
							if updateMode {
								return
							}
							atomic.AddUint64(&alreadyExists, 1)
							continue

						}

						atomic.AddUint64(&totalFound, 1)
						imageChannel <- i
					}

				}
				wg.Done()
			}()

		}

		fmt.Println("Done scraping for", user.name)

		// This is needed because otherwise, the defer call closes a channel early, and
		// the program panics.
		wg.Wait()
	}()
	return imageChannel
}
<|endoftext|>


<commit_before>package kubernetes

import (
	"crypto/tls"
	"fmt"
	"net"
	"strconv"
	"strings"
	"time"

	"github.com/golang/glog"
	kapp "k8s.io/kubernetes/cmd/kubelet/app"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/kubelet"
	"k8s.io/kubernetes/pkg/kubelet/dockertools"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/errors"

	configapi "github.com/openshift/origin/pkg/cmd/server/api"
	cmdutil "github.com/openshift/origin/pkg/cmd/util"
	cmdflags "github.com/openshift/origin/pkg/cmd/util/flags"
	"github.com/openshift/origin/pkg/cmd/util/variable"
)

// NodeConfig represents the required parameters to start the OpenShift node
// through Kubernetes. All fields are required.
type NodeConfig struct {
	// BindAddress is the address to bind to
	BindAddress string
	// VolumeDir is the directory that volumes will be stored under
	VolumeDir string
	// AllowDisabledDocker if true, will make the Kubelet ignore errors from Docker
	AllowDisabledDocker bool
	// Client to connect to the master.
	Client *client.Client
	// DockerClient is a client to connect to Docker
	DockerClient dockertools.DockerInterface

	// KubeletServer contains the KubeletServer configuration
	KubeletServer *kapp.KubeletServer
	// KubeletConfig is the configuration for the kubelet, fully initialized
	KubeletConfig *kapp.KubeletConfig
}

func BuildKubernetesNodeConfig(options configapi.NodeConfig) (*NodeConfig, error) {
	kubeClient, _, err := configapi.GetKubeClient(options.MasterKubeConfig)
	if err != nil {
		return nil, err
	}

	if options.NodeName == "localhost" {
		glog.Warningf(`Using "localhost" as node name will not resolve from all locations`)
	}

	var dnsIP net.IP
	if len(options.DNSIP) > 0 {
		dnsIP = net.ParseIP(options.DNSIP)
		if dnsIP == nil {
			return nil, fmt.Errorf("Invalid DNS IP: %s", options.DNSIP)
		}
	}

	clientCAs, err := util.CertPoolFromFile(options.ServingInfo.ClientCA)
	if err != nil {
		return nil, err
	}

	imageTemplate := variable.NewDefaultImageTemplate()
	imageTemplate.Format = options.ImageConfig.Format
	imageTemplate.Latest = options.ImageConfig.Latest

	var path string
	var fileCheckInterval int64
	if options.PodManifestConfig != nil {
		path = options.PodManifestConfig.Path
		fileCheckInterval = options.PodManifestConfig.FileCheckIntervalSeconds
	}

	var dockerExecHandler dockertools.ExecHandler

	switch options.DockerConfig.ExecHandlerName {
	case configapi.DockerExecHandlerNative:
		dockerExecHandler = &dockertools.NativeExecHandler{}
	case configapi.DockerExecHandlerNsenter:
		dockerExecHandler = &dockertools.NsenterExecHandler{}
	}

	kubeAddressStr, kubePortStr, err := net.SplitHostPort(options.ServingInfo.BindAddress)
	if err != nil {
		return nil, fmt.Errorf("cannot parse node address: %v", err)
	}
	kubePort, err := strconv.Atoi(kubePortStr)
	if err != nil {
		return nil, fmt.Errorf("cannot parse node port: %v", err)
	}
	kubeAddress := net.ParseIP(kubeAddressStr)
	if kubeAddress == nil {
		return nil, fmt.Errorf("Invalid DNS IP: %s", kubeAddressStr)
	}

	// declare the OpenShift defaults from config
	server := kapp.NewKubeletServer()
	server.Config = path
	server.RootDirectory = options.VolumeDirectory

	// kubelet finds the node IP address by doing net.ParseIP(hostname) and if that fails,
	// it does net.LookupIP(NodeName) and picks the first non-loopback address.
	// Pass node IP as hostname to make kubelet use the desired IP address.
	if len(options.NodeIP) > 0 {
		server.HostnameOverride = options.NodeIP
	} else {
		server.HostnameOverride = options.NodeName
	}
	server.AllowPrivileged = true
	server.RegisterNode = true
	server.Address = kubeAddress
	server.Port = uint(kubePort)
	server.ReadOnlyPort = 0 // no read only access
	server.CadvisorPort = 0 // no unsecured cadvisor access
	server.HealthzPort = 0  // no unsecured healthz access
	server.ClusterDNS = dnsIP
	server.ClusterDomain = options.DNSDomain
	server.NetworkPluginName = options.NetworkConfig.NetworkPluginName
	server.HostNetworkSources = strings.Join([]string{kubelet.ApiserverSource, kubelet.FileSource}, ",")
	server.HTTPCheckFrequency = 0 // no remote HTTP pod creation access
	server.FileCheckFrequency = time.Duration(fileCheckInterval) * time.Second
	server.PodInfraContainerImage = imageTemplate.ExpandOrDie("pod")

	// prevents kube from generating certs
	server.TLSCertFile = options.ServingInfo.ServerCert.CertFile
	server.TLSPrivateKeyFile = options.ServingInfo.ServerCert.KeyFile

	if value := cmdutil.Env("OPENSHIFT_CONTAINERIZED", ""); len(value) > 0 {
		server.Containerized = value == "true"
	}

	// resolve extended arguments
	// TODO: this should be done in config validation (along with the above) so we can provide
	// proper errors
	if err := cmdflags.Resolve(options.KubeletArguments, server.AddFlags); len(err) > 0 {
		return nil, errors.NewAggregate(err)
	}

	cfg, err := server.KubeletConfig()
	if err != nil {
		return nil, err
	}

	// provide any config overrides
	cfg.NodeName = options.NodeName
	cfg.StreamingConnectionIdleTimeout = 5 * time.Minute // TODO: should be set
	cfg.KubeClient = kubeClient
	cfg.DockerExecHandler = dockerExecHandler

	// Make sure the node doesn't think it is in standalone mode
	// This is required for the node to enforce nodeSelectors on pods, to set hostIP on pod status updates, etc
	cfg.StandaloneMode = false

	// TODO: could be cleaner
	if configapi.UseTLS(options.ServingInfo) {
		cfg.TLSOptions = &kubelet.TLSOptions{
			Config: &tls.Config{
				// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability)
				MinVersion: tls.VersionTLS10,
				// RequireAndVerifyClientCert lets us limit requests to ones with a valid client certificate
				ClientAuth: tls.RequireAndVerifyClientCert,
				ClientCAs:  clientCAs,
			},
			CertFile: options.ServingInfo.ServerCert.CertFile,
			KeyFile:  options.ServingInfo.ServerCert.KeyFile,
		}
	} else {
		cfg.TLSOptions = nil
	}

	config := &NodeConfig{
		BindAddress: options.ServingInfo.BindAddress,

		AllowDisabledDocker: options.AllowDisabledDocker,

		Client: kubeClient,

		VolumeDir: options.VolumeDirectory,

		KubeletServer: server,
		KubeletConfig: cfg,
	}

	return config, nil
}
<commit_msg>Enable cpu cfs quota by default<commit_after>package kubernetes

import (
	"crypto/tls"
	"fmt"
	"net"
	"strconv"
	"strings"
	"time"

	"github.com/golang/glog"
	kapp "k8s.io/kubernetes/cmd/kubelet/app"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/kubelet"
	"k8s.io/kubernetes/pkg/kubelet/dockertools"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/errors"

	configapi "github.com/openshift/origin/pkg/cmd/server/api"
	cmdutil "github.com/openshift/origin/pkg/cmd/util"
	cmdflags "github.com/openshift/origin/pkg/cmd/util/flags"
	"github.com/openshift/origin/pkg/cmd/util/variable"
)

// NodeConfig represents the required parameters to start the OpenShift node
// through Kubernetes. All fields are required.
type NodeConfig struct {
	// BindAddress is the address to bind to
	BindAddress string
	// VolumeDir is the directory that volumes will be stored under
	VolumeDir string
	// AllowDisabledDocker if true, will make the Kubelet ignore errors from Docker
	AllowDisabledDocker bool
	// Client to connect to the master.
	Client *client.Client
	// DockerClient is a client to connect to Docker
	DockerClient dockertools.DockerInterface

	// KubeletServer contains the KubeletServer configuration
	KubeletServer *kapp.KubeletServer
	// KubeletConfig is the configuration for the kubelet, fully initialized
	KubeletConfig *kapp.KubeletConfig
}

func BuildKubernetesNodeConfig(options configapi.NodeConfig) (*NodeConfig, error) {
	kubeClient, _, err := configapi.GetKubeClient(options.MasterKubeConfig)
	if err != nil {
		return nil, err
	}

	if options.NodeName == "localhost" {
		glog.Warningf(`Using "localhost" as node name will not resolve from all locations`)
	}

	var dnsIP net.IP
	if len(options.DNSIP) > 0 {
		dnsIP = net.ParseIP(options.DNSIP)
		if dnsIP == nil {
			return nil, fmt.Errorf("Invalid DNS IP: %s", options.DNSIP)
		}
	}

	clientCAs, err := util.CertPoolFromFile(options.ServingInfo.ClientCA)
	if err != nil {
		return nil, err
	}

	imageTemplate := variable.NewDefaultImageTemplate()
	imageTemplate.Format = options.ImageConfig.Format
	imageTemplate.Latest = options.ImageConfig.Latest

	var path string
	var fileCheckInterval int64
	if options.PodManifestConfig != nil {
		path = options.PodManifestConfig.Path
		fileCheckInterval = options.PodManifestConfig.FileCheckIntervalSeconds
	}

	var dockerExecHandler dockertools.ExecHandler

	switch options.DockerConfig.ExecHandlerName {
	case configapi.DockerExecHandlerNative:
		dockerExecHandler = &dockertools.NativeExecHandler{}
	case configapi.DockerExecHandlerNsenter:
		dockerExecHandler = &dockertools.NsenterExecHandler{}
	}

	kubeAddressStr, kubePortStr, err := net.SplitHostPort(options.ServingInfo.BindAddress)
	if err != nil {
		return nil, fmt.Errorf("cannot parse node address: %v", err)
	}
	kubePort, err := strconv.Atoi(kubePortStr)
	if err != nil {
		return nil, fmt.Errorf("cannot parse node port: %v", err)
	}
	kubeAddress := net.ParseIP(kubeAddressStr)
	if kubeAddress == nil {
		return nil, fmt.Errorf("Invalid DNS IP: %s", kubeAddressStr)
	}

	// declare the OpenShift defaults from config
	server := kapp.NewKubeletServer()
	server.Config = path
	server.RootDirectory = options.VolumeDirectory

	// kubelet finds the node IP address by doing net.ParseIP(hostname) and if that fails,
	// it does net.LookupIP(NodeName) and picks the first non-loopback address.
	// Pass node IP as hostname to make kubelet use the desired IP address.
	if len(options.NodeIP) > 0 {
		server.HostnameOverride = options.NodeIP
	} else {
		server.HostnameOverride = options.NodeName
	}
	server.AllowPrivileged = true
	server.RegisterNode = true
	server.Address = kubeAddress
	server.Port = uint(kubePort)
	server.ReadOnlyPort = 0 // no read only access
	server.CadvisorPort = 0 // no unsecured cadvisor access
	server.HealthzPort = 0  // no unsecured healthz access
	server.ClusterDNS = dnsIP
	server.ClusterDomain = options.DNSDomain
	server.NetworkPluginName = options.NetworkConfig.NetworkPluginName
	server.HostNetworkSources = strings.Join([]string{kubelet.ApiserverSource, kubelet.FileSource}, ",")
	server.HTTPCheckFrequency = 0 // no remote HTTP pod creation access
	server.FileCheckFrequency = time.Duration(fileCheckInterval) * time.Second
	server.PodInfraContainerImage = imageTemplate.ExpandOrDie("pod")
	server.CPUCFSQuota = true // enable cpu cfs quota enforcement by default

	// prevents kube from generating certs
	server.TLSCertFile = options.ServingInfo.ServerCert.CertFile
	server.TLSPrivateKeyFile = options.ServingInfo.ServerCert.KeyFile

	if value := cmdutil.Env("OPENSHIFT_CONTAINERIZED", ""); len(value) > 0 {
		server.Containerized = value == "true"
	}

	// resolve extended arguments
	// TODO: this should be done in config validation (along with the above) so we can provide
	// proper errors
	if err := cmdflags.Resolve(options.KubeletArguments, server.AddFlags); len(err) > 0 {
		return nil, errors.NewAggregate(err)
	}

	cfg, err := server.KubeletConfig()
	if err != nil {
		return nil, err
	}

	// provide any config overrides
	cfg.NodeName = options.NodeName
	cfg.StreamingConnectionIdleTimeout = 5 * time.Minute // TODO: should be set
	cfg.KubeClient = kubeClient
	cfg.DockerExecHandler = dockerExecHandler

	// Make sure the node doesn't think it is in standalone mode
	// This is required for the node to enforce nodeSelectors on pods, to set hostIP on pod status updates, etc
	cfg.StandaloneMode = false

	// TODO: could be cleaner
	if configapi.UseTLS(options.ServingInfo) {
		cfg.TLSOptions = &kubelet.TLSOptions{
			Config: &tls.Config{
				// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability)
				MinVersion: tls.VersionTLS10,
				// RequireAndVerifyClientCert lets us limit requests to ones with a valid client certificate
				ClientAuth: tls.RequireAndVerifyClientCert,
				ClientCAs:  clientCAs,
			},
			CertFile: options.ServingInfo.ServerCert.CertFile,
			KeyFile:  options.ServingInfo.ServerCert.KeyFile,
		}
	} else {
		cfg.TLSOptions = nil
	}

	config := &NodeConfig{
		BindAddress: options.ServingInfo.BindAddress,

		AllowDisabledDocker: options.AllowDisabledDocker,

		Client: kubeClient,

		VolumeDir: options.VolumeDirectory,

		KubeletServer: server,
		KubeletConfig: cfg,
	}

	return config, nil
}
<|endoftext|>


<commit_before>package paypal_rest

import (
	"code.google.com/p/godec/dec"
	"database/sql"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	"gopkg.in/inconshreveable/log15.v2"

	"github.com/fritzpay/paymentd/pkg/paymentd/payment"
	"github.com/fritzpay/paymentd/pkg/paymentd/payment_method"
)

const (
	paypalPaymentPath = "/v1/payments/payment"
)

func (d *Driver) InitPayment(p *payment.Payment, method *payment_method.Method) (http.Handler, error) {
	log := d.log.New(log15.Ctx{
		"method":          "InitPayment",
		"projectID":       p.ProjectID(),
		"paymentID":       p.ID(),
		"paymentMethodID": method.ID,
	})

	var tx *sql.Tx
	var err error
	var commit bool
	defer func() {
		if tx != nil && !commit {
			err = tx.Rollback()
			if err != nil {
				log.Crit("error on rollback", log15.Ctx{"err": err})
			}
		}
	}()
	tx, err = d.ctx.PaymentDB().Begin()
	if err != nil {
		commit = true
		log.Crit("error on begin tx", log15.Ctx{"err": err})
		return nil, ErrDatabase
	}

	currentTx, err := TransactionCurrentByPaymentIDTx(tx, p.PaymentID())
	if err != nil && err != ErrTransactionNotFound {
		log.Error("error retrieving transaction", log15.Ctx{"err": err})
		return nil, ErrDatabase
	}
	if err == nil {
		if Debug {
			log.Debug("already initialized payment")
		}
		switch currentTx.Type {
		case TransactionTypeError:
			return d.PaymentErrorHandler(p), nil
		default:
			return d.InitPageHandler(p), nil
		}
	}

	cfg, err := ConfigByPaymentMethodTx(tx, method)
	if err != nil {
		log.Error("error retrieving PayPal config", log15.Ctx{"err": err})
		return nil, ErrDatabase
	}

	// create payment request
	req := &PayPalPaymentRequest{}
	if cfg.Type != "sale" && cfg.Type != "authorize" {
		log.Crit("invalid config type", log15.Ctx{"configType": cfg.Type})
		return nil, ErrInternal
	}
	req.Intent = cfg.Type
	req.Payer.PaymentMethod = PayPalPaymentMethodPayPal
	req.RedirectURLs, err = d.redirectURLs()
	if err != nil {
		log.Error("error creating redirect urls", log15.Ctx{"err": err})
		return nil, ErrInternal
	}
	req.Transactions = []PayPalTransaction{
		d.payPalTransactionFromPayment(p),
	}
	if Debug {
		log.Debug("created paypal payment request", log15.Ctx{"request": req})
	}

	endpoint, err := url.Parse(cfg.Endpoint)
	if err != nil {
		log.Error("error on endpoint URL", log15.Ctx{"err": err})
		return nil, ErrInternal
	}
	endpoint.Path = paypalPaymentPath

	jsonBytes, err := json.Marshal(req)
	if err != nil {
		log.Error("error encoding request", log15.Ctx{"err": err})
		return nil, ErrInternal
	}

	paypalTx := &Transaction{
		ProjectID: p.ProjectID(),
		PaymentID: p.ID(),
		Timestamp: time.Now(),
		Type:      TransactionTypeCreatePayment,
	}
	paypalTx.SetIntent(cfg.Type)
	paypalTx.Data = jsonBytes

	err = InsertTransactionTx(tx, paypalTx)
	if err != nil {
		log.Error("error saving transaction", log15.Ctx{"err": err})
		return nil, ErrDatabase
	}

	commit = true
	err = tx.Commit()
	if err != nil {
		log.Crit("error on commit", log15.Ctx{"err": err})
		return nil, ErrDatabase
	}

	errors := make(chan error)
	go func() {
		for {
			select {
			case err := <-errors:
				if err == nil {
					return
				}
				log.Error("error on initializing", log15.Ctx{"err": err})
				return
			case <-d.ctx.Done():
				log.Warn("cancelled initialization", log15.Ctx{"err": d.ctx.Err()})
				return
			}
		}
	}()
	go d.doInit(errors, cfg, endpoint, p, string(jsonBytes))

	return d.InitPageHandler(p), nil
}

func (d *Driver) redirectURLs() (PayPalRedirectURLs, error) {
	u := PayPalRedirectURLs{}
	returnRoute, err := d.mux.Get("returnHandler").URLPath()
	if err != nil {
		return u, err
	}
	cancelRoute, err := d.mux.Get("cancelHandler").URLPath()
	if err != nil {
		return u, err
	}

	returnURL := &(*d.baseURL)
	returnURL.Path = returnRoute.Path
	u.ReturnURL = returnURL.String()

	cancelURL := &(*d.baseURL)
	cancelURL.Path = cancelRoute.Path
	u.CancelURL = cancelURL.String()

	return u, nil
}

func (d *Driver) payPalTransactionFromPayment(p *payment.Payment) PayPalTransaction {
	t := PayPalTransaction{}
	encPaymentID := d.paymentService.EncodedPaymentID(p.PaymentID())
	t.Custom = encPaymentID.String()
	t.InvoiceNumber = encPaymentID.String()
	amnt := &p.Decimal().Dec
	amnt.Round(amnt, dec.Scale(2), dec.RoundHalfUp)
	t.Amount = PayPalAmount{
		Currency: p.Currency,
		Total:    amnt.String(),
	}
	return t
}

func (d *Driver) doInit(errors chan<- error, cfg *Config, reqURL *url.URL, p *payment.Payment, body string) {
	log := d.log.New(log15.Ctx{
		"method":      "doInit",
		"projectID":   p.ProjectID(),
		"paymentID":   p.ID(),
		"methodKey":   cfg.MethodKey,
		"requestBody": body,
	})
	if Debug {
		log.Debug("posting...")
	}

	tr, err := d.oAuthTransport(log)(p, cfg)
	if err != nil {
		log.Error("error on auth transport", log15.Ctx{"err": err})
		errors <- err
		return
	}
	err = tr.AuthenticateClient()
	if err != nil {
		log.Error("error authenticating", log15.Ctx{"err": err})
		errors <- err
		return
	}
	cl := tr.Client()
	resp, err := cl.Post(reqURL.String(), "application/json", strings.NewReader(body))
	if err != nil {
		log.Error("error on HTTP POST", log15.Ctx{"err": err})
		errors <- err
		return
	}
	if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
		log.Error("error on HTTP request", log15.Ctx{"HTTPStatusCode": resp.StatusCode})
		d.setPayPalErrorResponse(p, nil)
		errors <- ErrHTTP
		return
	}
	respBody, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		log.Error("error reading response body", log15.Ctx{"err": err})
		d.setPayPalErrorResponse(p, nil)
		errors <- ErrHTTP
		return
	}
	log = log.New(log15.Ctx{"responseBody": string(respBody)})
	if Debug {
		log.Debug("received response")
	}
	paypalP := &PaypalPayment{}
	err = json.Unmarshal(respBody, paypalP)
	if err != nil {
		log.Error("error decoding PayPal response", log15.Ctx{"err": err})
		d.setPayPalErrorResponse(p, respBody)
		errors <- ErrProvider
	}

	paypalTx := &Transaction{
		ProjectID: p.ProjectID(),
		PaymentID: p.ID(),
		Timestamp: time.Now(),
		Type:      TransactionTypeCreatePaymentResponse,
	}
	if paypalP.ID != "" {
		paypalTx.SetPaypalID(paypalP.ID)
	}
	if paypalP.State != "" {
		paypalTx.SetState(paypalP.State)
	}
	if paypalP.CreateTime != "" {
		t, err := time.Parse(time.RFC3339, paypalP.CreateTime)
		if err != nil {
			log.Warn("error parsing paypal create time", log15.Ctx{"err": err})
		} else {
			paypalTx.PaypalCreateTime = &t
		}
	}
	if paypalP.UpdateTime != "" {
		t, err := time.Parse(time.RFC3339, paypalP.UpdateTime)
		if err != nil {
			log.Warn("error parsing paypal update time", log15.Ctx{"err": err})
		} else {
			paypalTx.PaypalUpdateTime = &t
		}
	}
	paypalTx.Links, err = json.Marshal(paypalP.Links)
	if err != nil {
		log.Error("error on saving links on response", log15.Ctx{"err": err})
		d.setPayPalErrorResponse(p, respBody)
		errors <- ErrProvider
		return
	}
	paypalTx.Data, err = json.Marshal(paypalP)
	if err != nil {
		log.Error("error marshalling paypal payment response", log15.Ctx{"err": err})
		d.setPayPalErrorResponse(p, respBody)
		errors <- ErrProvider
		return
	}
	err = InsertTransactionDB(d.ctx.PaymentDB(), paypalTx)
	if err != nil {
		log.Error("error saving paypal response", log15.Ctx{"err": err})
		d.setPayPalErrorResponse(p, respBody)
		errors <- ErrProvider
		return
	}

	close(errors)
}
<commit_msg>save intent on response<commit_after>package paypal_rest

import (
	"code.google.com/p/godec/dec"
	"database/sql"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	"gopkg.in/inconshreveable/log15.v2"

	"github.com/fritzpay/paymentd/pkg/paymentd/payment"
	"github.com/fritzpay/paymentd/pkg/paymentd/payment_method"
)

const (
	paypalPaymentPath = "/v1/payments/payment"
)

func (d *Driver) InitPayment(p *payment.Payment, method *payment_method.Method) (http.Handler, error) {
	log := d.log.New(log15.Ctx{
		"method":          "InitPayment",
		"projectID":       p.ProjectID(),
		"paymentID":       p.ID(),
		"paymentMethodID": method.ID,
	})

	var tx *sql.Tx
	var err error
	var commit bool
	defer func() {
		if tx != nil && !commit {
			err = tx.Rollback()
			if err != nil {
				log.Crit("error on rollback", log15.Ctx{"err": err})
			}
		}
	}()
	tx, err = d.ctx.PaymentDB().Begin()
	if err != nil {
		commit = true
		log.Crit("error on begin tx", log15.Ctx{"err": err})
		return nil, ErrDatabase
	}

	currentTx, err := TransactionCurrentByPaymentIDTx(tx, p.PaymentID())
	if err != nil && err != ErrTransactionNotFound {
		log.Error("error retrieving transaction", log15.Ctx{"err": err})
		return nil, ErrDatabase
	}
	if err == nil {
		if Debug {
			log.Debug("already initialized payment")
		}
		switch currentTx.Type {
		case TransactionTypeError:
			return d.PaymentErrorHandler(p), nil
		default:
			return d.InitPageHandler(p), nil
		}
	}

	cfg, err := ConfigByPaymentMethodTx(tx, method)
	if err != nil {
		log.Error("error retrieving PayPal config", log15.Ctx{"err": err})
		return nil, ErrDatabase
	}

	// create payment request
	req := &PayPalPaymentRequest{}
	if cfg.Type != "sale" && cfg.Type != "authorize" {
		log.Crit("invalid config type", log15.Ctx{"configType": cfg.Type})
		return nil, ErrInternal
	}
	req.Intent = cfg.Type
	req.Payer.PaymentMethod = PayPalPaymentMethodPayPal
	req.RedirectURLs, err = d.redirectURLs()
	if err != nil {
		log.Error("error creating redirect urls", log15.Ctx{"err": err})
		return nil, ErrInternal
	}
	req.Transactions = []PayPalTransaction{
		d.payPalTransactionFromPayment(p),
	}
	if Debug {
		log.Debug("created paypal payment request", log15.Ctx{"request": req})
	}

	endpoint, err := url.Parse(cfg.Endpoint)
	if err != nil {
		log.Error("error on endpoint URL", log15.Ctx{"err": err})
		return nil, ErrInternal
	}
	endpoint.Path = paypalPaymentPath

	jsonBytes, err := json.Marshal(req)
	if err != nil {
		log.Error("error encoding request", log15.Ctx{"err": err})
		return nil, ErrInternal
	}

	paypalTx := &Transaction{
		ProjectID: p.ProjectID(),
		PaymentID: p.ID(),
		Timestamp: time.Now(),
		Type:      TransactionTypeCreatePayment,
	}
	paypalTx.SetIntent(cfg.Type)
	paypalTx.Data = jsonBytes

	err = InsertTransactionTx(tx, paypalTx)
	if err != nil {
		log.Error("error saving transaction", log15.Ctx{"err": err})
		return nil, ErrDatabase
	}

	commit = true
	err = tx.Commit()
	if err != nil {
		log.Crit("error on commit", log15.Ctx{"err": err})
		return nil, ErrDatabase
	}

	errors := make(chan error)
	go func() {
		for {
			select {
			case err := <-errors:
				if err == nil {
					return
				}
				log.Error("error on initializing", log15.Ctx{"err": err})
				return
			case <-d.ctx.Done():
				log.Warn("cancelled initialization", log15.Ctx{"err": d.ctx.Err()})
				return
			}
		}
	}()
	go d.doInit(errors, cfg, endpoint, p, string(jsonBytes))

	return d.InitPageHandler(p), nil
}

func (d *Driver) redirectURLs() (PayPalRedirectURLs, error) {
	u := PayPalRedirectURLs{}
	returnRoute, err := d.mux.Get("returnHandler").URLPath()
	if err != nil {
		return u, err
	}
	cancelRoute, err := d.mux.Get("cancelHandler").URLPath()
	if err != nil {
		return u, err
	}

	returnURL := &(*d.baseURL)
	returnURL.Path = returnRoute.Path
	u.ReturnURL = returnURL.String()

	cancelURL := &(*d.baseURL)
	cancelURL.Path = cancelRoute.Path
	u.CancelURL = cancelURL.String()

	return u, nil
}

func (d *Driver) payPalTransactionFromPayment(p *payment.Payment) PayPalTransaction {
	t := PayPalTransaction{}
	encPaymentID := d.paymentService.EncodedPaymentID(p.PaymentID())
	t.Custom = encPaymentID.String()
	t.InvoiceNumber = encPaymentID.String()
	amnt := &p.Decimal().Dec
	amnt.Round(amnt, dec.Scale(2), dec.RoundHalfUp)
	t.Amount = PayPalAmount{
		Currency: p.Currency,
		Total:    amnt.String(),
	}
	return t
}

func (d *Driver) doInit(errors chan<- error, cfg *Config, reqURL *url.URL, p *payment.Payment, body string) {
	log := d.log.New(log15.Ctx{
		"method":      "doInit",
		"projectID":   p.ProjectID(),
		"paymentID":   p.ID(),
		"methodKey":   cfg.MethodKey,
		"requestBody": body,
	})
	if Debug {
		log.Debug("posting...")
	}

	tr, err := d.oAuthTransport(log)(p, cfg)
	if err != nil {
		log.Error("error on auth transport", log15.Ctx{"err": err})
		errors <- err
		return
	}
	err = tr.AuthenticateClient()
	if err != nil {
		log.Error("error authenticating", log15.Ctx{"err": err})
		errors <- err
		return
	}
	cl := tr.Client()
	resp, err := cl.Post(reqURL.String(), "application/json", strings.NewReader(body))
	if err != nil {
		log.Error("error on HTTP POST", log15.Ctx{"err": err})
		errors <- err
		return
	}
	if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
		log.Error("error on HTTP request", log15.Ctx{"HTTPStatusCode": resp.StatusCode})
		d.setPayPalErrorResponse(p, nil)
		errors <- ErrHTTP
		return
	}
	respBody, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		log.Error("error reading response body", log15.Ctx{"err": err})
		d.setPayPalErrorResponse(p, nil)
		errors <- ErrHTTP
		return
	}
	log = log.New(log15.Ctx{"responseBody": string(respBody)})
	if Debug {
		log.Debug("received response")
	}
	paypalP := &PaypalPayment{}
	err = json.Unmarshal(respBody, paypalP)
	if err != nil {
		log.Error("error decoding PayPal response", log15.Ctx{"err": err})
		d.setPayPalErrorResponse(p, respBody)
		errors <- ErrProvider
	}

	paypalTx := &Transaction{
		ProjectID: p.ProjectID(),
		PaymentID: p.ID(),
		Timestamp: time.Now(),
		Type:      TransactionTypeCreatePaymentResponse,
	}
	if paypalP.Intent != "" {
		paypalTx.SetIntent(paypalP.Intent)
	}
	if paypalP.ID != "" {
		paypalTx.SetPaypalID(paypalP.ID)
	}
	if paypalP.State != "" {
		paypalTx.SetState(paypalP.State)
	}
	if paypalP.CreateTime != "" {
		t, err := time.Parse(time.RFC3339, paypalP.CreateTime)
		if err != nil {
			log.Warn("error parsing paypal create time", log15.Ctx{"err": err})
		} else {
			paypalTx.PaypalCreateTime = &t
		}
	}
	if paypalP.UpdateTime != "" {
		t, err := time.Parse(time.RFC3339, paypalP.UpdateTime)
		if err != nil {
			log.Warn("error parsing paypal update time", log15.Ctx{"err": err})
		} else {
			paypalTx.PaypalUpdateTime = &t
		}
	}
	paypalTx.Links, err = json.Marshal(paypalP.Links)
	if err != nil {
		log.Error("error on saving links on response", log15.Ctx{"err": err})
		d.setPayPalErrorResponse(p, respBody)
		errors <- ErrProvider
		return
	}
	paypalTx.Data, err = json.Marshal(paypalP)
	if err != nil {
		log.Error("error marshalling paypal payment response", log15.Ctx{"err": err})
		d.setPayPalErrorResponse(p, respBody)
		errors <- ErrProvider
		return
	}
	err = InsertTransactionDB(d.ctx.PaymentDB(), paypalTx)
	if err != nil {
		log.Error("error saving paypal response", log15.Ctx{"err": err})
		d.setPayPalErrorResponse(p, respBody)
		errors <- ErrProvider
		return
	}

	close(errors)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package spdy

import (
	"bufio"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strings"

	"k8s.io/kubernetes/pkg/api"
	apierrors "k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/util/httpstream"
	"k8s.io/kubernetes/third_party/golang/netutil"
)

// SpdyRoundTripper knows how to upgrade an HTTP request to one that supports
// multiplexed streams. After RoundTrip() is invoked, Conn will be set
// and usable. SpdyRoundTripper implements the UpgradeRoundTripper interface.
type SpdyRoundTripper struct {
	//tlsConfig holds the TLS configuration settings to use when connecting
	//to the remote server.
	tlsConfig *tls.Config

	/* TODO according to http://golang.org/pkg/net/http/#RoundTripper, a RoundTripper
	   must be safe for use by multiple concurrent goroutines. If this is absolutely
	   necessary, we could keep a map from http.Request to net.Conn. In practice,
	   a client will create an http.Client, set the transport to a new insteace of
	   SpdyRoundTripper, and use it a single time, so this hopefully won't be an issue.
	*/
	// conn is the underlying network connection to the remote server.
	conn net.Conn

	// Dialer is the dialer used to connect.  Used if non-nil.
	Dialer *net.Dialer
}

// NewRoundTripper creates a new SpdyRoundTripper that will use
// the specified tlsConfig.
func NewRoundTripper(tlsConfig *tls.Config) httpstream.UpgradeRoundTripper {
	return NewSpdyRoundTripper(tlsConfig)
}

// NewSpdyRoundTripper creates a new SpdyRoundTripper that will use
// the specified tlsConfig. This function is mostly meant for unit tests.
func NewSpdyRoundTripper(tlsConfig *tls.Config) *SpdyRoundTripper {
	return &SpdyRoundTripper{tlsConfig: tlsConfig}
}

// dial dials the host specified by req, using TLS if appropriate, optionally
// using a proxy server if one is configured via environment variables.
func (s *SpdyRoundTripper) dial(req *http.Request) (net.Conn, error) {
	proxyURL, err := http.ProxyFromEnvironment(req)
	if err != nil {
		return nil, err
	}

	if proxyURL == nil {
		return s.dialWithoutProxy(req.URL)
	}

	// proxying logic adapted from http://blog.h6t.eu/post/74098062923/golang-websocket-with-http-proxy-support
	proxyReq := http.Request{
		Method: "CONNECT",
		URL:    &url.URL{},
		Host:   req.URL.Host,
	}

	proxyDialConn, err := s.dialWithoutProxy(proxyURL)
	if err != nil {
		return nil, err
	}

	proxyClientConn := httputil.NewProxyClientConn(proxyDialConn, nil)
	_, err = proxyClientConn.Do(&proxyReq)
	if err != nil && err != httputil.ErrPersistEOF {
		return nil, err
	}

	rwc, _ := proxyClientConn.Hijack()

	if req.URL.Scheme != "https" {
		return rwc, nil
	}

	host, _, err := net.SplitHostPort(req.URL.Host)
	if err != nil {
		return nil, err
	}

	if len(s.tlsConfig.ServerName) == 0 {
		s.tlsConfig.ServerName = host
	}

	tlsConn := tls.Client(rwc, s.tlsConfig)

	// need to manually call Handshake() so we can call VerifyHostname() below
	if err := tlsConn.Handshake(); err != nil {
		return nil, err
	}

	if err := tlsConn.VerifyHostname(host); err != nil {
		return nil, err
	}

	return tlsConn, nil
}

// dialWithoutProxy dials the host specified by url, using TLS if appropriate.
func (s *SpdyRoundTripper) dialWithoutProxy(url *url.URL) (net.Conn, error) {
	dialAddr := netutil.CanonicalAddr(url)

	if url.Scheme == "http" {
		if s.Dialer == nil {
			return net.Dial("tcp", dialAddr)
		} else {
			return s.Dialer.Dial("tcp", dialAddr)
		}
	}

	// TODO validate the TLSClientConfig is set up?
	var conn *tls.Conn
	var err error
	if s.Dialer == nil {
		conn, err = tls.Dial("tcp", dialAddr, s.tlsConfig)
	} else {
		conn, err = tls.DialWithDialer(s.Dialer, "tcp", dialAddr, s.tlsConfig)
	}
	if err != nil {
		return nil, err
	}

	host, _, err := net.SplitHostPort(dialAddr)
	if err != nil {
		return nil, err
	}
	err = conn.VerifyHostname(host)
	if err != nil {
		return nil, err
	}

	return conn, nil
}

// RoundTrip executes the Request and upgrades it. After a successful upgrade,
// clients may call SpdyRoundTripper.Connection() to retrieve the upgraded
// connection.
func (s *SpdyRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	// TODO what's the best way to clone the request?
	r := *req
	req = &r
	req.Header.Add(httpstream.HeaderConnection, httpstream.HeaderUpgrade)
	req.Header.Add(httpstream.HeaderUpgrade, HeaderSpdy31)

	conn, err := s.dial(req)
	if err != nil {
		return nil, err
	}

	err = req.Write(conn)
	if err != nil {
		return nil, err
	}

	resp, err := http.ReadResponse(bufio.NewReader(conn), req)
	if err != nil {
		return nil, err
	}

	s.conn = conn

	return resp, nil
}

// NewConnection validates the upgrade response, creating and returning a new
// httpstream.Connection if there were no errors.
func (s *SpdyRoundTripper) NewConnection(resp *http.Response) (httpstream.Connection, error) {
	connectionHeader := strings.ToLower(resp.Header.Get(httpstream.HeaderConnection))
	upgradeHeader := strings.ToLower(resp.Header.Get(httpstream.HeaderUpgrade))
	if !strings.Contains(connectionHeader, strings.ToLower(httpstream.HeaderUpgrade)) || !strings.Contains(upgradeHeader, strings.ToLower(HeaderSpdy31)) {
		defer resp.Body.Close()
		responseError := ""
		responseErrorBytes, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			responseError = "unable to read error from server response"
		} else {
			if obj, err := api.Scheme.Decode(responseErrorBytes); err == nil {
				if status, ok := obj.(*unversioned.Status); ok {
					return nil, &apierrors.StatusError{ErrStatus: *status}
				}
			}
			responseError = string(responseErrorBytes)
			responseError = strings.TrimSpace(responseError)
		}

		return nil, fmt.Errorf("unable to upgrade connection: %s", responseError)
	}

	return NewClientConnection(s.conn)
}
<commit_msg>url.URL.Host now canonical before use in spdy roundtripper.<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package spdy

import (
	"bufio"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strings"

	"k8s.io/kubernetes/pkg/api"
	apierrors "k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/util/httpstream"
	"k8s.io/kubernetes/third_party/golang/netutil"
)

// SpdyRoundTripper knows how to upgrade an HTTP request to one that supports
// multiplexed streams. After RoundTrip() is invoked, Conn will be set
// and usable. SpdyRoundTripper implements the UpgradeRoundTripper interface.
type SpdyRoundTripper struct {
	//tlsConfig holds the TLS configuration settings to use when connecting
	//to the remote server.
	tlsConfig *tls.Config

	/* TODO according to http://golang.org/pkg/net/http/#RoundTripper, a RoundTripper
	   must be safe for use by multiple concurrent goroutines. If this is absolutely
	   necessary, we could keep a map from http.Request to net.Conn. In practice,
	   a client will create an http.Client, set the transport to a new insteace of
	   SpdyRoundTripper, and use it a single time, so this hopefully won't be an issue.
	*/
	// conn is the underlying network connection to the remote server.
	conn net.Conn

	// Dialer is the dialer used to connect.  Used if non-nil.
	Dialer *net.Dialer
}

// NewRoundTripper creates a new SpdyRoundTripper that will use
// the specified tlsConfig.
func NewRoundTripper(tlsConfig *tls.Config) httpstream.UpgradeRoundTripper {
	return NewSpdyRoundTripper(tlsConfig)
}

// NewSpdyRoundTripper creates a new SpdyRoundTripper that will use
// the specified tlsConfig. This function is mostly meant for unit tests.
func NewSpdyRoundTripper(tlsConfig *tls.Config) *SpdyRoundTripper {
	return &SpdyRoundTripper{tlsConfig: tlsConfig}
}

// dial dials the host specified by req, using TLS if appropriate, optionally
// using a proxy server if one is configured via environment variables.
func (s *SpdyRoundTripper) dial(req *http.Request) (net.Conn, error) {
	proxyURL, err := http.ProxyFromEnvironment(req)
	if err != nil {
		return nil, err
	}

	if proxyURL == nil {
		return s.dialWithoutProxy(req.URL)
	}

	// ensure we use a canonical host with proxyReq
	targetHost := netutil.CanonicalAddr(req.URL)

	// proxying logic adapted from http://blog.h6t.eu/post/74098062923/golang-websocket-with-http-proxy-support
	proxyReq := http.Request{
		Method: "CONNECT",
		URL:    &url.URL{},
		Host:   targetHost,
	}

	proxyDialConn, err := s.dialWithoutProxy(proxyURL)
	if err != nil {
		return nil, err
	}

	proxyClientConn := httputil.NewProxyClientConn(proxyDialConn, nil)
	_, err = proxyClientConn.Do(&proxyReq)
	if err != nil && err != httputil.ErrPersistEOF {
		return nil, err
	}

	rwc, _ := proxyClientConn.Hijack()

	if req.URL.Scheme != "https" {
		return rwc, nil
	}

	host, _, err := net.SplitHostPort(req.URL.Host)
	if err != nil {
		return nil, err
	}

	if len(s.tlsConfig.ServerName) == 0 {
		s.tlsConfig.ServerName = host
	}

	tlsConn := tls.Client(rwc, s.tlsConfig)

	// need to manually call Handshake() so we can call VerifyHostname() below
	if err := tlsConn.Handshake(); err != nil {
		return nil, err
	}

	if err := tlsConn.VerifyHostname(host); err != nil {
		return nil, err
	}

	return tlsConn, nil
}

// dialWithoutProxy dials the host specified by url, using TLS if appropriate.
func (s *SpdyRoundTripper) dialWithoutProxy(url *url.URL) (net.Conn, error) {
	dialAddr := netutil.CanonicalAddr(url)

	if url.Scheme == "http" {
		if s.Dialer == nil {
			return net.Dial("tcp", dialAddr)
		} else {
			return s.Dialer.Dial("tcp", dialAddr)
		}
	}

	// TODO validate the TLSClientConfig is set up?
	var conn *tls.Conn
	var err error
	if s.Dialer == nil {
		conn, err = tls.Dial("tcp", dialAddr, s.tlsConfig)
	} else {
		conn, err = tls.DialWithDialer(s.Dialer, "tcp", dialAddr, s.tlsConfig)
	}
	if err != nil {
		return nil, err
	}

	host, _, err := net.SplitHostPort(dialAddr)
	if err != nil {
		return nil, err
	}
	err = conn.VerifyHostname(host)
	if err != nil {
		return nil, err
	}

	return conn, nil
}

// RoundTrip executes the Request and upgrades it. After a successful upgrade,
// clients may call SpdyRoundTripper.Connection() to retrieve the upgraded
// connection.
func (s *SpdyRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	// TODO what's the best way to clone the request?
	r := *req
	req = &r
	req.Header.Add(httpstream.HeaderConnection, httpstream.HeaderUpgrade)
	req.Header.Add(httpstream.HeaderUpgrade, HeaderSpdy31)

	conn, err := s.dial(req)
	if err != nil {
		return nil, err
	}

	err = req.Write(conn)
	if err != nil {
		return nil, err
	}

	resp, err := http.ReadResponse(bufio.NewReader(conn), req)
	if err != nil {
		return nil, err
	}

	s.conn = conn

	return resp, nil
}

// NewConnection validates the upgrade response, creating and returning a new
// httpstream.Connection if there were no errors.
func (s *SpdyRoundTripper) NewConnection(resp *http.Response) (httpstream.Connection, error) {
	connectionHeader := strings.ToLower(resp.Header.Get(httpstream.HeaderConnection))
	upgradeHeader := strings.ToLower(resp.Header.Get(httpstream.HeaderUpgrade))
	if !strings.Contains(connectionHeader, strings.ToLower(httpstream.HeaderUpgrade)) || !strings.Contains(upgradeHeader, strings.ToLower(HeaderSpdy31)) {
		defer resp.Body.Close()
		responseError := ""
		responseErrorBytes, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			responseError = "unable to read error from server response"
		} else {
			if obj, err := api.Scheme.Decode(responseErrorBytes); err == nil {
				if status, ok := obj.(*unversioned.Status); ok {
					return nil, &apierrors.StatusError{ErrStatus: *status}
				}
			}
			responseError = string(responseErrorBytes)
			responseError = strings.TrimSpace(responseError)
		}

		return nil, fmt.Errorf("unable to upgrade connection: %s", responseError)
	}

	return NewClientConnection(s.conn)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/andelf/go-curl"
	"github.com/nu7hatch/gouuid"
)

type Config struct {
	Location         string
	ChecksUrl        string
	MeasurementsUrl  string
	MeasurementsUser string
	MeasurementsPass string
	MeasurerCount    int
	RecorderCount    int
}

type Check struct {
	Id  string `json:"id"`
	Url string `json:"url"`
}

type Measurement struct {
	Check             Check   `json:"check"`
	Id                string  `json:"id"`
	Location          string  `json:"location"`
	T                 int     `json:"t"`
	ExitStatus        int     `json:"exit_status"`
	ConnectTime       float64 `json:"connect_time,omitempty"`
	StartTransferTime float64 `json:"starttransfer_time,omitempty"`
	LocalIp           string  `json:"local_ip,omitempty"`
	PrimaryIp         string  `json:"primary_ip,omitempty"`
	TotalTime         float64 `json:"total_time,omitempty"`
	HttpStatus        int     `json:"http_status,omitempty"`
	NameLookupTime    float64 `json:"namelookup_time,omitempty"`
}

func (c *Check) Measure(config Config) Measurement {
	var m Measurement

	id, _ := uuid.NewV4()
	m.Id = id.String()
	m.Check = *c
	m.Location = config.Location

	easy := curl.EasyInit()
	defer easy.Cleanup()

	easy.Setopt(curl.OPT_URL, c.Url)

	// dummy func for curl output
	noOut := func(buf []byte, userdata interface{}) bool {
		return true
	}

	easy.Setopt(curl.OPT_WRITEFUNCTION, noOut)
	easy.Setopt(curl.OPT_CONNECTTIMEOUT, 10)
	easy.Setopt(curl.OPT_TIMEOUT, 10)

	now := time.Now()
	m.T = int(now.Unix())

	if err := easy.Perform(); err != nil {
		if e, ok := err.(curl.CurlError); ok {
			m.ExitStatus = (int(e))
			return m
		}
		os.Exit(1)
	}

	m.ExitStatus = 0
	http_status, _ := easy.Getinfo(curl.INFO_RESPONSE_CODE)
	m.HttpStatus = http_status.(int)

	connect_time, _ := easy.Getinfo(curl.INFO_CONNECT_TIME)
	m.ConnectTime = connect_time.(float64)

	namelookup_time, _ := easy.Getinfo(curl.INFO_NAMELOOKUP_TIME)
	m.NameLookupTime = namelookup_time.(float64)

	starttransfer_time, _ := easy.Getinfo(curl.INFO_STARTTRANSFER_TIME)
	m.StartTransferTime = starttransfer_time.(float64)

	total_time, _ := easy.Getinfo(curl.INFO_TOTAL_TIME)
	m.TotalTime = total_time.(float64)

	local_ip, _ := easy.Getinfo(curl.INFO_LOCAL_IP)
	m.LocalIp = local_ip.(string)

	primary_ip, _ := easy.Getinfo(curl.INFO_PRIMARY_IP)
	m.PrimaryIp = primary_ip.(string)

	return m
}

func measurer(config Config, toMeasurer chan Check, toStreamer chan Measurement) {
	for {
		c := <-toMeasurer
		m := c.Measure(config)

		toStreamer <- m
	}
}

func streamer(config Config, toStreamer chan Measurement) {
	h := func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		enc := json.NewEncoder(w)
		for {
			enc.Encode(<-toStreamer)
		}
	}

	http.HandleFunc("/measurements", h)
	http.ListenAndServe(":5000", nil)
}

func getChecks(config Config) []Check {
	url := config.ChecksUrl

	res, err := http.Get(url)
	if err != nil {
		panic(err)
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		panic(err)
	}

	var checks []Check
	err = json.Unmarshal(body, &checks)
	if err != nil {
		panic(err)
	}

	return checks
}

func scheduler(check Check, toMeasurer chan Check) {
	for {
		toMeasurer <- check
		time.Sleep(1000 * time.Millisecond)
	}
}

func main() {
	config := Config{}
	flag.StringVar(&config.Location, "location", "undefined", "location of this sensor")
	flag.StringVar(&config.ChecksUrl, "checks_url", "https://s3.amazonaws.com/canary-public-data/checks.json", "URL for check data")
	flag.StringVar(&config.MeasurementsUrl, "measurements_url", "http://localhost:5000/measurements", "URL to POST measurements to")
	flag.IntVar(&config.MeasurerCount, "measurer_count", 1, "number of measurers to run")
	flag.Parse()

	u, err := url.Parse(config.MeasurementsUrl)
	if err != nil {
		panic(err)
	}

	if u.User != nil {
		config.MeasurementsUser = u.User.Username()
		config.MeasurementsPass, _ = u.User.Password()
	}

	check_list := getChecks(config)

	toMeasurer := make(chan Check)
	toStreamer := make(chan Measurement)

	for i := 0; i < config.MeasurerCount; i++ {
		go measurer(config, toMeasurer, toStreamer)
	}

	go streamer(config, toStreamer)

	for _, c := range check_list {
		go scheduler(c, toMeasurer)
	}

	select {}
}
<commit_msg>remove more recorder() related things<commit_after>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"net/http"
	"os"
	"time"

	"github.com/andelf/go-curl"
	"github.com/nu7hatch/gouuid"
)

type Config struct {
	Location      string
	ChecksUrl     string
	MeasurerCount int
}

type Check struct {
	Id  string `json:"id"`
	Url string `json:"url"`
}

type Measurement struct {
	Check             Check   `json:"check"`
	Id                string  `json:"id"`
	Location          string  `json:"location"`
	T                 int     `json:"t"`
	ExitStatus        int     `json:"exit_status"`
	ConnectTime       float64 `json:"connect_time,omitempty"`
	StartTransferTime float64 `json:"starttransfer_time,omitempty"`
	LocalIp           string  `json:"local_ip,omitempty"`
	PrimaryIp         string  `json:"primary_ip,omitempty"`
	TotalTime         float64 `json:"total_time,omitempty"`
	HttpStatus        int     `json:"http_status,omitempty"`
	NameLookupTime    float64 `json:"namelookup_time,omitempty"`
}

func (c *Check) Measure(config Config) Measurement {
	var m Measurement

	id, _ := uuid.NewV4()
	m.Id = id.String()
	m.Check = *c
	m.Location = config.Location

	easy := curl.EasyInit()
	defer easy.Cleanup()

	easy.Setopt(curl.OPT_URL, c.Url)

	// dummy func for curl output
	noOut := func(buf []byte, userdata interface{}) bool {
		return true
	}

	easy.Setopt(curl.OPT_WRITEFUNCTION, noOut)
	easy.Setopt(curl.OPT_CONNECTTIMEOUT, 10)
	easy.Setopt(curl.OPT_TIMEOUT, 10)

	now := time.Now()
	m.T = int(now.Unix())

	if err := easy.Perform(); err != nil {
		if e, ok := err.(curl.CurlError); ok {
			m.ExitStatus = (int(e))
			return m
		}
		os.Exit(1)
	}

	m.ExitStatus = 0
	http_status, _ := easy.Getinfo(curl.INFO_RESPONSE_CODE)
	m.HttpStatus = http_status.(int)

	connect_time, _ := easy.Getinfo(curl.INFO_CONNECT_TIME)
	m.ConnectTime = connect_time.(float64)

	namelookup_time, _ := easy.Getinfo(curl.INFO_NAMELOOKUP_TIME)
	m.NameLookupTime = namelookup_time.(float64)

	starttransfer_time, _ := easy.Getinfo(curl.INFO_STARTTRANSFER_TIME)
	m.StartTransferTime = starttransfer_time.(float64)

	total_time, _ := easy.Getinfo(curl.INFO_TOTAL_TIME)
	m.TotalTime = total_time.(float64)

	local_ip, _ := easy.Getinfo(curl.INFO_LOCAL_IP)
	m.LocalIp = local_ip.(string)

	primary_ip, _ := easy.Getinfo(curl.INFO_PRIMARY_IP)
	m.PrimaryIp = primary_ip.(string)

	return m
}

func measurer(config Config, toMeasurer chan Check, toStreamer chan Measurement) {
	for {
		c := <-toMeasurer
		m := c.Measure(config)

		toStreamer <- m
	}
}

func streamer(config Config, toStreamer chan Measurement) {
	h := func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		enc := json.NewEncoder(w)
		for {
			enc.Encode(<-toStreamer)
		}
	}

	http.HandleFunc("/measurements", h)
	http.ListenAndServe(":5000", nil)
}

func getChecks(config Config) []Check {
	url := config.ChecksUrl

	res, err := http.Get(url)
	if err != nil {
		panic(err)
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		panic(err)
	}

	var checks []Check
	err = json.Unmarshal(body, &checks)
	if err != nil {
		panic(err)
	}

	return checks
}

func scheduler(check Check, toMeasurer chan Check) {
	for {
		toMeasurer <- check
		time.Sleep(1000 * time.Millisecond)
	}
}

func main() {
	config := Config{}
	flag.StringVar(&config.Location, "location", "undefined", "location of this sensor")
	flag.StringVar(&config.ChecksUrl, "checks_url", "https://s3.amazonaws.com/canary-public-data/checks.json", "URL for check data")
	flag.IntVar(&config.MeasurerCount, "measurer_count", 1, "number of measurers to run")
	flag.Parse()

	check_list := getChecks(config)

	toMeasurer := make(chan Check)
	toStreamer := make(chan Measurement)

	for i := 0; i < config.MeasurerCount; i++ {
		go measurer(config, toMeasurer, toStreamer)
	}

	go streamer(config, toStreamer)

	for _, c := range check_list {
		go scheduler(c, toMeasurer)
	}

	select {}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/andelf/go-curl"
	"github.com/nu7hatch/gouuid"
)

type Config struct {
	Location         string
	ChecksUrl        string
	MeasurementsUrl  string
	MeasurementsUser string
	MeasurementsPass string
	MeasurerCount    int
	RecorderCount    int
}

type Check struct {
	Id  string `json:"id"`
	Url string `json:"url"`
}

type Measurement struct {
	Check             Check   `json:"check"`
	Id                string  `json:"id"`
	Location          string  `json:"location"`
	T                 int     `json:"t"`
	ExitStatus        int     `json:"exit_status"`
	ConnectTime       float64 `json:"connect_time,omitempty"`
	StartTransferTime float64 `json:"starttransfer_time,omitempty"`
	LocalIp           string  `json:"local_ip,omitempty"`
	PrimaryIp         string  `json:"primary_ip,omitempty"`
	TotalTime         float64 `json:"total_time,omitempty"`
	HttpStatus        int     `json:"http_status,omitempty"`
	NameLookupTime    float64 `json:"namelookup_time,omitempty"`
}

func (c *Check) Measure(config Config) Measurement {
	var m Measurement

	id, _ := uuid.NewV4()
	m.Id = id.String()
	m.Check = *c
	m.Location = config.Location

	easy := curl.EasyInit()
	defer easy.Cleanup()

	easy.Setopt(curl.OPT_URL, c.Url)

	// dummy func for curl output
	noOut := func(buf []byte, userdata interface{}) bool {
		return true
	}

	easy.Setopt(curl.OPT_WRITEFUNCTION, noOut)
	easy.Setopt(curl.OPT_CONNECTTIMEOUT, 10)
	easy.Setopt(curl.OPT_TIMEOUT, 10)

	now := time.Now()
	m.T = int(now.Unix())

	if err := easy.Perform(); err != nil {
		if e, ok := err.(curl.CurlError); ok {
			m.ExitStatus = (int(e))
			return m
		}
		os.Exit(1)
	}

	m.ExitStatus = 0
	http_status, _ := easy.Getinfo(curl.INFO_RESPONSE_CODE)
	m.HttpStatus = http_status.(int)

	connect_time, _ := easy.Getinfo(curl.INFO_CONNECT_TIME)
	m.ConnectTime = connect_time.(float64)

	namelookup_time, _ := easy.Getinfo(curl.INFO_NAMELOOKUP_TIME)
	m.NameLookupTime = namelookup_time.(float64)

	starttransfer_time, _ := easy.Getinfo(curl.INFO_STARTTRANSFER_TIME)
	m.StartTransferTime = starttransfer_time.(float64)

	total_time, _ := easy.Getinfo(curl.INFO_TOTAL_TIME)
	m.TotalTime = total_time.(float64)

	local_ip, _ := easy.Getinfo(curl.INFO_LOCAL_IP)
	m.LocalIp = local_ip.(string)

	primary_ip, _ := easy.Getinfo(curl.INFO_PRIMARY_IP)
	m.PrimaryIp = primary_ip.(string)

	return m
}

func measurer(config Config, toMeasurer chan Check, toRecorder chan Measurement) {
	for {
		c := <-toMeasurer
		m := c.Measure(config)

		toRecorder <- m
	}
}

func recorder(config Config, toRecorder chan Measurement) {
	h := func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		enc := json.NewEncoder(w)
		for {
			enc.Encode(<-toRecorder)
		}
	}

	http.HandleFunc("/measurements", h)
	http.ListenAndServe(":5000", nil)
}

func getChecks(config Config) []Check {
	url := config.ChecksUrl

	res, err := http.Get(url)
	if err != nil {
		panic(err)
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		panic(err)
	}

	var checks []Check
	err = json.Unmarshal(body, &checks)
	if err != nil {
		panic(err)
	}

	return checks
}

func scheduler(check Check, toMeasurer chan Check) {
	for {
		toMeasurer <- check
		time.Sleep(1000 * time.Millisecond)
	}
}

func main() {
	config := Config{}
	flag.StringVar(&config.Location, "location", "undefined", "location of this sensor")
	flag.StringVar(&config.ChecksUrl, "checks_url", "https://s3.amazonaws.com/canary-public-data/checks.json", "URL for check data")
	flag.StringVar(&config.MeasurementsUrl, "measurements_url", "http://localhost:5000/measurements", "URL to POST measurements to")
	flag.IntVar(&config.MeasurerCount, "measurer_count", 1, "number of measurers to run")
	flag.Parse()

	u, err := url.Parse(config.MeasurementsUrl)
	if err != nil {
		panic(err)
	}

	if u.User != nil {
		config.MeasurementsUser = u.User.Username()
		config.MeasurementsPass, _ = u.User.Password()
	}

	check_list := getChecks(config)

	toMeasurer := make(chan Check)
	toRecorder := make(chan Measurement)

	for i := 0; i < config.MeasurerCount; i++ {
		go measurer(config, toMeasurer, toRecorder)
	}

	go recorder(config, toRecorder)

	for _, c := range check_list {
		go scheduler(c, toMeasurer)
	}

	select {}
}
<commit_msg>s/recorder/streamer<commit_after>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/andelf/go-curl"
	"github.com/nu7hatch/gouuid"
)

type Config struct {
	Location         string
	ChecksUrl        string
	MeasurementsUrl  string
	MeasurementsUser string
	MeasurementsPass string
	MeasurerCount    int
	RecorderCount    int
}

type Check struct {
	Id  string `json:"id"`
	Url string `json:"url"`
}

type Measurement struct {
	Check             Check   `json:"check"`
	Id                string  `json:"id"`
	Location          string  `json:"location"`
	T                 int     `json:"t"`
	ExitStatus        int     `json:"exit_status"`
	ConnectTime       float64 `json:"connect_time,omitempty"`
	StartTransferTime float64 `json:"starttransfer_time,omitempty"`
	LocalIp           string  `json:"local_ip,omitempty"`
	PrimaryIp         string  `json:"primary_ip,omitempty"`
	TotalTime         float64 `json:"total_time,omitempty"`
	HttpStatus        int     `json:"http_status,omitempty"`
	NameLookupTime    float64 `json:"namelookup_time,omitempty"`
}

func (c *Check) Measure(config Config) Measurement {
	var m Measurement

	id, _ := uuid.NewV4()
	m.Id = id.String()
	m.Check = *c
	m.Location = config.Location

	easy := curl.EasyInit()
	defer easy.Cleanup()

	easy.Setopt(curl.OPT_URL, c.Url)

	// dummy func for curl output
	noOut := func(buf []byte, userdata interface{}) bool {
		return true
	}

	easy.Setopt(curl.OPT_WRITEFUNCTION, noOut)
	easy.Setopt(curl.OPT_CONNECTTIMEOUT, 10)
	easy.Setopt(curl.OPT_TIMEOUT, 10)

	now := time.Now()
	m.T = int(now.Unix())

	if err := easy.Perform(); err != nil {
		if e, ok := err.(curl.CurlError); ok {
			m.ExitStatus = (int(e))
			return m
		}
		os.Exit(1)
	}

	m.ExitStatus = 0
	http_status, _ := easy.Getinfo(curl.INFO_RESPONSE_CODE)
	m.HttpStatus = http_status.(int)

	connect_time, _ := easy.Getinfo(curl.INFO_CONNECT_TIME)
	m.ConnectTime = connect_time.(float64)

	namelookup_time, _ := easy.Getinfo(curl.INFO_NAMELOOKUP_TIME)
	m.NameLookupTime = namelookup_time.(float64)

	starttransfer_time, _ := easy.Getinfo(curl.INFO_STARTTRANSFER_TIME)
	m.StartTransferTime = starttransfer_time.(float64)

	total_time, _ := easy.Getinfo(curl.INFO_TOTAL_TIME)
	m.TotalTime = total_time.(float64)

	local_ip, _ := easy.Getinfo(curl.INFO_LOCAL_IP)
	m.LocalIp = local_ip.(string)

	primary_ip, _ := easy.Getinfo(curl.INFO_PRIMARY_IP)
	m.PrimaryIp = primary_ip.(string)

	return m
}

func measurer(config Config, toMeasurer chan Check, toStreamer chan Measurement) {
	for {
		c := <-toMeasurer
		m := c.Measure(config)

		toStreamer <- m
	}
}

func streamer(config Config, toStreamer chan Measurement) {
	h := func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		enc := json.NewEncoder(w)
		for {
			enc.Encode(<-toStreamer)
		}
	}

	http.HandleFunc("/measurements", h)
	http.ListenAndServe(":5000", nil)
}

func getChecks(config Config) []Check {
	url := config.ChecksUrl

	res, err := http.Get(url)
	if err != nil {
		panic(err)
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		panic(err)
	}

	var checks []Check
	err = json.Unmarshal(body, &checks)
	if err != nil {
		panic(err)
	}

	return checks
}

func scheduler(check Check, toMeasurer chan Check) {
	for {
		toMeasurer <- check
		time.Sleep(1000 * time.Millisecond)
	}
}

func main() {
	config := Config{}
	flag.StringVar(&config.Location, "location", "undefined", "location of this sensor")
	flag.StringVar(&config.ChecksUrl, "checks_url", "https://s3.amazonaws.com/canary-public-data/checks.json", "URL for check data")
	flag.StringVar(&config.MeasurementsUrl, "measurements_url", "http://localhost:5000/measurements", "URL to POST measurements to")
	flag.IntVar(&config.MeasurerCount, "measurer_count", 1, "number of measurers to run")
	flag.Parse()

	u, err := url.Parse(config.MeasurementsUrl)
	if err != nil {
		panic(err)
	}

	if u.User != nil {
		config.MeasurementsUser = u.User.Username()
		config.MeasurementsPass, _ = u.User.Password()
	}

	check_list := getChecks(config)

	toMeasurer := make(chan Check)
	toStreamer := make(chan Measurement)

	for i := 0; i < config.MeasurerCount; i++ {
		go measurer(config, toMeasurer, toStreamer)
	}

	go streamer(config, toStreamer)

	for _, c := range check_list {
		go scheduler(c, toMeasurer)
	}

	select {}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/stripe/sequins/backend"
	"github.com/stripe/sequins/index"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

type sequinsOptions struct {
	LocalPath           string
	CheckForSuccessFile bool
}

type sequins struct {
	options      sequinsOptions
	backend      backend.Backend
	indexReference index.IndexReference
	http         *http.Server
	started      time.Time
	updated      time.Time
	reloadLock   sync.Mutex
}

type status struct {
	Path    string `json:"path"`
	Started int64  `json:"started"`
	Updated int64  `json:"updated"`
	Count   int    `json:"count"`
	Version string `json:"version"`
}

func newSequins(backend backend.Backend, options sequinsOptions) *sequins {
	return &sequins{
		options:    options,
		backend:    backend,
		reloadLock: sync.Mutex{},
	}
}

func (s *sequins) init() error {
	err := s.refresh()
	if err != nil {
		return err
	}

	now := time.Now()
	s.started = now
	s.updated = now

	return nil
}

func (s *sequins) start(address string) error {
	// TODO: we may need a more graceful way of shutting down, since this will
	// cause requests that start processing after this runs to 500
	// However, this may not be a problem, since you have to shift traffic to
	// another instance before shutting down anyway, otherwise you'd have downtime

	defer s.indexReference.Replace(nil).Close()

	log.Printf("Listening on %s", address)
	return http.ListenAndServe(address, s)
}

func (s *sequins) reloadLatest() error {
	err := s.refresh()
	if err != nil {
		return err
	}

	s.updated = time.Now()

	return nil
}

func (s *sequins) refresh() error {
	s.reloadLock.Lock()
	defer s.reloadLock.Unlock()

	version, err := s.backend.LatestVersion(s.options.CheckForSuccessFile)
	if err != nil {
		return err
	}

	// We can use unsafe ref, since closing the index would not affect the version string
	var currentVersion string
	currentIndex := s.indexReference.UnsafeGet()
	if currentIndex != nil {
		currentVersion = currentIndex.Version
	}

	if version != currentVersion {
		path := filepath.Join(s.options.LocalPath, version)

		err := os.Mkdir(path, 0700|os.ModeDir)
		if err != nil && !os.IsExist(err) {
			return err
		}

		if os.IsExist(err) {
			log.Printf("Version %s is already downloaded", version)
		} else {
			log.Printf("Downloading version %s from %s", version, s.backend.DisplayPath(version))
			err = s.backend.Download(version, path)
			if err != nil {
				return err
			}
		}

		log.Printf("Preparing version %s at %s", version, path)
		index := index.New(path, version)
		err = index.Load()
		if err != nil {
			return fmt.Errorf("Error while indexing: %s", err)
		}

		log.Printf("Switching to version %s!", version)

		oldIndex := s.indexReference.Replace(index)
		if oldIndex != nil {
			oldIndex.Close()
		}
	} else {
		log.Printf("%s is already the newest version, so not reloading.", version)
	}

	return nil
}

func (s *sequins) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "/" {
		index := s.indexReference.Get()
		count, err := index.Count()
		currentVersion := index.Version
		s.indexReference.Release(index)

		if err != nil {
			log.Fatal(err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}

		status := status{
			Path:    s.backend.DisplayPath(currentVersion),
			Version: currentVersion,
			Started: s.started.Unix(),
			Updated: s.updated.Unix(),
			Count:   count,
		}

		jsonBytes, err := json.Marshal(status)
		if err != nil {
			log.Fatal(err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}

		w.Write(jsonBytes)
		return
	}

	key := strings.TrimPrefix(r.URL.Path, "/")

	currentIndex := s.indexReference.Get()
	res, err := currentIndex.Get(key)
	currentVersion := currentIndex.Version
	s.indexReference.Release(currentIndex)

	if err == index.ErrNotFound {
		w.WriteHeader(http.StatusNotFound)
	} else if err != nil {
		log.Fatal(fmt.Errorf("Error fetching value for %s: %s", key, err))
		w.WriteHeader(http.StatusInternalServerError)
	} else {
		// Explicitly unset Content-Type, so ServeContent doesn't try to do any
		// sniffing.
		w.Header()["Content-Type"] = nil

		w.Header().Add("X-Sequins-Version", currentVersion)

		http.ServeContent(w, r, key, s.updated, bytes.NewReader(res))
	}
}
<commit_msg>Fix bug in cleanup code<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/stripe/sequins/backend"
	"github.com/stripe/sequins/index"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

type sequinsOptions struct {
	LocalPath           string
	CheckForSuccessFile bool
}

type sequins struct {
	options      sequinsOptions
	backend      backend.Backend
	indexReference index.IndexReference
	http         *http.Server
	started      time.Time
	updated      time.Time
	reloadLock   sync.Mutex
}

type status struct {
	Path    string `json:"path"`
	Started int64  `json:"started"`
	Updated int64  `json:"updated"`
	Count   int    `json:"count"`
	Version string `json:"version"`
}

func newSequins(backend backend.Backend, options sequinsOptions) *sequins {
	return &sequins{
		options:    options,
		backend:    backend,
		reloadLock: sync.Mutex{},
	}
}

func (s *sequins) init() error {
	err := s.refresh()
	if err != nil {
		return err
	}

	now := time.Now()
	s.started = now
	s.updated = now

	return nil
}

func (s *sequins) start(address string) error {
	// TODO: we may need a more graceful way of shutting down, since this will
	// cause requests that start processing after this runs to 500
	// However, this may not be a problem, since you have to shift traffic to
	// another instance before shutting down anyway, otherwise you'd have downtime

	defer func() {
		s.indexReference.Replace(nil).Close()
	}()

	log.Printf("Listening on %s", address)
	return http.ListenAndServe(address, s)
}

func (s *sequins) reloadLatest() error {
	err := s.refresh()
	if err != nil {
		return err
	}

	s.updated = time.Now()

	return nil
}

func (s *sequins) refresh() error {
	s.reloadLock.Lock()
	defer s.reloadLock.Unlock()

	version, err := s.backend.LatestVersion(s.options.CheckForSuccessFile)
	if err != nil {
		return err
	}

	// We can use unsafe ref, since closing the index would not affect the version string
	var currentVersion string
	currentIndex := s.indexReference.UnsafeGet()
	if currentIndex != nil {
		currentVersion = currentIndex.Version
	}

	if version != currentVersion {
		path := filepath.Join(s.options.LocalPath, version)

		err := os.Mkdir(path, 0700|os.ModeDir)
		if err != nil && !os.IsExist(err) {
			return err
		}

		if os.IsExist(err) {
			log.Printf("Version %s is already downloaded", version)
		} else {
			log.Printf("Downloading version %s from %s", version, s.backend.DisplayPath(version))
			err = s.backend.Download(version, path)
			if err != nil {
				return err
			}
		}

		log.Printf("Preparing version %s at %s", version, path)
		index := index.New(path, version)
		err = index.Load()
		if err != nil {
			return fmt.Errorf("Error while indexing: %s", err)
		}

		log.Printf("Switching to version %s!", version)

		oldIndex := s.indexReference.Replace(index)
		if oldIndex != nil {
			oldIndex.Close()
		}
	} else {
		log.Printf("%s is already the newest version, so not reloading.", version)
	}

	return nil
}

func (s *sequins) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "/" {
		index := s.indexReference.Get()
		count, err := index.Count()
		currentVersion := index.Version
		s.indexReference.Release(index)

		if err != nil {
			log.Fatal(err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}

		status := status{
			Path:    s.backend.DisplayPath(currentVersion),
			Version: currentVersion,
			Started: s.started.Unix(),
			Updated: s.updated.Unix(),
			Count:   count,
		}

		jsonBytes, err := json.Marshal(status)
		if err != nil {
			log.Fatal(err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}

		w.Write(jsonBytes)
		return
	}

	key := strings.TrimPrefix(r.URL.Path, "/")

	currentIndex := s.indexReference.Get()
	res, err := currentIndex.Get(key)
	currentVersion := currentIndex.Version
	s.indexReference.Release(currentIndex)

	if err == index.ErrNotFound {
		w.WriteHeader(http.StatusNotFound)
	} else if err != nil {
		log.Fatal(fmt.Errorf("Error fetching value for %s: %s", key, err))
		w.WriteHeader(http.StatusInternalServerError)
	} else {
		// Explicitly unset Content-Type, so ServeContent doesn't try to do any
		// sniffing.
		w.Header()["Content-Type"] = nil

		w.Header().Add("X-Sequins-Version", currentVersion)

		http.ServeContent(w, r, key, s.updated, bytes.NewReader(res))
	}
}
<|endoftext|>