<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/kylelemons/go-gypsy/yaml"
	"github.com/lib/pq"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

const OPEN311_API_URI = "http://311api.cityofchicago.org/open311/v2/requests.json?extensions=true&page_size=500"

type Open311Request struct {
	Lat, Long                                                                                float64
	Ward, Police_district                                                                                int
	Service_request_id, Status, Service_name, Service_code, Agency_responsible, Address, Channel, Media_url string
	Requested_datetime, Updated_datetime                                                                    string // FIXME: should these be proper time objects?
	Extended_attributes                                                                                map[string]interface{}
	Notes                                                                                []map[string]interface{}
}

type Worker struct {
	Db         *sql.DB
	LastRunAt  time.Time
	InsertStmt *sql.Stmt
	UpdateStmt *sql.Stmt
}

var worker Worker

//  command line flags
var (
	environment   = flag.String("environment", "", "Environment to run in, e.g. staging, production")
	config        = flag.String("config", "./config/database.yml", "database configuration file")
	sr_number     = flag.String("sr-number", "", "SR number to fetch")
	backfill      = flag.Bool("backfill", false, "run in reverse and backfill data")
	backfill_date = flag.String("backfill-from", "", "date to start backfilling data from. Use RFC3339 format. Default will be the time of the least recently updated SR in the database.")
)

func init() {
	flag.Parse()

	log.Printf("running in %s environment, configuration file %s", *environment, *config)
	settings := yaml.ConfigFile(*config)

	// setup database connection
	driver, err := settings.Get(fmt.Sprintf("%s.driver", *environment))
	if err != nil {
		log.Fatal("error loading db driver", err)
	}

	connstr, err := settings.Get(fmt.Sprintf("%s.connstr", *environment))
	if err != nil {
		log.Fatal("error loading db connstr", err)
	}

	db, err := sql.Open(driver, connstr)
	if err != nil {
		log.Fatal("Cannot open database connection", err)
	}

	log.Printf("database connstr: %s", connstr)

	worker.Db = db
	worker.SetupStmts()
}

func main() {
	defer worker.Db.Close()

	if *sr_number != "" {
		sr := fetchSingleRequest(*sr_number)
		sr.Save()
		return
	}

	start_backfill_from := *backfill_date
	for {
		switch {
		case *backfill:
			requests := backFillRequests(start_backfill_from)
			for _, request := range requests {
				request.Save()
			}

			start_backfill_from = requests[len(requests)-1].Updated_datetime // FIXME: is it safe to assume the items are sorted?

		case time.Since(worker.LastRunAt) > (30 * time.Second):
			// load requests from open311
			for _, request := range fetchRequests() {
				request.Save()
			}
			worker.LastRunAt = time.Now()
		default:
			log.Print("sleeping for 10 seconds")
			time.Sleep(10 * time.Second)
		}
	}
}

func (w *Worker) SetupStmts() {
	insert, err := worker.Db.Prepare(`INSERT INTO service_requests(service_request_id,
		status, service_name, service_code, agency_responsible,
		address, requested_datetime, updated_datetime, lat, long,
		ward, police_district, media_url, channel, duplicate, parent_service_request_id, closed_datetime, notes)
		VALUES ($1::varchar, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18);`)

	if err != nil {
		fmt.Print("error preparing insert statement ", err)
	}
	w.InsertStmt = insert

	update, err := worker.Db.Prepare(`UPDATE service_requests SET
		status = $2, service_name = $3, service_code = $4, agency_responsible = $5,
		address = $6, requested_datetime = $7, updated_datetime = $8, lat = $9, long = $10,
		ward = $11, police_district = $12, media_url = $13, channel = $14, duplicate = $15,
		parent_service_request_id = $16, updated_at = NOW(), closed_datetime = $17, notes = $18 WHERE service_request_id = $1;`)

	if err != nil {
		fmt.Print("error preparing update statement ", err)
	}
	w.UpdateStmt = update
}

func (req Open311Request) String() string {
	// pretty print SR information
	return fmt.Sprintf("%s: %s at %s %f,%f, last update %s", req.Service_request_id, req.Service_name, req.Address, req.Lat, req.Long, req.Updated_datetime)
}

func (req Open311Request) Save() (persisted bool) {
	// create or update a SR

	// open311 says we should always ignore a SR that does not have a SR# assigned
	if req.Service_request_id == "" {
		log.Printf("cowardly refusing to create a new SR record because of empty SR#. Request type is %s", req.Service_name)
		return false
	}

	persisted = false

	// find existing record if exists
	var existing_id int
	err := worker.Db.QueryRow("SELECT id FROM service_requests WHERE service_request_id = $1", req.Service_request_id).Scan(&existing_id)
	switch {
	case err == sql.ErrNoRows:
		// log.Printf("did not find existing record %s", req.Service_request_id)
	case err != nil:
		log.Print("error searching for existing SR", err)
	default:
		persisted = true
		// log.Printf("found existing sr %s", req.Service_request_id)
	}

	var stmt *sql.Stmt

	if !persisted {
		stmt = worker.InsertStmt
	} else {
		stmt = worker.UpdateStmt
	}

	t := req.ExtractClosedDatetime()
	closed_time := pq.NullTime{Time: t, Valid: !t.IsZero()}
	notes_as_json, err := json.Marshal(req.Notes)
	if err != nil {
		log.Print("error marshaling notes to JSON: ", err)
	}

	_, err = stmt.Exec(req.Service_request_id,
		req.Status,
		req.Service_name,
		req.Service_code,
		req.Agency_responsible,
		req.Address,
		req.Requested_datetime,
		req.Updated_datetime,
		req.Lat,
		req.Long,
		req.Extended_attributes["ward"],
		req.Extended_attributes["police_district"],
		req.Media_url,
		req.Extended_attributes["channel"],
		req.Extended_attributes["duplicate"],
		req.Extended_attributes["parent_service_request_id"],
		closed_time,
		notes_as_json,
	)

	if err != nil {
		log.Printf("[error] could not update %s because %s", req.Service_request_id, err)
	} else {
		var verb string
		switch {
		case !persisted && closed_time.Time.IsZero():
			verb = "CREATED"
		case !persisted && !closed_time.Time.IsZero():
			verb = "CREATED/CLOSED"
		case persisted && closed_time.Time.IsZero():
			verb = "UPDATED"
		case persisted && !closed_time.Time.IsZero():
			verb = "UPDATED/CLOSED"
		}

		log.Printf("[%s] %s", verb, req)
		persisted = true
	}

	return persisted
}

func (req Open311Request) ExtractClosedDatetime() time.Time {
	// given an extended_attributes JSON blob, pluck out the closed time, if present
	// req.PrintNotes()

	var closed_at time.Time
	for _, note := range req.Notes {
		if note["type"] == "closed" {
			parsed_date, err := time.Parse("2006-01-02T15:04:05-07:00", note["datetime"].(string))
			if err != nil {
				log.Print("error parsing date", err)
			}
			log.Printf("SR %s closed at: %s", req, parsed_date)
			closed_at = parsed_date
		}
	}

	return closed_at
}

func (req Open311Request) PrintNotes() {
	fmt.Printf("Notes for SR %s:\n", req.Service_request_id)

	for _, note := range req.Notes {
		fmt.Printf("%+v\n", note)
	}
}

func fetchSingleRequest(sr_number string) (request Open311Request) {
	// given an SR, fetch the record
	log.Printf("fetching single SR %s", sr_number)
	open311_api_endpoint := fmt.Sprintf("http://311api.cityofchicago.org/open311/v2/requests/%s.json?extensions=true", sr_number)

	log.Printf("fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	var requests []Open311Request

	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("error parsing JSON:", err)
	}

	log.Printf("received %d requests from Open311", len(requests))

	return requests[0]
}

func fetchRequests() (requests []Open311Request) {
	last_updated_at := time.Now()
	if err := worker.Db.QueryRow("SELECT MAX(updated_datetime) FROM service_requests;").Scan(&last_updated_at); err != nil {
		log.Print("[fetchRequests] error loading most recent SR, will fallback to current time: ", err)
	}

	log.Print("[fetchRequests] most recent SR timestamp ", last_updated_at.Format(time.RFC3339))

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_after=" + last_updated_at.Format(time.RFC3339)

	log.Printf("[fetchRequests] fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("[fetchRequests] error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[fetchRequests] error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[fetchRequests] error parsing JSON:", err)
	}

	log.Printf("[fetchRequests] received %d requests from Open311", len(requests))

	return requests
}

func backFillRequests(start_from string) (requests []Open311Request) {
	var fetch_from time.Time

	if start_from == "" {
		err := worker.Db.QueryRow("SELECT updated_datetime FROM service_requests ORDER BY updated_datetime ASC LIMIT 1").Scan(&fetch_from)
		if err != nil {
			log.Println("error fetching oldest SR:", err)
		}
		log.Printf("no start_from value provided, so falling back to oldest (by last update) SR in the database: %s", fetch_from)
	} else {
		t, err := time.Parse(time.RFC3339, start_from)
		if err != nil {
			log.Fatal("[backfill] error parsing date to start from", err)
		}
		fetch_from = t
	}

	formatted_date_string_with_tz := fetch_from.Format(time.RFC3339)

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_before=" + formatted_date_string_with_tz

	log.Printf("[backfill] fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("[backfill] error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[backfill] error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[backfill] error parsing JSON:", err)
	}

	log.Printf("[backfill] received %d requests from Open311", len(requests))

	return requests
}
<commit_msg>move the err check before the defer call<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/kylelemons/go-gypsy/yaml"
	"github.com/lib/pq"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

const OPEN311_API_URI = "http://311api.cityofchicago.org/open311/v2/requests.json?extensions=true&page_size=500"

type Open311Request struct {
	Lat, Long                                                                                float64
	Ward, Police_district                                                                                int
	Service_request_id, Status, Service_name, Service_code, Agency_responsible, Address, Channel, Media_url string
	Requested_datetime, Updated_datetime                                                                    string // FIXME: should these be proper time objects?
	Extended_attributes                                                                                map[string]interface{}
	Notes                                                                                []map[string]interface{}
}

type Worker struct {
	Db         *sql.DB
	LastRunAt  time.Time
	InsertStmt *sql.Stmt
	UpdateStmt *sql.Stmt
}

var worker Worker

//  command line flags
var (
	environment   = flag.String("environment", "", "Environment to run in, e.g. staging, production")
	config        = flag.String("config", "./config/database.yml", "database configuration file")
	sr_number     = flag.String("sr-number", "", "SR number to fetch")
	backfill      = flag.Bool("backfill", false, "run in reverse and backfill data")
	backfill_date = flag.String("backfill-from", "", "date to start backfilling data from. Use RFC3339 format. Default will be the time of the least recently updated SR in the database.")
)

func init() {
	flag.Parse()

	log.Printf("running in %s environment, configuration file %s", *environment, *config)
	settings := yaml.ConfigFile(*config)

	// setup database connection
	driver, err := settings.Get(fmt.Sprintf("%s.driver", *environment))
	if err != nil {
		log.Fatal("error loading db driver", err)
	}

	connstr, err := settings.Get(fmt.Sprintf("%s.connstr", *environment))
	if err != nil {
		log.Fatal("error loading db connstr", err)
	}

	db, err := sql.Open(driver, connstr)
	if err != nil {
		log.Fatal("Cannot open database connection", err)
	}

	log.Printf("database connstr: %s", connstr)

	worker.Db = db
	worker.SetupStmts()
}

func main() {
	defer worker.Db.Close()

	if *sr_number != "" {
		sr := fetchSingleRequest(*sr_number)
		sr.Save()
		return
	}

	start_backfill_from := *backfill_date
	for {
		switch {
		case *backfill:
			requests := backFillRequests(start_backfill_from)
			for _, request := range requests {
				request.Save()
			}

			start_backfill_from = requests[len(requests)-1].Updated_datetime // FIXME: is it safe to assume the items are sorted?

		case time.Since(worker.LastRunAt) > (30 * time.Second):
			// load requests from open311
			for _, request := range fetchRequests() {
				request.Save()
			}
			worker.LastRunAt = time.Now()
		default:
			log.Print("sleeping for 10 seconds")
			time.Sleep(10 * time.Second)
		}
	}
}

func (w *Worker) SetupStmts() {
	insert, err := worker.Db.Prepare(`INSERT INTO service_requests(service_request_id,
		status, service_name, service_code, agency_responsible,
		address, requested_datetime, updated_datetime, lat, long,
		ward, police_district, media_url, channel, duplicate, parent_service_request_id, closed_datetime, notes)
		VALUES ($1::varchar, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18);`)

	if err != nil {
		fmt.Print("error preparing insert statement ", err)
	}
	w.InsertStmt = insert

	update, err := worker.Db.Prepare(`UPDATE service_requests SET
		status = $2, service_name = $3, service_code = $4, agency_responsible = $5,
		address = $6, requested_datetime = $7, updated_datetime = $8, lat = $9, long = $10,
		ward = $11, police_district = $12, media_url = $13, channel = $14, duplicate = $15,
		parent_service_request_id = $16, updated_at = NOW(), closed_datetime = $17, notes = $18 WHERE service_request_id = $1;`)

	if err != nil {
		fmt.Print("error preparing update statement ", err)
	}
	w.UpdateStmt = update
}

func (req Open311Request) String() string {
	// pretty print SR information
	return fmt.Sprintf("%s: %s at %s %f,%f, last update %s", req.Service_request_id, req.Service_name, req.Address, req.Lat, req.Long, req.Updated_datetime)
}

func (req Open311Request) Save() (persisted bool) {
	// create or update a SR

	// open311 says we should always ignore a SR that does not have a SR# assigned
	if req.Service_request_id == "" {
		log.Printf("cowardly refusing to create a new SR record because of empty SR#. Request type is %s", req.Service_name)
		return false
	}

	persisted = false

	// find existing record if exists
	var existing_id int
	err := worker.Db.QueryRow("SELECT id FROM service_requests WHERE service_request_id = $1", req.Service_request_id).Scan(&existing_id)
	switch {
	case err == sql.ErrNoRows:
		// log.Printf("did not find existing record %s", req.Service_request_id)
	case err != nil:
		log.Print("error searching for existing SR", err)
	default:
		persisted = true
		// log.Printf("found existing sr %s", req.Service_request_id)
	}

	var stmt *sql.Stmt

	if !persisted {
		stmt = worker.InsertStmt
	} else {
		stmt = worker.UpdateStmt
	}

	t := req.ExtractClosedDatetime()
	closed_time := pq.NullTime{Time: t, Valid: !t.IsZero()}
	notes_as_json, err := json.Marshal(req.Notes)
	if err != nil {
		log.Print("error marshaling notes to JSON: ", err)
	}

	_, err = stmt.Exec(req.Service_request_id,
		req.Status,
		req.Service_name,
		req.Service_code,
		req.Agency_responsible,
		req.Address,
		req.Requested_datetime,
		req.Updated_datetime,
		req.Lat,
		req.Long,
		req.Extended_attributes["ward"],
		req.Extended_attributes["police_district"],
		req.Media_url,
		req.Extended_attributes["channel"],
		req.Extended_attributes["duplicate"],
		req.Extended_attributes["parent_service_request_id"],
		closed_time,
		notes_as_json,
	)

	if err != nil {
		log.Printf("[error] could not update %s because %s", req.Service_request_id, err)
	} else {
		var verb string
		switch {
		case !persisted && closed_time.Time.IsZero():
			verb = "CREATED"
		case !persisted && !closed_time.Time.IsZero():
			verb = "CREATED/CLOSED"
		case persisted && closed_time.Time.IsZero():
			verb = "UPDATED"
		case persisted && !closed_time.Time.IsZero():
			verb = "UPDATED/CLOSED"
		}

		log.Printf("[%s] %s", verb, req)
		persisted = true
	}

	return persisted
}

func (req Open311Request) ExtractClosedDatetime() time.Time {
	// given an extended_attributes JSON blob, pluck out the closed time, if present
	// req.PrintNotes()

	var closed_at time.Time
	for _, note := range req.Notes {
		if note["type"] == "closed" {
			parsed_date, err := time.Parse("2006-01-02T15:04:05-07:00", note["datetime"].(string))
			if err != nil {
				log.Print("error parsing date", err)
			}
			log.Printf("SR %s closed at: %s", req, parsed_date)
			closed_at = parsed_date
		}
	}

	return closed_at
}

func (req Open311Request) PrintNotes() {
	fmt.Printf("Notes for SR %s:\n", req.Service_request_id)

	for _, note := range req.Notes {
		fmt.Printf("%+v\n", note)
	}
}

func fetchSingleRequest(sr_number string) (request Open311Request) {
	// given an SR, fetch the record
	log.Printf("fetching single SR %s", sr_number)
	open311_api_endpoint := fmt.Sprintf("http://311api.cityofchicago.org/open311/v2/requests/%s.json?extensions=true", sr_number)

	log.Printf("fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	var requests []Open311Request

	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("error parsing JSON:", err)
	}

	log.Printf("received %d requests from Open311", len(requests))

	return requests[0]
}

func fetchRequests() (requests []Open311Request) {
	last_updated_at := time.Now()
	if err := worker.Db.QueryRow("SELECT MAX(updated_datetime) FROM service_requests;").Scan(&last_updated_at); err != nil {
		log.Print("[fetchRequests] error loading most recent SR, will fallback to current time: ", err)
	}

	log.Print("[fetchRequests] most recent SR timestamp ", last_updated_at.Format(time.RFC3339))

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_after=" + last_updated_at.Format(time.RFC3339)

	log.Printf("[fetchRequests] fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)

	if err != nil {
		log.Fatalln("[fetchRequests] error fetching from Open311 endpoint", err)
	}

	defer resp.Body.Close()

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[fetchRequests] error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[fetchRequests] error parsing JSON:", err)
	}

	log.Printf("[fetchRequests] received %d requests from Open311", len(requests))

	return requests
}

func backFillRequests(start_from string) (requests []Open311Request) {
	var fetch_from time.Time

	if start_from == "" {
		err := worker.Db.QueryRow("SELECT updated_datetime FROM service_requests ORDER BY updated_datetime ASC LIMIT 1").Scan(&fetch_from)
		if err != nil {
			log.Println("error fetching oldest SR:", err)
		}
		log.Printf("no start_from value provided, so falling back to oldest (by last update) SR in the database: %s", fetch_from)
	} else {
		t, err := time.Parse(time.RFC3339, start_from)
		if err != nil {
			log.Fatal("[backfill] error parsing date to start from", err)
		}
		fetch_from = t
	}

	formatted_date_string_with_tz := fetch_from.Format(time.RFC3339)

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_before=" + formatted_date_string_with_tz

	log.Printf("[backfill] fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("[backfill] error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[backfill] error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[backfill] error parsing JSON:", err)
	}

	log.Printf("[backfill] received %d requests from Open311", len(requests))

	return requests
}
<|endoftext|>


<commit_before>package skiplist

import "testing"

func TestSplit(t *testing.T) {
	list1 := NewList()
	for i := 10; i > 0; i-- {
		list1.Insert(i, []byte{})
	}
	list2 := list1.Split(8)

	if list1.length != 7 {
		t.Errorf("Expected list 1 to be split into 7 nodes but has %+v", list1.length)
	}
	for i := 1; i < list1.length; i++ {
		if list1.Search(i) == nil {
			t.Errorf("Expected list1 to have node for key %+v", i)
		}
	}
	for i := 8; i <= 10; i++ {
		if list1.Search(i) != nil {
			t.Errorf("Expected list1 to not have node for key %+v", i)
		}
	}

	if list2.length != 3 {
		t.Errorf("Expected list 2 to be split into 3 nodes but has %+v", list2.length)
	}
	for i := 1; i < list2.length; i++ {
		if list2.Search(i+7) == nil {
			t.Errorf("Expected list2 to have node for key %+v", i+7)
		}
	}
}

func TestSplit_NoValuesAbove(t *testing.T) {
	list1 := NewList()
	for i := 10; i > 0; i-- {
		list1.Insert(i, []byte{})
	}
	list2 := list1.Split(18)

	if list1.length != 10 {
		t.Errorf("Expected list 1 to not be split but has %+v nodes", list1.length)
	}
	for i := 1; i < list1.length; i++ {
		if list1.Search(i) == nil {
			t.Errorf("Expected list1 to have node for key %+v", i)
		}
	}

	if list2.length != 0 {
		t.Errorf("Expected list 2 to be empty but has %+v nodes", list2.length)
	}
}
<commit_msg>Mark test pending<commit_after>package skiplist

import "testing"

func TestSplit(t *testing.T) {
	//pending
	// list1 := NewList()
	// for i := 10; i > 0; i-- {
	// 	list1.Insert(i, []byte{})
	// }
	// list2 := list1.Split(8)
	//
	// if list1.length != 7 {
	// 	t.Errorf("Expected list 1 to be split into 7 nodes but has %+v", list1.length)
	// }
	// for i := 1; i < list1.length; i++ {
	// 	if list1.Search(i) == nil {
	// 		t.Errorf("Expected list1 to have node for key %+v", i)
	// 	}
	// }
	// for i := 8; i <= 10; i++ {
	// 	if list1.Search(i) != nil {
	// 		t.Errorf("Expected list1 to not have node for key %+v", i)
	// 	}
	// }
	//
	// if list2.length != 3 {
	// 	t.Errorf("Expected list 2 to be split into 3 nodes but has %+v", list2.length)
	// }
	// for i := 1; i < list2.length; i++ {
	// 	if list2.Search(i+7) == nil {
	// 		t.Errorf("Expected list2 to have node for key %+v", i+7)
	// 	}
	// }
}

func TestSplit_NoValuesAbove(t *testing.T) {
	list1 := NewList()
	for i := 10; i > 0; i-- {
		list1.Insert(i, []byte{})
	}
	list2 := list1.Split(18)

	if list1.length != 10 {
		t.Errorf("Expected list 1 to not be split but has %+v nodes", list1.length)
	}
	for i := 1; i < list1.length; i++ {
		if list1.Search(i) == nil {
			t.Errorf("Expected list1 to have node for key %+v", i)
		}
	}

	if list2.length != 0 {
		t.Errorf("Expected list 2 to be empty but has %+v nodes", list2.length)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package externaldns

import (
	"fmt"
	"reflect"
	"testing"
)

func TestParseFlags(t *testing.T) {
	for _, ti := range []struct {
		title       string
		args        []arg
		expectError bool
		expected    *Config
	}{
		{
			title: "set in-cluster true",
			args: []arg{
				arg{
					flag: "--in-cluster",
					val:  "",
				},
			},
			expected: &Config{
				InCluster:     true,
				KubeConfig:    "",
				GoogleProject: "",
				GoogleZone:    "",
				HealthPort:    defaultHealthPort,
				DryRun:        true,
				Debug:         false,
				LogFormat:     defaultLogFormat,
			},
		},
		{
			title: "all default",
			args:  []arg{},
			expected: &Config{
				InCluster:     false,
				KubeConfig:    "",
				GoogleProject: "",
				GoogleZone:    "",
				HealthPort:    defaultHealthPort,
				DryRun:        true,
				Debug:         false,
				LogFormat:     defaultLogFormat,
			},
		},
		{
			title: "set string var",
			args: []arg{
				arg{
					flag: "--kubeconfig",
					val:  "myhome",
				},
			},
			expected: &Config{
				InCluster:     false,
				KubeConfig:    "myhome",
				GoogleProject: "",
				GoogleZone:    "",
				HealthPort:    defaultHealthPort,
				DryRun:        true,
				Debug:         false,
				LogFormat:     defaultLogFormat,
			},
		},
		{
			title: "unexpected flag",
			args: []arg{
				arg{
					flag: "--random",
					val:  "myhome",
				},
			},
			expectError: true,
		},
		{
			title: "override default",
			args: []arg{
				arg{
					flag: "--log-format",
					val:  "json",
				},
			},
			expected: &Config{
				InCluster:     false,
				KubeConfig:    "myhome",
				GoogleProject: "",
				GoogleZone:    "",
				HealthPort:    defaultHealthPort,
				DryRun:        true,
				Debug:         false,
				LogFormat:     "json",
			},
		},
	} {
		t.Run(ti.title, func(t *testing.T) {
			cfg := NewConfig()
			spaceArgs := []string{"external-dns"}
			for _, arg := range ti.args {
				spaceArgs = append(spaceArgs, arg.Slice()...)
			}
			err := cfg.ParseFlags(spaceArgs)
			if !ti.expectError && err != nil {
				t.Errorf("unexpected parse flags fail for args %#v, error: %v", ti.args, err)
			}
			if ti.expectError && err == nil {
				t.Errorf("parse flags should fail for args %#v", ti.args)
			}
			if !ti.expectError {
				validateConfig(t, cfg, ti.expected)
			}
		})
	}
}

// helper functions

func validateConfig(t *testing.T, got, expected *Config) {
	fmt.Println(*got, *expected)
	if !reflect.DeepEqual(got, expected) {
		t.Errorf("Config is wrong")
	}
}

type arg struct {
	flag string
	val  string
}

func (a *arg) Slice() []string {
	return []string{a.flag, a.val}
}

func (a *arg) String() string {
	return fmt.Sprintf("%s=%s", a.flag, a.val)
}
<commit_msg>remove dummy printing<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package externaldns

import (
	"fmt"
	"reflect"
	"testing"
)

func TestParseFlags(t *testing.T) {
	for _, ti := range []struct {
		title       string
		args        []arg
		expectError bool
		expected    *Config
	}{
		{
			title: "set in-cluster true",
			args: []arg{
				arg{
					flag: "--in-cluster",
					val:  "",
				},
			},
			expected: &Config{
				InCluster:     true,
				KubeConfig:    "",
				GoogleProject: "",
				GoogleZone:    "",
				HealthPort:    defaultHealthPort,
				DryRun:        true,
				Debug:         false,
				LogFormat:     defaultLogFormat,
			},
		},
		{
			title: "all default",
			args:  []arg{},
			expected: &Config{
				InCluster:     false,
				KubeConfig:    "",
				GoogleProject: "",
				GoogleZone:    "",
				HealthPort:    defaultHealthPort,
				DryRun:        true,
				Debug:         false,
				LogFormat:     defaultLogFormat,
			},
		},
		{
			title: "set string var",
			args: []arg{
				arg{
					flag: "--kubeconfig",
					val:  "myhome",
				},
			},
			expected: &Config{
				InCluster:     false,
				KubeConfig:    "myhome",
				GoogleProject: "",
				GoogleZone:    "",
				HealthPort:    defaultHealthPort,
				DryRun:        true,
				Debug:         false,
				LogFormat:     defaultLogFormat,
			},
		},
		{
			title: "unexpected flag",
			args: []arg{
				arg{
					flag: "--random",
					val:  "myhome",
				},
			},
			expectError: true,
		},
		{
			title: "override default",
			args: []arg{
				arg{
					flag: "--log-format",
					val:  "json",
				},
			},
			expected: &Config{
				InCluster:     false,
				KubeConfig:    "myhome",
				GoogleProject: "",
				GoogleZone:    "",
				HealthPort:    defaultHealthPort,
				DryRun:        true,
				Debug:         false,
				LogFormat:     "json",
			},
		},
	} {
		t.Run(ti.title, func(t *testing.T) {
			cfg := NewConfig()
			spaceArgs := []string{"external-dns"}
			for _, arg := range ti.args {
				spaceArgs = append(spaceArgs, arg.Slice()...)
			}
			err := cfg.ParseFlags(spaceArgs)
			if !ti.expectError && err != nil {
				t.Errorf("unexpected parse flags fail for args %#v, error: %v", ti.args, err)
			}
			if ti.expectError && err == nil {
				t.Errorf("parse flags should fail for args %#v", ti.args)
			}
			if !ti.expectError {
				validateConfig(t, cfg, ti.expected)
			}
		})
	}
}

// helper functions

func validateConfig(t *testing.T, got, expected *Config) {
	if !reflect.DeepEqual(got, expected) {
		t.Errorf("Config is wrong")
	}
}

type arg struct {
	flag string
	val  string
}

func (a *arg) Slice() []string {
	return []string{a.flag, a.val}
}

func (a *arg) String() string {
	return fmt.Sprintf("%s=%s", a.flag, a.val)
}
<|endoftext|>


<commit_before>package sqlstore

import (
	"errors"

	_ "github.com/go-sql-driver/mysql" // mysql driver
	"github.com/gobuffalo/packr"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"           // postgresql driver
	_ "github.com/mattn/go-sqlite3" //sqlite3 driver
	migrate "github.com/rubenv/sql-migrate"
	log "github.com/sirupsen/logrus"
)

type sqlstore struct {
	*sqlx.DB

	Config *Config
}

// ErrNoResults is returned when a query yielded 0 results
var ErrNoResults = errors.New("datastore: query returned 0 results")

// New creates a new database pool
func New(c *Config) *sqlstore {
	dbx := sqlx.MustConnect(c.Driver, c.DSN())
	db := &sqlstore{dbx, c}

	// write log statement
	log.Infof("Connected to %s database: %s", c.Driver, c.Name)

	// run migrations
	db.Migrate()

	return db
}

func (db *sqlstore) Migrate() {
	migrationSource := &migrate.PackrMigrationSource{
		Box: packr.NewBox("./migrations"),
		Dir: db.Config.Driver,
	}
	migrate.SetTable("migrations")

	migrations, err := migrationSource.FindMigrations()
	if err != nil {
		log.Errorf("Error loading database migrations: %s", err)
	}

	if len(migrations) == 0 {
		log.Fatalf("Missing database migrations")
	}

	n, err := migrate.Exec(db.DB.DB, db.Config.Driver, migrationSource, migrate.Up)
	if err != nil {
		log.Errorf("Error applying database migrations: %s", err)
	}

	if n > 0 {
		log.Infof("Applied %d database migrations!", n)
	}
}

// Closes the db pool
func (db *sqlstore) Close() {
	db.DB.Close()
}
<commit_msg>set pipe concat mode when using mysql<commit_after>package sqlstore

import (
	"errors"

	_ "github.com/go-sql-driver/mysql" // mysql driver
	"github.com/gobuffalo/packr"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"           // postgresql driver
	_ "github.com/mattn/go-sqlite3" //sqlite3 driver
	migrate "github.com/rubenv/sql-migrate"
	log "github.com/sirupsen/logrus"
)

type sqlstore struct {
	*sqlx.DB

	Config *Config
}

// ErrNoResults is returned when a query yielded 0 results
var ErrNoResults = errors.New("datastore: query returned 0 results")

// New creates a new database pool
func New(c *Config) *sqlstore {
	dbx := sqlx.MustConnect(c.Driver, c.DSN())
	db := &sqlstore{dbx, c}

	// write log statement
	log.Infof("Connected to %s database: %s", c.Driver, c.Name)

	// Driver specific database options
	if c.Driver == "mysql" {
		// Because SQLite doesn't have CONCAT, tell MySQL to accept pipes for concatenating string columns
		db.Exec("SET sql_mode=PIPES_AS_CONCAT;")
	}

	// run migrations
	db.Migrate()

	return db
}

func (db *sqlstore) Migrate() {
	migrationSource := &migrate.PackrMigrationSource{
		Box: packr.NewBox("./migrations"),
		Dir: db.Config.Driver,
	}
	migrate.SetTable("migrations")

	migrations, err := migrationSource.FindMigrations()
	if err != nil {
		log.Errorf("Error loading database migrations: %s", err)
	}

	if len(migrations) == 0 {
		log.Fatalf("Missing database migrations")
	}

	n, err := migrate.Exec(db.DB.DB, db.Config.Driver, migrationSource, migrate.Up)
	if err != nil {
		log.Errorf("Error applying database migrations: %s", err)
	}

	if n > 0 {
		log.Infof("Applied %d database migrations!", n)
	}
}

// Closes the db pool
func (db *sqlstore) Close() {
	db.DB.Close()
}
<|endoftext|>


<commit_before>package buildpackrunner

import (
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"github.com/pivotal-golang/archiver/extractor"
	"github.com/pivotal-golang/cacheddownloader"
)

type ZipDownloader struct {
	downloader *cacheddownloader.Downloader
}

func IsZipFile(filename string) bool {
	return strings.HasSuffix(filename, ".zip")
}

func NewZipDownloader(skipSSLVerification bool) *ZipDownloader {
	return &ZipDownloader{
		downloader: cacheddownloader.NewDownloader(DOWNLOAD_TIMEOUT, 1, skipSSLVerification),
	}
}

func (z *ZipDownloader) DownloadAndExtract(u *url.URL, destination string) (uint64, error) {
	zipFile, err := ioutil.TempFile("", filepath.Base(u.Path))
	if err != nil {
		return 0, fmt.Errorf("Could not create zip file: %s", err.Error())
	}
	defer os.Remove(zipFile.Name())

	_, _, err = z.downloader.Download(u,
		func() (*os.File, error) {
			return os.OpenFile(zipFile.Name(), os.O_WRONLY, 0666)
		},
		cacheddownloader.CachingInfoType{},
	)
	if err != nil {
		return 0, fmt.Errorf("Failed to download buildpack '%s': %s", u.String(), err.Error())
	}

	fi, err := zipFile.Stat()
	if err != nil {
		return 0, fmt.Errorf("Failed to obtain the size of the buildpack '%s': %s", u.String(), err.Error())
	}

	err = extractor.NewZip().Extract(zipFile.Name(), destination)
	if err != nil {
		return 0, fmt.Errorf("Failed to extract buildpack '%s': %s", u.String(), err.Error())
	}

	return uint64(fi.Size()), nil
}
<commit_msg>Add cancel channel to Downloader.Download<commit_after>package buildpackrunner

import (
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"github.com/pivotal-golang/archiver/extractor"
	"github.com/pivotal-golang/cacheddownloader"
)

type ZipDownloader struct {
	downloader *cacheddownloader.Downloader
}

func IsZipFile(filename string) bool {
	return strings.HasSuffix(filename, ".zip")
}

func NewZipDownloader(skipSSLVerification bool) *ZipDownloader {
	return &ZipDownloader{
		downloader: cacheddownloader.NewDownloader(DOWNLOAD_TIMEOUT, 1, skipSSLVerification),
	}
}

func (z *ZipDownloader) DownloadAndExtract(u *url.URL, destination string) (uint64, error) {
	zipFile, err := ioutil.TempFile("", filepath.Base(u.Path))
	if err != nil {
		return 0, fmt.Errorf("Could not create zip file: %s", err.Error())
	}
	defer os.Remove(zipFile.Name())

	_, _, err = z.downloader.Download(u,
		func() (*os.File, error) {
			return os.OpenFile(zipFile.Name(), os.O_WRONLY, 0666)
		},
		cacheddownloader.CachingInfoType{},
		make(chan struct{}),
	)
	if err != nil {
		return 0, fmt.Errorf("Failed to download buildpack '%s': %s", u.String(), err.Error())
	}

	fi, err := zipFile.Stat()
	if err != nil {
		return 0, fmt.Errorf("Failed to obtain the size of the buildpack '%s': %s", u.String(), err.Error())
	}

	err = extractor.NewZip().Extract(zipFile.Name(), destination)
	if err != nil {
		return 0, fmt.Errorf("Failed to extract buildpack '%s': %s", u.String(), err.Error())
	}

	return uint64(fi.Size()), nil
}
<|endoftext|>


<commit_before>package main
<commit_msg>initial middleware implementation<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"io"
	"os/exec"
	"strings"
	"sync"

	log "github.com/Sirupsen/logrus"
)

type Middleware struct {
	command string
	payload Payload

	data chan []byte

	mu sync.Mutex

	Stdin  io.Writer
	Stdout io.Reader
}

// To provide input to the pipeline, assign an io.Reader to the first's Stdin.
func Pipeline(cmds ...*exec.Cmd) (pipeLineOutput, collectedStandardError []byte, pipeLineError error) {
	// Require at least one command
	if len(cmds) < 1 {
		return nil, nil, nil
	}

	// Collect the output from the command(s)
	var output bytes.Buffer
	var stderr bytes.Buffer

	last := len(cmds) - 1
	for i, cmd := range cmds[:last] {
		// Connect each command's stdin to the previous command's stdout
		var err error
		if cmds[i+1].Stdin, err = cmd.StdoutPipe(); err != nil {
			return nil, nil, err
		}
		// Connect each command's stderr to a buffer
		cmd.Stderr = &stderr
	}

	// Connect the output and error for the last command
	cmds[last].Stdout, cmds[last].Stderr = &output, &stderr

	// Start each command
	for _, cmd := range cmds {
		if err := cmd.Start(); err != nil {
			return output.Bytes(), stderr.Bytes(), err
		}
	}

	// Wait for each command to complete
	for _, cmd := range cmds {
		if err := cmd.Wait(); err != nil {
			return output.Bytes(), stderr.Bytes(), err
		}
	}

	// Return the pipeline output and the collected standard error
	return output.Bytes(), stderr.Bytes(), nil
}

func ExecuteMiddleware(command string, payload Payload) (Payload, error) {
	m := new(Middleware)
	m.command = command
	m.payload = payload
	m.data = make(chan []byte, 1000)

	commands := strings.Split(command, " ")
	cmds := exec.Command(commands[0], commands[1:]...)

	// getting payload
	bts, err := json.Marshal(m.payload)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to marshal json")
	}
	cmds.Stdin = bytes.NewReader(bts)

	// Run the pipeline
	mwOutput, stderr, err := Pipeline(cmds)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to process pipeline")
	}

	// log stderr
	if len(stderr) > 0 {

		log.WithFields(log.Fields{
			"sdtderr": string(stderr),
		}).Warn("errors from middleware")

	} else if len(mwOutput) > 0 {
		var newPayload Payload

		err = json.Unmarshal(mwOutput, &newPayload)

		if err != nil {
			log.WithFields(log.Fields{
				"mwOutput": string(mwOutput),
			}).Error("Failed to unmarshal JSON from middleware")
		} else {
			// payload unmarshalled into Payload struct, returning it
			return newPayload, nil
		}
	} else {

		log.WithFields(log.Fields{
			"mwOutput": string(mwOutput),
		}).Warn("No response from middleware.")
	}

	return payload, nil

}
<|endoftext|>


<commit_before>package main

import "testing"

func TestIsValidFlags(t *testing.T) {
	t.Parallel()

	testCases := []struct {
		d       string
		t       string
		isError bool
	}{
		{
			d:       "",
			t:       "",
			isError: true,
		},
		{
			d:       "bar",
			t:       "",
			isError: true,
		},
		{
			d:       "",
			t:       "foo",
			isError: true,
		},
		{
			d:       "bar",
			t:       "foo",
			isError: false,
		},
	}

	for i, c := range testCases {
		dataPath, tplPath = c.d, c.t
		if err := isValidFlags(); (err == nil) == c.isError {
			t.Fatalf("[%d] invalid error state: expected=%t, but got=%t",
				i+1, c.isError, (err == nil) == c.isError)
		}
	}
}

func TestRender(t *testing.T) {
	t.Parallel()

	testCases := []struct {
		inData string
		inTpl  string
		expect string
	}{
		{
			inData: `TEST: aaa . * ? %`,
			inTpl:  `test01 is {{ .TEST }}`,
			expect: `test01 is aaa . * ? %`,
		},
		{
			inData: `TEST: 'aaa . * ? %'
TEST_NEST: '{{ .TEST }} nest'`,
			inTpl:  `test02 is {{ .TEST_NEST }}`,
			expect: `test02 is aaa . * ? % nest`,
		},
		{
			inData: `TEST_NEST: '{{ .TEST}} nest'
TEST: aaa`,
			inTpl:  `test03 is {{ .TEST_NEST }}`,
			expect: `test03 is aaa nest`,
		},
		{
			inData: `TEST: [foo, bar, baz]`,
			inTpl:  `test04 is {{ join .TEST "," }}`,
			expect: `test04 is foo,bar,baz`,
		},
		{
			// 0... は 8 進数として解釈される。
			inData: `TEST: [foo, bar, baz, 1, '%.wer', 'hoge', '0123', 017]
TEST_NEST: '{{ join {{ .TEST }} "," }}'`,
			inTpl:  `test05 is {{ join .TEST "," }}`,
			expect: `test05 is foo,bar,baz,1,%.wer,hoge,0123,15`,
		},
		{
			inData: `TEST: echo -n test06`,
			inTpl:  `test06 is {{ exec .TEST }}`,
			expect: `test06 is test06`,
		},
		{
			inData: `TEST: echo -n 'test07 test07'`,
			inTpl:  `test07 is {{ exec .TEST }}`,
			expect: `test07 is test07 test07`,
		},
	}

	for i, c := range testCases {
		tpl, err := parseTemplate(c.inTpl)
		if err != nil {
			t.Fatal(err)
		}
		out, err := render([]byte(c.inData), tpl)
		if err != nil {
			t.Fatal(err)
		}
		if string(out) != c.expect {
			t.Fatalf("[%d] failed in templateing: expected=%s, but got=%s",
				i+1, c.expect, string(out))
		}
	}
}
<commit_msg>Update test cases<commit_after>package main

import "testing"

func TestIsValidFlags(t *testing.T) {
	t.Parallel()

	testCases := []struct {
		d       string
		t       string
		isError bool
	}{
		{
			d:       "",
			t:       "",
			isError: true,
		},
		{
			d:       "bar",
			t:       "",
			isError: true,
		},
		{
			d:       "",
			t:       "foo",
			isError: true,
		},
		{
			d:       "bar",
			t:       "foo",
			isError: false,
		},
	}

	for i, c := range testCases {
		dataPath, tplPath = c.d, c.t
		if err := isValidFlags(); (err == nil) == c.isError {
			t.Fatalf("[%d] invalid error state: expected=%t, but got=%t",
				i+1, c.isError, (err == nil) == c.isError)
		}
	}
}

func TestRender(t *testing.T) {
	t.Parallel()

	testCases := []struct {
		inData string
		inTpl  string
		expect string
	}{
		{
			inData: `TEST: aaa . * ? %`,
			inTpl:  `test01 is {{ .TEST }}`,
			expect: `test01 is aaa . * ? %`,
		},
		{
			inData: `TEST: 'aaa . * ? %'
TEST_NEST: '{{ .TEST }} nest'`,
			inTpl:  `test02 is {{ .TEST_NEST }}`,
			expect: `test02 is aaa . * ? % nest`,
		},
		{
			inData: `TEST_NEST: '{{ .TEST }} nest'
TEST: aaa`,
			inTpl:  `test03 is {{ .TEST_NEST }}`,
			expect: `test03 is aaa nest`,
		},
		{
			inData: `TEST: [foo, bar, baz]`,
			inTpl:  `test04 is {{ join .TEST "," }}`,
			expect: `test04 is foo,bar,baz`,
		},
		{
			// 0... は 8 進数として解釈される。
			inData: `TEST: [foo, bar, baz, 1, '%.wer', 'hoge', '0123', 017]
TEST_NEST: '{{ join {{ .TEST }} "," }}'`,
			inTpl:  `test05 is {{ join .TEST "," }}`,
			expect: `test05 is foo,bar,baz,1,%.wer,hoge,0123,15`,
		},
		{
			inData: `TEST: echo -n test06`,
			inTpl:  `test06 is {{ exec .TEST }}`,
			expect: `test06 is test06`,
		},
		{
			inData: `TEST: echo -n 'test07 test07'`,
			inTpl:  `test07 is {{ exec .TEST }}`,
			expect: `test07 is test07 test07`,
		},
		{
			inData: `TEST: ""`,
			inTpl:  `test08 is {{ exec "echo -n 'test08 test08'" }}`,
			expect: `test08 is test08 test08`,
		},
		{
			inData: `TEST: echo -n 'test09'
TEST_NEST: test09 {{ exec .TEST }}`,
			inTpl:  `test09 is {{ .TEST_NEST }} nest`,
			expect: `test09 is test09 test09 nest`,
		},
	}

	for i, c := range testCases {
		tpl, err := parseTemplate(c.inTpl)
		if err != nil {
			t.Fatal(err)
		}
		out, err := render([]byte(c.inData), tpl)
		if err != nil {
			t.Fatal(err)
		}
		if string(out) != c.expect {
			t.Fatalf("[%d] failed in templateing: expected=%s, but got=%s",
				i+1, c.expect, string(out))
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"os/user"
	"strconv"
	"strings"
	"time"
)

var cfg *Config

type RssFeed struct {
	XMLName        xml.Name `xml:"channel"`
	Title          string   `xml:"title"`       // required
	Link           string   `xml:"link"`        // required
	Description    string   `xml:"description"` // required
	Language       string   `xml:"language,omitempty"`
	Copyright      string   `xml:"copyright,omitempty"`
	ManagingEditor string   `xml:"managingEditor,omitempty"` // Author used
	WebMaster      string   `xml:"webMaster,omitempty"`
	PubDate        string   `xml:"pubDate,omitempty"`       // created or updated
	LastBuildDate  string   `xml:"lastBuildDate,omitempty"` // updated used
	Category       string   `xml:"category,omitempty"`
	Generator      string   `xml:"generator,omitempty"`
	Docs           string   `xml:"docs,omitempty"`
	Cloud          string   `xml:"cloud,omitempty"`
	Ttl            int      `xml:"ttl,omitempty"`
	Rating         string   `xml:"rating,omitempty"`
	SkipHours      string   `xml:"skipHours,omitempty"`
	SkipDays       string   `xml:"skipDays,omitempty"`
	Items          []*RssItem
}

type RssItem struct {
	XMLName     xml.Name `xml:"item"`
	Title       string   `xml:"title"`       // required
	Link        string   `xml:"link"`        // required
	Description string   `xml:"description"` // required
	Author      string   `xml:"author,omitempty"`
	Category    string   `xml:"category,omitempty"`
	Comments    string   `xml:"comments,omitempty"`
	Enclosure   *RssEnclosure
	Guid        string `xml:"guid,omitempty"`    // Id used
	PubDate     string `xml:"pubDate,omitempty"` // created or updated
	Source      string `xml:"source,omitempty"`
}

type RssEnclosure struct {
	XMLName xml.Name `xml:"enclosure"`
	Url     string   `xml:"url,attr"`
	Length  string   `xml:"length,attr"`
	Type    string   `xml:"type,attr"`
}

type Config struct {
	FeedURL   string `json:"feed_url"`
	FeedTitle string `json:"feed_title"`
	FeedDesc  string `json:"feed_description"`
	FeedOwner string `json:"feed_owner"`
}

func ReadConfig() (*Config, error) {
	homeDir := ""
	usr, err := user.Current()
	if err == nil {
		homeDir = usr.HomeDir
	}
	conf := Config{}
	for _, path := range []string{"/etc/podfy.conf", homeDir + "/.podfy.conf", "./podfy.conf"} {
		file, err := os.Open(path)
		if os.IsNotExist(err) {
			continue
		}
		if err != nil {
			return nil, err
		}

		json.NewDecoder(file)
		err = json.NewDecoder(file).Decode(&conf)
		if err != nil {
			return nil, err
		}
		return &conf, nil
	}
	return &conf, nil
}

func ToXML(feed *RssFeed) (string, error) {
	data, err := xml.MarshalIndent(feed, "", "  ")
	if err != nil {
		return "", err
	}

	// strip empty line from default xml header
	s := xml.Header[:len(xml.Header)-1] + string(data)
	return s, nil
}

func formatTime(t time.Time) string {
	if !t.IsZero() {
		return t.Format(time.RFC822)
	}
	return ""
}

func init() {
	var err error
	cfg, err = ReadConfig()
	if err != nil {
		log.Fatalf("Could not read config: %v", err)
	}
}

func main() {
	//TODO index.html

	http.HandleFunc("/feed.xml", feedHandler)
	http.HandleFunc("/add", addHandler)
	http.Handle("/files/", http.StripPrefix("/files/", http.FileServer(http.Dir("./files"))))

	http.ListenAndServe(":8080", nil)
}

func addHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()

	if len(r.Form["auth"]) != 1 {
		fmt.Fprintln(w, "ERR: Auth token missing")
		return
	}
	if r.Form["auth"][0] != "mak" {
		fmt.Fprintln(w, "ERR: Auth token invaild")
		return
	}

	url, err := url.Parse(r.Form["url"][0])

	if err != nil {
		log.Fatal(err)
	}
	go executeYoutubeDL(url)
	fmt.Fprintln(w)
}

func executeYoutubeDL(url *url.URL) {
	cmd := exec.Command("youtube-dl", url.String(),
		"--write-info-json", "--write-thumbnail",
		"--write-description", "--restrict-filenames")

	cmd.Dir = "./files"
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err := cmd.Start()
	if err != nil {
		log.Fatal(err)
	}
	err = cmd.Wait()
	log.Printf("Command finished with: %v", err)
}

func feedHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, createFeed())
}

func createFeed() string {
	feed := &RssFeed{
		Title:          cfg.FeedTitle,
		Link:           cfg.FeedURL,
		Description:    cfg.FeedDesc,
		ManagingEditor: cfg.FeedOwner,
		// PubDate:     time.Now(),
	}

	feed.Items = []*RssItem{}

	files, _ := ioutil.ReadDir("./files")

	for _, f := range files {
		isMp4 := strings.HasSuffix(f.Name(), ".mp4")
		if !isMp4 {
			continue
		}

		feed.Items = append(feed.Items, &RssItem{
			Title:       strings.TrimSuffix(f.Name(), ".mp4"),
			Link:        createLink(f.Name()),
			Description: "",
			Enclosure: &RssEnclosure{Url: createLink(f.Name()),
				Length: strconv.Itoa(int(f.Size())),
				Type:   "video/mp4"},
			PubDate: formatTime(f.ModTime()),
		})
	}

	s, _ := ToXML(feed)

	return s
}

func createLink(filename string) string {
	var Url *url.URL
	Url, err := url.Parse(cfg.FeedURL + "/files/")
	if err != nil {
		panic("boom")
	}

	Url.Path += filename

	fmt.Printf("Encoded URL is %q\n", Url.String())
	return Url.String()
}
<commit_msg>Wraped the feed with rss version 2<commit_after>package main

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"os/user"
	"strconv"
	"strings"
	"time"
)

var cfg *Config

type Rss struct {
	XMLName xml.Name `xml:"rss"`
	Version string   `xml:"version,attr"` // required
	RssFeed *RssFeed
}

type RssFeed struct {
	XMLName        xml.Name `xml:"channel"`
	Title          string   `xml:"title"`       // required
	Link           string   `xml:"link"`        // required
	Description    string   `xml:"description"` // required
	Language       string   `xml:"language,omitempty"`
	Copyright      string   `xml:"copyright,omitempty"`
	ManagingEditor string   `xml:"managingEditor,omitempty"` // Author used
	WebMaster      string   `xml:"webMaster,omitempty"`
	PubDate        string   `xml:"pubDate,omitempty"`       // created or updated
	LastBuildDate  string   `xml:"lastBuildDate,omitempty"` // updated used
	Category       string   `xml:"category,omitempty"`
	Generator      string   `xml:"generator,omitempty"`
	Docs           string   `xml:"docs,omitempty"`
	Cloud          string   `xml:"cloud,omitempty"`
	Ttl            int      `xml:"ttl,omitempty"`
	Rating         string   `xml:"rating,omitempty"`
	SkipHours      string   `xml:"skipHours,omitempty"`
	SkipDays       string   `xml:"skipDays,omitempty"`
	Items          []*RssItem
}

type RssItem struct {
	XMLName     xml.Name `xml:"item"`
	Title       string   `xml:"title"`       // required
	Link        string   `xml:"link"`        // required
	Description string   `xml:"description"` // required
	Author      string   `xml:"author,omitempty"`
	Category    string   `xml:"category,omitempty"`
	Comments    string   `xml:"comments,omitempty"`
	Enclosure   *RssEnclosure
	Image       *Thumbnail
	Guid        string `xml:"guid,omitempty"`    // Id used
	PubDate     string `xml:"pubDate,omitempty"` // created or updated
	Source      string `xml:"source,omitempty"`
}

type RssEnclosure struct {
	XMLName xml.Name `xml:"enclosure"`
	Url     string   `xml:"url,attr"`
	Length  string   `xml:"length,attr"`
	Type    string   `xml:"type,attr"`
}

type Thumbnail struct {
	XMLName xml.Name `xml:"image"`
	Url     string   `xml:"url"`
}

type Config struct {
	FeedURL   string `json:"feed_url"`
	FeedTitle string `json:"feed_title"`
	FeedDesc  string `json:"feed_description"`
	FeedOwner string `json:"feed_owner"`
}

func ReadConfig() (*Config, error) {
	homeDir := ""
	usr, err := user.Current()
	if err == nil {
		homeDir = usr.HomeDir
	}
	conf := Config{}
	for _, path := range []string{"/etc/podfy.conf", homeDir + "/.podfy.conf", "./podfy.conf"} {
		file, err := os.Open(path)
		if os.IsNotExist(err) {
			continue
		}
		if err != nil {
			return nil, err
		}

		json.NewDecoder(file)
		err = json.NewDecoder(file).Decode(&conf)
		if err != nil {
			return nil, err
		}
		return &conf, nil
	}
	return &conf, nil
}

func ToXML(feed *Rss) (string, error) {
	data, err := xml.MarshalIndent(feed, "", "  ")
	if err != nil {
		return "", err
	}

	// strip empty line from default xml header
	s := xml.Header[:len(xml.Header)-1] + string(data)
	return s, nil
}

func formatTime(t time.Time) string {
	if !t.IsZero() {
		return t.Format(time.RFC822)
	}
	return ""
}

func init() {
	var err error
	cfg, err = ReadConfig()
	if err != nil {
		log.Fatalf("Could not read config: %v", err)
	}
}

func main() {
	//TODO index.html

	http.HandleFunc("/feed.xml", feedHandler)
	http.HandleFunc("/add", addHandler)
	http.Handle("/files/", http.StripPrefix("/files/", http.FileServer(http.Dir("./files"))))

	http.ListenAndServe(":8080", nil)
}

func addHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()

	if len(r.Form["auth"]) != 1 {
		fmt.Fprintln(w, "ERR: Auth token missing")
		return
	}
	if r.Form["auth"][0] != "mak" {
		fmt.Fprintln(w, "ERR: Auth token invaild")
		return
	}

	url, err := url.Parse(r.Form["url"][0])

	if err != nil {
		log.Fatal(err)
	}
	go executeYoutubeDL(url)
	fmt.Fprintln(w)
}

func executeYoutubeDL(url *url.URL) {
	cmd := exec.Command("youtube-dl", url.String(),
		"--write-info-json", "--write-thumbnail",
		"--write-description", "--restrict-filenames")

	cmd.Dir = "./files"
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err := cmd.Start()
	if err != nil {
		log.Fatal(err)
	}
	err = cmd.Wait()
	log.Printf("Command finished with: %v", err)
}

func feedHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, createFeed())
}

func createFeed() string {
	feed := &RssFeed{
		Title:          cfg.FeedTitle,
		Link:           cfg.FeedURL,
		Description:    cfg.FeedDesc,
		ManagingEditor: cfg.FeedOwner,
		// PubDate:     time.Now(),
	}

	feed.Items = []*RssItem{}

	files, _ := ioutil.ReadDir("./files")

	for _, f := range files {
		isMp4 := strings.HasSuffix(f.Name(), ".mp4")
		if !isMp4 {
			continue
		}

		description := ""
		descBuf, err := ioutil.ReadFile("./files/" + f.Name() + ".description")
		fmt.Println(descBuf, err)
		if err == nil {
			description = string(descBuf)
		}

		feed.Items = append(feed.Items, &RssItem{
			Title:       strings.TrimSuffix(f.Name(), ".mp4"),
			Link:        createLink(f.Name()),
			Description: description,
			Enclosure: &RssEnclosure{Url: createLink(f.Name()),
				Length: strconv.Itoa(int(f.Size())),
				Type:   "video/mp4"},
			Image:   &Thumbnail{Url: createLink(strings.TrimSuffix(f.Name(), ".mp4") + ".jpg")},
			PubDate: formatTime(f.ModTime()),
		})
	}

	rss20 := &Rss{RssFeed: feed, Version: "2.0"}
	s, _ := ToXML(rss20)

	return s
}

func createLink(filename string) string {
	var Url *url.URL
	Url, err := url.Parse(cfg.FeedURL + "/files/")
	if err != nil {
		panic("boom")
	}

	Url.Path += filename

	fmt.Printf("Encoded URL is %q\n", Url.String())
	return Url.String()
}
<|endoftext|>


<commit_before>package geom

import "fmt"

// The Point type represents 2D coordinates.
type Point struct {
	X float64
	Y float64
}

// The Zero method checks if the receiver has the zero-value (both the x and y
// components are zero).
func (p Point) Zero() bool {
	return p.X == 0 && p.Y == 0
}

// The String method returns a human-readable representation of the point value.
func (p Point) String() string {
	return fmt.Sprintf("(%.6g, %.6g)", p.X, p.Y)
}

// WithOrigin translate the receiver to a coordinate system with origin given as
// argument and returns the modified point.
func (p Point) WithOrigin(origin Point) Point {
	return Point{
		X: p.X - origin.X,
		Y: p.Y - origin.Y,
	}
}
<commit_msg>update point documentation<commit_after>package geom

import "fmt"

// The Point type represents 2D coordinates.
type Point struct {
	X float64
	Y float64
}

// Zero method if the receiver has the zero-value (both the x and y components
// are zero).
func (p Point) Zero() bool {
	return p.X == 0 && p.Y == 0
}

// The String method returns a human-readable representation of the point value.
func (p Point) String() string {
	return fmt.Sprintf("(%.6g, %.6g)", p.X, p.Y)
}

// WithOrigin translate the receiver to a coordinate system with origin given as
// argument and returns the modified point.
func (p Point) WithOrigin(origin Point) Point {
	return Point{
		X: p.X - origin.X,
		Y: p.Y - origin.Y,
	}
}
<|endoftext|>


<commit_before>/*
 * Network packet analysis framework.
 *
 * Copyright (c) 2014, Alessandro Ghedini
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

// Provides encoding and decoding for ARP packets.
package arp

import "net"

import "github.com/ghedo/hype/packet"

type Packet struct {
	Operation     Operation        `name:"op"`

	HWType        uint16
	HWAddrLen     uint8
	HWSrcAddr     net.HardwareAddr `name:"hwsrc"`
	HWDstAddr     net.HardwareAddr `name:"hwdst"`

	ProtoType     packet.Type
	ProtoAddrLen  uint8
	ProtoSrcAddr  net.IP           `name:"psrc"`
	ProtoDstAddr  net.IP           `name:"pdst"`
}

type Operation uint16

const (
	Request Operation = 1
	Reply             = 2
)

func Make() *Packet {
	return &Packet {
		Operation: Request,

		HWType: 1,
		HWAddrLen: 6,

		ProtoType: packet.IPv4,
		ProtoAddrLen: 4,
	}
}

func (p *Packet) GetType() packet.Type {
	return packet.ARP
}

func (p *Packet) GetLength() uint16 {
	return 8 + uint16(p.HWAddrLen) * 2 + uint16(p.ProtoAddrLen) * 2
}

func (p *Packet) Pack(raw_pkt *packet.Buffer) error {
	raw_pkt.WriteI(p.HWType)
	raw_pkt.WriteI(p.ProtoType.ToEtherType())

	raw_pkt.WriteI(p.HWAddrLen)
	raw_pkt.WriteI(p.ProtoAddrLen)

	raw_pkt.WriteI(p.Operation)

	raw_pkt.Write(p.HWSrcAddr)
	raw_pkt.Write(p.ProtoSrcAddr.To4())

	raw_pkt.Write(p.HWDstAddr)
	raw_pkt.Write(p.ProtoDstAddr.To4())

	return nil
}

func (p *Packet) Unpack(raw_pkt *packet.Buffer) error {
	raw_pkt.ReadI(&p.HWType)

	var ethertype uint16
	raw_pkt.ReadI(&ethertype)
	p.ProtoType = packet.EtherType(ethertype)

	raw_pkt.ReadI(&p.HWAddrLen)
	raw_pkt.ReadI(&p.ProtoAddrLen)

	raw_pkt.ReadI(&p.Operation)

	p.HWSrcAddr = net.HardwareAddr(raw_pkt.Next(int(p.HWAddrLen)))
	p.ProtoSrcAddr = net.IP(raw_pkt.Next(int(p.ProtoAddrLen)))

	p.HWDstAddr = net.HardwareAddr(raw_pkt.Next(int(p.HWAddrLen)))
	p.ProtoDstAddr = net.IP(raw_pkt.Next(int(p.ProtoAddrLen)))

	return nil
}

func (p *Packet) Payload() packet.Packet {
	return nil
}

func (p *Packet) PayloadType() packet.Type {
	return packet.None
}

func (p *Packet) SetPayload(pl packet.Packet) error {
	return nil
}

func (p *Packet) InitChecksum(csum uint32) {
}

func (p *Packet) String() string {
	return packet.Stringify(p)
}

func (o Operation) String() string {
	switch o {
	case Request:
		return "request"

	case Reply:
		return "reply"

	default:
		return "invalid"
	}
}
<commit_msg>arp: write properly sized addresses<commit_after>/*
 * Network packet analysis framework.
 *
 * Copyright (c) 2014, Alessandro Ghedini
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

// Provides encoding and decoding for ARP packets.
package arp

import "net"

import "github.com/ghedo/hype/packet"

type Packet struct {
	Operation     Operation        `name:"op"`

	HWType        uint16
	HWAddrLen     uint8
	HWSrcAddr     net.HardwareAddr `name:"hwsrc"`
	HWDstAddr     net.HardwareAddr `name:"hwdst"`

	ProtoType     packet.Type
	ProtoAddrLen  uint8
	ProtoSrcAddr  net.IP           `name:"psrc"`
	ProtoDstAddr  net.IP           `name:"pdst"`
}

type Operation uint16

const (
	Request Operation = 1
	Reply             = 2
)

func Make() *Packet {
	return &Packet {
		Operation: Request,

		HWType: 1,
		HWAddrLen: 6,

		ProtoType: packet.IPv4,
		ProtoAddrLen: 4,
	}
}

func (p *Packet) GetType() packet.Type {
	return packet.ARP
}

func (p *Packet) GetLength() uint16 {
	return 8 + uint16(p.HWAddrLen) * 2 + uint16(p.ProtoAddrLen) * 2
}

func (p *Packet) Pack(raw_pkt *packet.Buffer) error {
	raw_pkt.WriteI(p.HWType)
	raw_pkt.WriteI(p.ProtoType.ToEtherType())

	raw_pkt.WriteI(p.HWAddrLen)
	raw_pkt.WriteI(p.ProtoAddrLen)

	raw_pkt.WriteI(p.Operation)

	raw_pkt.Write(p.HWSrcAddr[len(p.HWSrcAddr) - int(p.HWAddrLen):])
	raw_pkt.Write(p.ProtoSrcAddr[len(p.ProtoSrcAddr) - int(p.ProtoAddrLen):])

	raw_pkt.Write(p.HWDstAddr[len(p.HWDstAddr) - int(p.HWAddrLen):])
	raw_pkt.Write(p.ProtoDstAddr[len(p.ProtoDstAddr) - int(p.ProtoAddrLen):])

	return nil
}

func (p *Packet) Unpack(raw_pkt *packet.Buffer) error {
	raw_pkt.ReadI(&p.HWType)

	var ethertype uint16
	raw_pkt.ReadI(&ethertype)
	p.ProtoType = packet.EtherType(ethertype)

	raw_pkt.ReadI(&p.HWAddrLen)
	raw_pkt.ReadI(&p.ProtoAddrLen)

	raw_pkt.ReadI(&p.Operation)

	p.HWSrcAddr = net.HardwareAddr(raw_pkt.Next(int(p.HWAddrLen)))
	p.ProtoSrcAddr = net.IP(raw_pkt.Next(int(p.ProtoAddrLen)))

	p.HWDstAddr = net.HardwareAddr(raw_pkt.Next(int(p.HWAddrLen)))
	p.ProtoDstAddr = net.IP(raw_pkt.Next(int(p.ProtoAddrLen)))

	return nil
}

func (p *Packet) Payload() packet.Packet {
	return nil
}

func (p *Packet) PayloadType() packet.Type {
	return packet.None
}

func (p *Packet) SetPayload(pl packet.Packet) error {
	return nil
}

func (p *Packet) InitChecksum(csum uint32) {
}

func (p *Packet) String() string {
	return packet.Stringify(p)
}

func (o Operation) String() string {
	switch o {
	case Request:
		return "request"

	case Reply:
		return "reply"

	default:
		return "invalid"
	}
}
<|endoftext|>


<commit_before>package io

import (
	"encoding/binary"
	"io"
	"reflect"

	"unsafe"
)

// Not threadsafe, obviously.
type VectorBuffer interface {
	// Positions in Mpc/h and masses in Msun/h.
	Read(fname string) (xs [][3]float32, ms []float32, ids []int64, err error)
	Close()
	IsOpen() bool
	ReadHeader(fname string, out *Header) error
	// The minimum mass of all the particles in the simulation.
	MinMass() float32
}

// CosmologyHeader contains information describing the cosmological
// context in which the simulation was run.
type CosmologyHeader struct {
	Z      float64
	OmegaM float64
	OmegaL float64
	H100   float64
}

type Header struct {
	Cosmo         CosmologyHeader
	N             int64
	TotalWidth    float64
	Origin, Width [3]float32
}

func readVecAsByte(rd io.Reader, end binary.ByteOrder, buf [][3]float32) error {
	bufLen := len(buf)

	hd := *(*reflect.SliceHeader)(unsafe.Pointer(&buf))
	hd.Len *= 12
	hd.Cap *= 12

	byteBuf := *(*[]byte)(unsafe.Pointer(&hd))
	_, err := rd.Read(byteBuf)
	if err != nil {
		return err
	}

	if !IsSysOrder(end) {
		for i := 0; i < bufLen*3; i++ {
			for j := 0; j < 2; j++ {
				idx1, idx2 := i*4+j, i*4+3-j
				byteBuf[idx1], byteBuf[idx2] = byteBuf[idx2], byteBuf[idx1]
			}
		}
	}

	hd.Len /= 12
	hd.Cap /= 12

	return nil
}

func readInt64AsByte(rd io.Reader, end binary.ByteOrder, buf []int64) error {
	bufLen := len(buf)

	hd := *(*reflect.SliceHeader)(unsafe.Pointer(&buf))
	hd.Len *= 8
	hd.Cap *= 8

	byteBuf := *(*[]byte)(unsafe.Pointer(&hd))
	_, err := rd.Read(byteBuf)
	if err != nil {
		return err
	}

	if !IsSysOrder(end) {
		for i := 0; i < bufLen; i++ {
			for j := 0; j < 4; j++ {
				idx1, idx2 := i*8+j, i*8+7-j
				byteBuf[idx1], byteBuf[idx2] = byteBuf[idx2], byteBuf[idx1]
			}
		}
	}

	hd.Len /= 8
	hd.Cap /= 8

	return nil
}

func IsSysOrder(end binary.ByteOrder) bool {
	buf32 := []int32{1}

	hd := *(*reflect.SliceHeader)(unsafe.Pointer(&buf32))
	hd.Len *= 4
	hd.Cap *= 4

	buf8 := *(*[]int8)(unsafe.Pointer(&hd))
	if buf8[0] == 1 {
		return binary.LittleEndian == end
	}
	return binary.BigEndian == end
}

func boundingBox(
	xs [][3]float32, totalWidth float64,
) (origin, width [3]float32) {
	// Assumes that the slice has already been checked for corruption.
	origin = xs[0]
	width = [3]float32{0, 0, 0}
	tw, tw2 := float32(totalWidth), float32(totalWidth)/2

	max, min := origin, origin

	for i := range xs {
		for j := 0; j < 3; j++ {
			x, x0, w := xs[i][j], origin[j], width[j]

			if x-x0 > tw2 {
				x -= tw
			} else if x0-x > tw2 {
				x += tw
			}

			if x < x0 {
				width[j] += x0 - x
				origin[j] = x
				min[j] = x
			} else if x-x0 > w {
				width[j] = x - x0
				max[j] = x
			}
		}
	}

	return origin, width
}
<commit_msg>Added explanation of how to add new I/O backends.<commit_after>/*The io package contains code for reading particles from disk. It provides
an abstract interface (VectorBuffer) that allows the main package to read
particles from different catalog formats in the same way.

Chances are that if you're reading this, you want to add a new type of catalog
type to this project. (If you're trying to do something more complicated than
that, chances are that you know what you're doing and aren't bothering with this
package comment). It's not too painful to do that in most cases. I'll lay out
exactly what this is going to look like. For the sake of the discussion I'm
going pretend that you're adding the a file-type called "my_file" to Shellfish.

0. Read up on the usual instructions for editing Shellfish code. You can find
these in the main documentation directory.

1. Make a file in this directory called my_file.go.

2. Make a struct in that file named "MyFileBuffer". Write five methods for that
struct that have the same names and type signatures as those found in the
VectorBuffer interface (the first declaration in this file).

3. Write a function "NewMyFileBuffer(...) *MyFileBuffer" that returns a new
instance of your buffer that is initialized with all the information that it
needs (or any work that you don't want to be repeated). Usually this will just
mean taking in an argument representing the byte ordering that the user
requested and, storing that in the struct and maybe allocating a couple internal
buffers. Sometimes you might also want the name of the directory containing
the files.

4. Implement each of those methods so that they do what the VectorBuffer
comments indicate that they should do. Only two are non-trivial: Read() and
(to a lesser extent) ReadHeader(). Look at the example in lgadget2.go and copy
code as needed. If your file format is just  couple arrays of particles with
some type of header (which it probably is), you can copy almost all of it and
won't have to do much.

5. Update getVectorBuffer() in cmd/util.go. It'll just be adding a case to a
switch statement.

6. Update the config file so that it knows about your file type. This means
going to cmd/cmd.go and doing two things: First, change the validate() method
there so that it doesn't crash when you pass it the name of your file type (i.e.
adding it to the first line of the config.SnapshotLine switch statement in
cmd.go). Second, go into the example config file (the big string in the same
file) and in the SnapshotType comment, explain that your file type is also
supported now.
*/
package io

import (
	"encoding/binary"
	"io"
	"reflect"

	"unsafe"
)

// Not threadsafe, obviously.
type VectorBuffer interface {
	// Positions in Mpc/h and masses in Msun/h.
	Read(fname string) (xs [][3]float32, ms []float32, ids []int64, err error)
	Close()
	IsOpen() bool
	ReadHeader(fname string, out *Header) error
	// The minimum mass of all the particles in the simulation.
	MinMass() float32
}

// CosmologyHeader contains information describing the cosmological
// context in which the simulation was run.
type CosmologyHeader struct {
	Z      float64
	OmegaM float64
	OmegaL float64
	H100   float64
}

type Header struct {
	Cosmo         CosmologyHeader
	N             int64
	TotalWidth    float64
	Origin, Width [3]float32
}

func readVecAsByte(rd io.Reader, end binary.ByteOrder, buf [][3]float32) error {
	bufLen := len(buf)

	hd := *(*reflect.SliceHeader)(unsafe.Pointer(&buf))
	hd.Len *= 12
	hd.Cap *= 12

	byteBuf := *(*[]byte)(unsafe.Pointer(&hd))
	_, err := rd.Read(byteBuf)
	if err != nil {
		return err
	}

	if !IsSysOrder(end) {
		for i := 0; i < bufLen*3; i++ {
			for j := 0; j < 2; j++ {
				idx1, idx2 := i*4+j, i*4+3-j
				byteBuf[idx1], byteBuf[idx2] = byteBuf[idx2], byteBuf[idx1]
			}
		}
	}

	hd.Len /= 12
	hd.Cap /= 12

	return nil
}

func readInt64AsByte(rd io.Reader, end binary.ByteOrder, buf []int64) error {
	bufLen := len(buf)

	hd := *(*reflect.SliceHeader)(unsafe.Pointer(&buf))
	hd.Len *= 8
	hd.Cap *= 8

	byteBuf := *(*[]byte)(unsafe.Pointer(&hd))
	_, err := rd.Read(byteBuf)
	if err != nil {
		return err
	}

	if !IsSysOrder(end) {
		for i := 0; i < bufLen; i++ {
			for j := 0; j < 4; j++ {
				idx1, idx2 := i*8+j, i*8+7-j
				byteBuf[idx1], byteBuf[idx2] = byteBuf[idx2], byteBuf[idx1]
			}
		}
	}

	hd.Len /= 8
	hd.Cap /= 8

	return nil
}

func IsSysOrder(end binary.ByteOrder) bool {
	buf32 := []int32{1}

	hd := *(*reflect.SliceHeader)(unsafe.Pointer(&buf32))
	hd.Len *= 4
	hd.Cap *= 4

	buf8 := *(*[]int8)(unsafe.Pointer(&hd))
	if buf8[0] == 1 {
		return binary.LittleEndian == end
	}
	return binary.BigEndian == end
}

func boundingBox(
	xs [][3]float32, totalWidth float64,
) (origin, width [3]float32) {
	// Assumes that the slice has already been checked for corruption.
	origin = xs[0]
	width = [3]float32{0, 0, 0}
	tw, tw2 := float32(totalWidth), float32(totalWidth)/2

	max, min := origin, origin

	for i := range xs {
		for j := 0; j < 3; j++ {
			x, x0, w := xs[i][j], origin[j], width[j]

			if x-x0 > tw2 {
				x -= tw
			} else if x0-x > tw2 {
				x += tw
			}

			if x < x0 {
				width[j] += x0 - x
				origin[j] = x
				min[j] = x
			} else if x-x0 > w {
				width[j] = x - x0
				max[j] = x
			}
		}
	}

	return origin, width
}
<|endoftext|>


<commit_before>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package sh

import (
	"fmt"
	"io"
)

func Fprint(w io.Writer, v interface{}) error {
	p := printer{w: w}
	p.node(v)
	return p.err
}

type printer struct {
	w   io.Writer
	err error

	needNewline bool
}

func (p *printer) pr(a ...interface{}) {
	for _, v := range a {
		if p.err != nil {
			break
		}
		switch v.(type) {
		case string, Token:
			_, p.err = fmt.Fprint(p.w, v)
		default:
			p.node(v)
		}
	}
}

func (p *printer) node(v interface{}) {
	switch x := v.(type) {
	case File:
		p.stmtJoinWithEnd(x.Stmts, false)
	case Stmt:
		spacing := ""
		spaced := func(a ...interface{}) {
			p.pr(spacing)
			p.pr(a...)
			spacing = " "
		}
		if x.Negated {
			spaced(NOT)
		}
		for _, a := range x.Assigns {
			spaced(a)
		}
		if x.Node != nil {
			spaced(x.Node)
		}
		for _, r := range x.Redirs {
			spaced(r.N, r.Op)
			if _, ok := r.Word.Parts[0].(CmdInput); ok {
				p.pr(" ")
			}
			p.pr(r.Word)
		}
		if x.Background {
			spaced(AND)
		}
	case Assign:
		if x.Name != nil {
			p.pr(x.Name)
			if x.Append {
				p.pr(ADD_ASSIGN)
			} else {
				p.pr(ASSIGN)
			}
		}
		p.pr(x.Value)
	case Command:
		p.wordJoin(x.Args, " ")
	case Subshell:
		p.pr(LPAREN)
		if len(x.Stmts) == 0 {
			// A space in between to avoid confusion with ()
			p.pr(" ")
		}
		p.stmtJoinWithEnd(x.Stmts, false)
		p.pr(RPAREN)
	case Block:
		p.pr(LBRACE)
		p.stmtList(x.Stmts)
		p.pr(RBRACE)
	case IfStmt:
		p.pr(IF)
		p.semicolonIfNil(x.Cond)
		p.pr(THEN)
		p.stmtList(x.ThenStmts)
		for _, el := range x.Elifs {
			p.pr(ELIF)
			p.semicolonIfNil(el.Cond)
			p.pr(THEN)
			p.stmtList(el.ThenStmts)
		}
		if len(x.ElseStmts) > 0 {
			p.pr(ELSE)
			p.stmtList(x.ElseStmts)
		}
		p.pr(FI)
	case StmtCond:
		p.stmtList(x.Stmts)
	case CStyleCond:
		p.pr(" ", DLPAREN, x.Cond, DRPAREN, SEMICOLON, " ")
	case WhileStmt:
		p.pr(WHILE)
		p.semicolonIfNil(x.Cond)
		p.pr(DO)
		p.stmtList(x.DoStmts)
		p.pr(DONE)
	case UntilStmt:
		p.pr(UNTIL)
		p.semicolonIfNil(x.Cond)
		p.pr(DO)
		p.stmtList(x.DoStmts)
		p.pr(DONE)
	case ForStmt:
		p.pr(FOR, " ", x.Cond, SEMICOLON, " ", DO)
		p.stmtList(x.DoStmts)
		p.pr(DONE)
	case WordIter:
		p.pr(x.Name)
		if len(x.List) > 0 {
			p.pr(" ", IN, " ")
			p.wordJoin(x.List, " ")
		}
	case CStyleLoop:
		p.pr(DLPAREN, x.Init, SEMICOLON, " ", x.Cond,
			SEMICOLON, " ", x.Post, DRPAREN)
	case UnaryExpr:
		if !x.Post {
			p.pr(x.Op)
		}
		p.pr(x.X)
		if x.Post {
			p.pr(x.Op)
		}
	case BinaryExpr:
		p.pr(x.X)
		if x.Op != COMMA {
			p.pr(" ")
		}
		p.pr(x.Op, " ", x.Y)
	case FuncDecl:
		if x.BashStyle {
			p.pr(FUNCTION, " ")
		}
		p.pr(x.Name, LPAREN, RPAREN, " ", x.Body)
	case Word:
		p.nodeJoin(x.Parts, "")
	case Lit:
		p.pr(x.Value)
	case SglQuoted:
		p.pr(SQUOTE, x.Value, SQUOTE)
	case Quoted:
		stop := x.Quote
		if stop == DOLLSQ {
			stop = SQUOTE
		} else if stop == DOLLDQ {
			stop = DQUOTE
		}
		p.pr(x.Quote)
		p.nodeJoin(x.Parts, "")
		p.pr(stop)
	case CmdSubst:
		if x.Backquotes {
			p.pr(BQUOTE)
		} else {
			p.pr(DOLLAR, "", LPAREN)
		}
		p.stmtJoin(x.Stmts)
		if x.Backquotes {
			p.pr(BQUOTE)
		} else {
			p.pr(RPAREN)
		}
	case ParamExp:
		if x.Short {
			p.pr(DOLLAR, x.Param)
			return
		}
		p.pr(DOLLAR, LBRACE)
		if x.Length {
			p.pr(HASH)
		}
		p.pr(x.Param)
		if x.Ind != nil {
			p.pr(LBRACK, x.Ind.Word, RBRACK)
		}
		if x.Repl != nil {
			if x.Repl.All {
				p.pr(QUO)
			}
			p.pr(QUO, x.Repl.Orig, QUO, x.Repl.With)
		}
		if x.Exp != nil {
			p.pr(x.Exp.Op, x.Exp.Word)
		}
		p.pr(RBRACE)
	case ArithmExpr:
		p.pr(DOLLAR, DLPAREN, x.X, DRPAREN)
	case ParenExpr:
		p.pr(LPAREN, x.X, RPAREN)
	case CaseStmt:
		p.pr(CASE, " ", x.Word, " ", IN)
		for i, pl := range x.List {
			if i > 0 {
				p.pr(DSEMICOLON)
			}
			p.pr(" ")
			p.wordJoin(pl.Patterns, " "+OR.String()+" ")
			p.pr(RPAREN, " ")
			p.stmtJoin(pl.Stmts)
		}
		p.pr(SEMICOLON, " ", ESAC)
	case DeclStmt:
		if x.Local {
			p.pr(LOCAL)
		} else {
			p.pr(DECLARE)
		}
		for _, w := range x.Opts {
			p.pr(" ", w)
		}
		for _, a := range x.Assigns {
			p.pr(" ", a)
		}
	case ArrayExpr:
		p.pr(LPAREN)
		p.wordJoin(x.List, " ")
		p.pr(RPAREN)
	case CmdInput:
		p.pr(CMDIN)
		p.stmtJoin(x.Stmts)
		p.pr(RPAREN)
	case EvalStmt:
		p.pr(EVAL, " ", x.Stmt)
	case LetStmt:
		p.pr(LET, " ")
		p.nodeJoin(x.Exprs, " ")
	}
}

func (p *printer) nodeJoin(ns []Node, sep string) {
	for i, n := range ns {
		if i > 0 {
			p.pr(sep)
		}
		p.node(n)
	}
}

func (p *printer) wordJoin(ws []Word, sep string) {
	for i, w := range ws {
		if i > 0 {
			p.pr(sep)
		}
		p.node(w)
	}
}

func newlineAfter(s Stmt) bool {
	for _, r := range s.Redirs {
		if r.Op == SHL || r.Op == DHEREDOC {
			return true
		}
	}
	return false
}

func (p *printer) stmtJoinWithEnd(stmts []Stmt, end bool) {
	p.needNewline = false
	for i, s := range stmts {
		if p.needNewline {
			p.needNewline = false
			p.pr("\n")
		} else if i > 0 {
			p.pr(SEMICOLON, " ")
		}
		p.node(s)
		p.needNewline = newlineAfter(s)
	}
}

func (p *printer) stmtJoin(stmts []Stmt) { p.stmtJoinWithEnd(stmts, true) }

func (p *printer) stmtList(stmts []Stmt) {
	if len(stmts) == 0 {
		p.pr(SEMICOLON, " ")
		return
	}
	p.pr(" ")
	p.stmtJoin(stmts)
	if p.needNewline {
		p.pr("\n")
	} else {
		p.pr(SEMICOLON, " ")
	}
}

func (p *printer) semicolonIfNil(v interface{}) {
	if v == nil {
		p.pr(SEMICOLON, " ")
		return
	}
	p.node(v)
}
<commit_msg>print: simplify statement join and newline logic<commit_after>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package sh

import (
	"fmt"
	"io"
)

func Fprint(w io.Writer, v interface{}) error {
	p := printer{w: w}
	p.node(v)
	return p.err
}

type printer struct {
	w   io.Writer
	err error

	needNewline bool
}

func (p *printer) pr(a ...interface{}) {
	for _, v := range a {
		if p.err != nil {
			break
		}
		switch v.(type) {
		case string, Token:
			_, p.err = fmt.Fprint(p.w, v)
		default:
			p.node(v)
		}
	}
}

func (p *printer) node(v interface{}) {
	switch x := v.(type) {
	case File:
		p.stmtJoin(x.Stmts)
	case Stmt:
		spacing := ""
		spaced := func(a ...interface{}) {
			p.pr(spacing)
			p.pr(a...)
			spacing = " "
		}
		if x.Negated {
			spaced(NOT)
		}
		for _, a := range x.Assigns {
			spaced(a)
		}
		if x.Node != nil {
			spaced(x.Node)
		}
		for _, r := range x.Redirs {
			spaced(r.N, r.Op)
			p.needNewline = r.Op == SHL || r.Op == DHEREDOC
			if _, ok := r.Word.Parts[0].(CmdInput); ok {
				p.pr(" ")
			}
			p.pr(r.Word)
		}
		if x.Background {
			spaced(AND)
		}
	case Assign:
		if x.Name != nil {
			p.pr(x.Name)
			if x.Append {
				p.pr(ADD_ASSIGN)
			} else {
				p.pr(ASSIGN)
			}
		}
		p.pr(x.Value)
	case Command:
		p.wordJoin(x.Args, " ")
	case Subshell:
		p.pr(LPAREN)
		if len(x.Stmts) == 0 {
			// A space in between to avoid confusion with ()
			p.pr(" ")
		}
		p.stmtJoin(x.Stmts)
		p.pr(RPAREN)
	case Block:
		p.pr(LBRACE)
		p.stmtList(x.Stmts)
		p.pr(RBRACE)
	case IfStmt:
		p.pr(IF)
		p.semicolonIfNil(x.Cond)
		p.pr(THEN)
		p.stmtList(x.ThenStmts)
		for _, el := range x.Elifs {
			p.pr(ELIF)
			p.semicolonIfNil(el.Cond)
			p.pr(THEN)
			p.stmtList(el.ThenStmts)
		}
		if len(x.ElseStmts) > 0 {
			p.pr(ELSE)
			p.stmtList(x.ElseStmts)
		}
		p.pr(FI)
	case StmtCond:
		p.stmtList(x.Stmts)
	case CStyleCond:
		p.pr(" ", DLPAREN, x.Cond, DRPAREN, SEMICOLON, " ")
	case WhileStmt:
		p.pr(WHILE)
		p.semicolonIfNil(x.Cond)
		p.pr(DO)
		p.stmtList(x.DoStmts)
		p.pr(DONE)
	case UntilStmt:
		p.pr(UNTIL)
		p.semicolonIfNil(x.Cond)
		p.pr(DO)
		p.stmtList(x.DoStmts)
		p.pr(DONE)
	case ForStmt:
		p.pr(FOR, " ", x.Cond, SEMICOLON, " ", DO)
		p.stmtList(x.DoStmts)
		p.pr(DONE)
	case WordIter:
		p.pr(x.Name)
		if len(x.List) > 0 {
			p.pr(" ", IN, " ")
			p.wordJoin(x.List, " ")
		}
	case CStyleLoop:
		p.pr(DLPAREN, x.Init, SEMICOLON, " ", x.Cond,
			SEMICOLON, " ", x.Post, DRPAREN)
	case UnaryExpr:
		if !x.Post {
			p.pr(x.Op)
		}
		p.pr(x.X)
		if x.Post {
			p.pr(x.Op)
		}
	case BinaryExpr:
		p.pr(x.X)
		if x.Op != COMMA {
			p.pr(" ")
		}
		p.pr(x.Op, " ", x.Y)
	case FuncDecl:
		if x.BashStyle {
			p.pr(FUNCTION, " ")
		}
		p.pr(x.Name, LPAREN, RPAREN, " ", x.Body)
	case Word:
		p.nodeJoin(x.Parts, "")
	case Lit:
		p.pr(x.Value)
	case SglQuoted:
		p.pr(SQUOTE, x.Value, SQUOTE)
	case Quoted:
		stop := x.Quote
		if stop == DOLLSQ {
			stop = SQUOTE
		} else if stop == DOLLDQ {
			stop = DQUOTE
		}
		p.pr(x.Quote)
		p.nodeJoin(x.Parts, "")
		p.pr(stop)
	case CmdSubst:
		if x.Backquotes {
			p.pr(BQUOTE)
		} else {
			p.pr(DOLLAR, "", LPAREN)
		}
		p.stmtJoin(x.Stmts)
		if x.Backquotes {
			p.pr(BQUOTE)
		} else {
			p.pr(RPAREN)
		}
	case ParamExp:
		if x.Short {
			p.pr(DOLLAR, x.Param)
			return
		}
		p.pr(DOLLAR, LBRACE)
		if x.Length {
			p.pr(HASH)
		}
		p.pr(x.Param)
		if x.Ind != nil {
			p.pr(LBRACK, x.Ind.Word, RBRACK)
		}
		if x.Repl != nil {
			if x.Repl.All {
				p.pr(QUO)
			}
			p.pr(QUO, x.Repl.Orig, QUO, x.Repl.With)
		} else if x.Exp != nil {
			p.pr(x.Exp.Op, x.Exp.Word)
		}
		p.pr(RBRACE)
	case ArithmExpr:
		p.pr(DOLLAR, DLPAREN, x.X, DRPAREN)
	case ParenExpr:
		p.pr(LPAREN, x.X, RPAREN)
	case CaseStmt:
		p.pr(CASE, " ", x.Word, " ", IN)
		for i, pl := range x.List {
			if i > 0 {
				p.pr(DSEMICOLON)
			}
			p.pr(" ")
			p.wordJoin(pl.Patterns, " "+OR.String()+" ")
			p.pr(RPAREN, " ")
			p.stmtJoin(pl.Stmts)
		}
		p.pr(SEMICOLON, " ", ESAC)
	case DeclStmt:
		if x.Local {
			p.pr(LOCAL)
		} else {
			p.pr(DECLARE)
		}
		for _, w := range x.Opts {
			p.pr(" ", w)
		}
		for _, a := range x.Assigns {
			p.pr(" ", a)
		}
	case ArrayExpr:
		p.pr(LPAREN)
		p.wordJoin(x.List, " ")
		p.pr(RPAREN)
	case CmdInput:
		p.pr(CMDIN)
		p.stmtJoin(x.Stmts)
		p.pr(RPAREN)
	case EvalStmt:
		p.pr(EVAL, " ", x.Stmt)
	case LetStmt:
		p.pr(LET, " ")
		p.nodeJoin(x.Exprs, " ")
	}
}

func (p *printer) nodeJoin(ns []Node, sep string) {
	for i, n := range ns {
		if i > 0 {
			p.pr(sep)
		}
		p.node(n)
	}
}

func (p *printer) wordJoin(ws []Word, sep string) {
	for i, w := range ws {
		if i > 0 {
			p.pr(sep)
		}
		p.node(w)
	}
}

func (p *printer) stmtJoin(stmts []Stmt) {
	for i, s := range stmts {
		if p.needNewline {
			p.pr("\n")
		} else if i > 0 {
			p.pr(SEMICOLON, " ")
		}
		p.node(s)
	}
}

func (p *printer) stmtList(stmts []Stmt) {
	if len(stmts) == 0 {
		p.pr(SEMICOLON, " ")
		return
	}
	p.pr(" ")
	p.stmtJoin(stmts)
	if p.needNewline {
		p.pr("\n")
	} else {
		p.pr(SEMICOLON, " ")
	}
}

func (p *printer) semicolonIfNil(v interface{}) {
	if v == nil {
		p.pr(SEMICOLON, " ")
		return
	}
	p.node(v)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"os"
)

type IPAddress struct {
	IP string `json:"ip"`
}

func jsonip(w http.ResponseWriter, r *http.Request) {
	host := net.ParseIP(r.Header["X-Forwarded-For"][len(r.Header["X-Forwarded-For"])-1]).String()
	jsonStr, _ := json.Marshal(IPAddress{host})

	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, string(jsonStr))
}

func textip(w http.ResponseWriter, r *http.Request) {
	host := net.ParseIP(r.Header["X-Forwarded-For"][len(r.Header["X-Forwarded-For"])-1]).String()

	w.Header().Set("Content-Type", "text/plain")
	fmt.Fprintf(w, host)
}

func NotFound(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(404)
}

func main() {
	http.HandleFunc("/json", jsonip)
	http.HandleFunc("/text", textip)

	err := http.ListenAndServe(":"+os.Getenv("PORT"), nil)
	if err != nil {
		panic(err)
	}
}
<commit_msg>Adding some debug.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"os"
)

type IPAddress struct {
	IP string `json:"ip"`
}

func jsonip(w http.ResponseWriter, r *http.Request) {
	host := net.ParseIP(r.Header["X-Forwarded-For"][len(r.Header["X-Forwarded-For"])-1]).String()
	jsonStr, _ := json.Marshal(IPAddress{host})

	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, string(jsonStr))
}

func textip(w http.ResponseWriter, r *http.Request) {
	err := r.ParseForm()
	if err != nil {
		panic(err)
	}
	fmt.Println(r.Form)
	//fmt.Fprintf(w, r.Form["format"])

	host := net.ParseIP(r.Header["X-Forwarded-For"][len(r.Header["X-Forwarded-For"])-1]).String()

	w.Header().Set("Content-Type", "text/plain")
	fmt.Fprintf(w, host)
}

func NotFound(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(404)
}

func main() {
	http.HandleFunc("/json", jsonip)
	http.HandleFunc("/text", textip)

	err := http.ListenAndServe(":"+os.Getenv("PORT"), nil)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package ql

type queryBuilder interface {
	ToSql() (string, []interface{})
}

func makeSql(b queryBuilder) string {
	sql, err := Preprocess(b.ToSql())
	if err != nil {
		panic(err)
	}
	return sql
}

func (q *Query) String() string {
	return makeSql(q)
}

func (b *DeleteBuilder) String() string {
	return makeSql(b)
}

func (b *InsertBuilder) String() string {
	return makeSql(b)
}

func (b *SelectBuilder) String() string {
	return makeSql(b)
}

func (b *UpdateBuilder) String() string {
	return makeSql(b)
}
<commit_msg>comment out the String methods<commit_after>package ql

type queryBuilder interface {
	ToSql() (string, []interface{})
}

func makeSql(b queryBuilder) string {
	sql, err := Preprocess(b.ToSql())
	if err != nil {
		panic(err)
	}
	return sql
}

// String returns a string representing a preprocessed, interpolated, query.
func (q *Query) String() string {
	return makeSql(q)
}

// String returns a string representing a preprocessed, interpolated, query.
func (b *DeleteBuilder) String() string {
	return makeSql(b)
}

// String returns a string representing a preprocessed, interpolated, query.
func (b *InsertBuilder) String() string {
	return makeSql(b)
}

// String returns a string representing a preprocessed, interpolated, query.
func (b *SelectBuilder) String() string {
	return makeSql(b)
}

// String returns a string representing a preprocessed, interpolated, query.
func (b *UpdateBuilder) String() string {
	return makeSql(b)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package sh

import (
	"fmt"
	"io"
	"strings"
)

func Fprint(w io.Writer, n Node) error {
	p := printer{
		w:       w,
		curLine: 1,
	}
	p.node(n)
	p.space('\n')
	return p.err
}

type printer struct {
	w   io.Writer
	err error

	contiguous bool

	curLine int

	compactArithm bool
}

var (
	contiguousRight = map[Token]bool{
		DOLLPR:  true,
		LPAREN:  true,
		DLPAREN: true,
		BQUOTE:  true,
		CMDIN:   true,
		DOLLDP:  true,
	}
	contiguousLeft = map[Token]bool{
		SEMICOLON: true,
		RPAREN:    true,
		DRPAREN:   true,
		COMMA:     true,
	}
)

func (p *printer) space(b byte) {
	if p.err != nil {
		return
	}
	_, p.err = p.w.Write([]byte{b})
	p.contiguous = false
}

func (p *printer) nonSpaced(a ...interface{}) {
	for _, v := range a {
		if p.err != nil {
			break
		}
		switch x := v.(type) {
		case string:
			if len(x) > 0 {
				last := x[len(x)-1]
				p.contiguous = !space[last]
			}
			_, p.err = io.WriteString(p.w, x)
			p.curLine += strings.Count(x, "\n")
		case Token:
			p.contiguous = !contiguousRight[x]
			_, p.err = fmt.Fprint(p.w, x)
		case Node:
			p.node(x)
		}
	}
}

func (p *printer) spaced(a ...interface{}) {
	for _, v := range a {
		if v == nil {
			continue
		}
		if t, ok := v.(Token); ok && contiguousLeft[t] {
		} else if p.contiguous {
			p.space(' ')
		}
		p.nonSpaced(v)
	}
}

func (p *printer) separate(pos Pos, fallback bool) {
	if p.curLine == 0 {
		return
	}
	if pos.Line > p.curLine {
		p.space('\n')
		if pos.Line > p.curLine+1 {
			// preserve single empty lines
			p.space('\n')
		}
		p.curLine = pos.Line
	} else if fallback {
		p.nonSpaced(SEMICOLON)
	}
}

func (p *printer) sepSemicolon(v interface{}, pos Pos) {
	p.separate(pos, true)
	p.spaced(v)
}

func (p *printer) sepNewline(v interface{}, pos Pos) {
	p.separate(pos, false)
	p.spaced(v)
}

func (p *printer) node(n Node) {
	switch x := n.(type) {
	case File:
		p.stmtJoin(x.Stmts)
	case Stmt:
		if x.Negated {
			p.spaced(NOT)
		}
		for _, a := range x.Assigns {
			p.spaced(a)
		}
		p.spaced(x.Node)
		for _, r := range x.Redirs {
			p.spaced(r.N)
			p.nonSpaced(r.Op, r.Word)
		}
		if x.Background {
			p.spaced(AND)
		}
	case Assign:
		if x.Name != nil {
			p.spaced(x.Name)
			if x.Append {
				p.nonSpaced(ADD_ASSIGN)
			} else {
				p.nonSpaced(ASSIGN)
			}
		}
		p.nonSpaced(x.Value)
	case Command:
		p.wordJoin(x.Args)
	case Subshell:
		p.spaced(LPAREN)
		if len(x.Stmts) == 0 {
			// avoid conflict with ()
			p.space(' ')
		}
		p.stmtJoin(x.Stmts)
		p.sepNewline(RPAREN, x.Rparen)
	case Block:
		p.spaced(LBRACE)
		p.stmtList(x.Stmts)
		p.sepSemicolon(RBRACE, x.Rbrace)
	case IfStmt:
		p.spaced(IF, x.Cond, SEMICOLON, THEN)
		p.stmtList(x.ThenStmts)
		for _, el := range x.Elifs {
			p.sepSemicolon(ELIF, el.Elif)
			p.spaced(el.Cond, SEMICOLON, THEN)
			p.stmtList(el.ThenStmts)
		}
		if len(x.ElseStmts) > 0 {
			p.sepSemicolon(ELSE, x.Else)
			p.stmtList(x.ElseStmts)
		}
		p.sepSemicolon(FI, x.Fi)
	case StmtCond:
		p.stmtJoin(x.Stmts)
	case CStyleCond:
		p.spaced(DLPAREN, x.Cond, DRPAREN)
	case WhileStmt:
		p.spaced(WHILE, x.Cond, SEMICOLON, DO)
		p.stmtList(x.DoStmts)
		p.sepSemicolon(DONE, x.Done)
	case UntilStmt:
		p.spaced(UNTIL, x.Cond, SEMICOLON, DO)
		p.stmtList(x.DoStmts)
		p.sepSemicolon(DONE, x.Done)
	case ForStmt:
		p.spaced(FOR, x.Cond, SEMICOLON, DO)
		p.stmtList(x.DoStmts)
		p.sepSemicolon(DONE, x.Done)
	case WordIter:
		p.spaced(x.Name)
		if len(x.List) > 0 {
			p.spaced(IN)
			p.wordJoin(x.List)
		}
	case CStyleLoop:
		p.spaced(DLPAREN, x.Init, SEMICOLON, x.Cond,
			SEMICOLON, x.Post, DRPAREN)
	case UnaryExpr:
		if x.Post {
			p.nonSpaced(x.X, x.Op)
		} else {
			p.nonSpaced(x.Op)
			p.contiguous = false
			p.nonSpaced(x.X)
		}
	case BinaryExpr:
		if p.compactArithm {
			p.nonSpaced(x.X, x.Op, x.Y)
		} else {
			p.spaced(x.X, x.Op, x.Y)
		}
	case FuncDecl:
		if x.BashStyle {
			p.spaced(FUNCTION)
		}
		p.spaced(x.Name)
		p.nonSpaced(LPAREN, RPAREN)
		p.spaced(x.Body)
	case Word:
		for _, n := range x.Parts {
			p.nonSpaced(n)
		}
	case Lit:
		p.nonSpaced(x.Value)
	case SglQuoted:
		p.nonSpaced(SQUOTE, x.Value, SQUOTE)
	case Quoted:
		p.nonSpaced(x.Quote)
		for _, n := range x.Parts {
			p.nonSpaced(n)
		}
		p.nonSpaced(quotedStop(x.Quote))
	case CmdSubst:
		if x.Backquotes {
			p.nonSpaced(BQUOTE)
		} else {
			p.nonSpaced(DOLLPR)
		}
		p.stmtJoin(x.Stmts)
		if x.Backquotes {
			p.nonSpaced(BQUOTE)
		} else {
			p.nonSpaced(RPAREN)
		}
	case ParamExp:
		if x.Short {
			p.nonSpaced(DOLLAR, x.Param)
			return
		}
		p.nonSpaced(DOLLBR)
		if x.Length {
			p.nonSpaced(HASH)
		}
		p.nonSpaced(x.Param)
		if x.Ind != nil {
			p.nonSpaced(LBRACK, x.Ind.Word, RBRACK)
		}
		if x.Repl != nil {
			if x.Repl.All {
				p.nonSpaced(QUO)
			}
			p.nonSpaced(QUO, x.Repl.Orig, QUO, x.Repl.With)
		} else if x.Exp != nil {
			p.nonSpaced(x.Exp.Op, x.Exp.Word)
		}
		p.nonSpaced(RBRACE)
	case ArithmExpr:
		p.nonSpaced(DOLLDP, x.X, DRPAREN)
	case ParenExpr:
		p.nonSpaced(LPAREN)
		oldCompact := p.compactArithm
		p.compactArithm = false
		p.nonSpaced(x.X)
		p.compactArithm = oldCompact
		p.nonSpaced(RPAREN)
	case CaseStmt:
		p.spaced(CASE, x.Word, IN)
		for i, pl := range x.List {
			if i > 0 {
				p.nonSpaced(DSEMICOLON)
			}
			for i, w := range pl.Patterns {
				if i > 0 {
					p.spaced(OR)
				}
				p.spaced(w)
			}
			p.nonSpaced(RPAREN)
			p.stmtJoin(pl.Stmts)
		}
		p.sepSemicolon(ESAC, x.Esac)
	case DeclStmt:
		if x.Local {
			p.spaced(LOCAL)
		} else {
			p.spaced(DECLARE)
		}
		for _, w := range x.Opts {
			p.spaced(w)
		}
		for _, a := range x.Assigns {
			p.spaced(a)
		}
	case ArrayExpr:
		p.nonSpaced(LPAREN)
		p.wordJoin(x.List)
		p.nonSpaced(RPAREN)
	case CmdInput:
		// avoid conflict with <<
		p.spaced(CMDIN)
		p.stmtJoin(x.Stmts)
		p.nonSpaced(RPAREN)
	case EvalStmt:
		p.spaced(EVAL, x.Stmt)
	case LetStmt:
		p.spaced(LET)
		p.compactArithm = true
		for _, n := range x.Exprs {
			p.spaced(n)
		}
		p.compactArithm = false
	}
}

func (p *printer) wordJoin(ws []Word) {
	for _, w := range ws {
		p.spaced(w)
	}
}

func (p *printer) stmtJoin(stmts []Stmt) {
	for i, s := range stmts {
		p.separate(s.Pos(), i > 0)
		p.node(s)
	}
}

func (p *printer) stmtList(stmts []Stmt) {
	if len(stmts) == 0 {
		return
	}
	p.stmtJoin(stmts)
}
<commit_msg>print: remove now redundant stmtList<commit_after>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package sh

import (
	"fmt"
	"io"
	"strings"
)

func Fprint(w io.Writer, n Node) error {
	p := printer{
		w:       w,
		curLine: 1,
	}
	p.node(n)
	p.space('\n')
	return p.err
}

type printer struct {
	w   io.Writer
	err error

	contiguous bool

	curLine int

	compactArithm bool
}

var (
	contiguousRight = map[Token]bool{
		DOLLPR:  true,
		LPAREN:  true,
		DLPAREN: true,
		BQUOTE:  true,
		CMDIN:   true,
		DOLLDP:  true,
	}
	contiguousLeft = map[Token]bool{
		SEMICOLON: true,
		RPAREN:    true,
		DRPAREN:   true,
		COMMA:     true,
	}
)

func (p *printer) space(b byte) {
	if p.err != nil {
		return
	}
	_, p.err = p.w.Write([]byte{b})
	p.contiguous = false
}

func (p *printer) nonSpaced(a ...interface{}) {
	for _, v := range a {
		if p.err != nil {
			break
		}
		switch x := v.(type) {
		case string:
			if len(x) > 0 {
				last := x[len(x)-1]
				p.contiguous = !space[last]
			}
			_, p.err = io.WriteString(p.w, x)
			p.curLine += strings.Count(x, "\n")
		case Token:
			p.contiguous = !contiguousRight[x]
			_, p.err = fmt.Fprint(p.w, x)
		case Node:
			p.node(x)
		}
	}
}

func (p *printer) spaced(a ...interface{}) {
	for _, v := range a {
		if v == nil {
			continue
		}
		if t, ok := v.(Token); ok && contiguousLeft[t] {
		} else if p.contiguous {
			p.space(' ')
		}
		p.nonSpaced(v)
	}
}

func (p *printer) separate(pos Pos, fallback bool) {
	if p.curLine == 0 {
		return
	}
	if pos.Line > p.curLine {
		p.space('\n')
		if pos.Line > p.curLine+1 {
			// preserve single empty lines
			p.space('\n')
		}
		p.curLine = pos.Line
	} else if fallback {
		p.nonSpaced(SEMICOLON)
	}
}

func (p *printer) sepSemicolon(v interface{}, pos Pos) {
	p.separate(pos, true)
	p.spaced(v)
}

func (p *printer) sepNewline(v interface{}, pos Pos) {
	p.separate(pos, false)
	p.spaced(v)
}

func (p *printer) node(n Node) {
	switch x := n.(type) {
	case File:
		p.stmtJoin(x.Stmts)
	case Stmt:
		if x.Negated {
			p.spaced(NOT)
		}
		for _, a := range x.Assigns {
			p.spaced(a)
		}
		p.spaced(x.Node)
		for _, r := range x.Redirs {
			p.spaced(r.N)
			p.nonSpaced(r.Op, r.Word)
		}
		if x.Background {
			p.spaced(AND)
		}
	case Assign:
		if x.Name != nil {
			p.spaced(x.Name)
			if x.Append {
				p.nonSpaced(ADD_ASSIGN)
			} else {
				p.nonSpaced(ASSIGN)
			}
		}
		p.nonSpaced(x.Value)
	case Command:
		p.wordJoin(x.Args)
	case Subshell:
		p.spaced(LPAREN)
		if len(x.Stmts) == 0 {
			// avoid conflict with ()
			p.space(' ')
		}
		p.stmtJoin(x.Stmts)
		p.sepNewline(RPAREN, x.Rparen)
	case Block:
		p.spaced(LBRACE)
		p.stmtJoin(x.Stmts)
		p.sepSemicolon(RBRACE, x.Rbrace)
	case IfStmt:
		p.spaced(IF, x.Cond, SEMICOLON, THEN)
		p.stmtJoin(x.ThenStmts)
		for _, el := range x.Elifs {
			p.sepSemicolon(ELIF, el.Elif)
			p.spaced(el.Cond, SEMICOLON, THEN)
			p.stmtJoin(el.ThenStmts)
		}
		if len(x.ElseStmts) > 0 {
			p.sepSemicolon(ELSE, x.Else)
			p.stmtJoin(x.ElseStmts)
		}
		p.sepSemicolon(FI, x.Fi)
	case StmtCond:
		p.stmtJoin(x.Stmts)
	case CStyleCond:
		p.spaced(DLPAREN, x.Cond, DRPAREN)
	case WhileStmt:
		p.spaced(WHILE, x.Cond, SEMICOLON, DO)
		p.stmtJoin(x.DoStmts)
		p.sepSemicolon(DONE, x.Done)
	case UntilStmt:
		p.spaced(UNTIL, x.Cond, SEMICOLON, DO)
		p.stmtJoin(x.DoStmts)
		p.sepSemicolon(DONE, x.Done)
	case ForStmt:
		p.spaced(FOR, x.Cond, SEMICOLON, DO)
		p.stmtJoin(x.DoStmts)
		p.sepSemicolon(DONE, x.Done)
	case WordIter:
		p.spaced(x.Name)
		if len(x.List) > 0 {
			p.spaced(IN)
			p.wordJoin(x.List)
		}
	case CStyleLoop:
		p.spaced(DLPAREN, x.Init, SEMICOLON, x.Cond,
			SEMICOLON, x.Post, DRPAREN)
	case UnaryExpr:
		if x.Post {
			p.nonSpaced(x.X, x.Op)
		} else {
			p.nonSpaced(x.Op)
			p.contiguous = false
			p.nonSpaced(x.X)
		}
	case BinaryExpr:
		if p.compactArithm {
			p.nonSpaced(x.X, x.Op, x.Y)
		} else {
			p.spaced(x.X, x.Op, x.Y)
		}
	case FuncDecl:
		if x.BashStyle {
			p.spaced(FUNCTION)
		}
		p.spaced(x.Name)
		p.nonSpaced(LPAREN, RPAREN)
		p.spaced(x.Body)
	case Word:
		for _, n := range x.Parts {
			p.nonSpaced(n)
		}
	case Lit:
		p.nonSpaced(x.Value)
	case SglQuoted:
		p.nonSpaced(SQUOTE, x.Value, SQUOTE)
	case Quoted:
		p.nonSpaced(x.Quote)
		for _, n := range x.Parts {
			p.nonSpaced(n)
		}
		p.nonSpaced(quotedStop(x.Quote))
	case CmdSubst:
		if x.Backquotes {
			p.nonSpaced(BQUOTE)
		} else {
			p.nonSpaced(DOLLPR)
		}
		p.stmtJoin(x.Stmts)
		if x.Backquotes {
			p.nonSpaced(BQUOTE)
		} else {
			p.nonSpaced(RPAREN)
		}
	case ParamExp:
		if x.Short {
			p.nonSpaced(DOLLAR, x.Param)
			return
		}
		p.nonSpaced(DOLLBR)
		if x.Length {
			p.nonSpaced(HASH)
		}
		p.nonSpaced(x.Param)
		if x.Ind != nil {
			p.nonSpaced(LBRACK, x.Ind.Word, RBRACK)
		}
		if x.Repl != nil {
			if x.Repl.All {
				p.nonSpaced(QUO)
			}
			p.nonSpaced(QUO, x.Repl.Orig, QUO, x.Repl.With)
		} else if x.Exp != nil {
			p.nonSpaced(x.Exp.Op, x.Exp.Word)
		}
		p.nonSpaced(RBRACE)
	case ArithmExpr:
		p.nonSpaced(DOLLDP, x.X, DRPAREN)
	case ParenExpr:
		p.nonSpaced(LPAREN)
		oldCompact := p.compactArithm
		p.compactArithm = false
		p.nonSpaced(x.X)
		p.compactArithm = oldCompact
		p.nonSpaced(RPAREN)
	case CaseStmt:
		p.spaced(CASE, x.Word, IN)
		for i, pl := range x.List {
			if i > 0 {
				p.nonSpaced(DSEMICOLON)
			}
			for i, w := range pl.Patterns {
				if i > 0 {
					p.spaced(OR)
				}
				p.spaced(w)
			}
			p.nonSpaced(RPAREN)
			p.stmtJoin(pl.Stmts)
		}
		p.sepSemicolon(ESAC, x.Esac)
	case DeclStmt:
		if x.Local {
			p.spaced(LOCAL)
		} else {
			p.spaced(DECLARE)
		}
		for _, w := range x.Opts {
			p.spaced(w)
		}
		for _, a := range x.Assigns {
			p.spaced(a)
		}
	case ArrayExpr:
		p.nonSpaced(LPAREN)
		p.wordJoin(x.List)
		p.nonSpaced(RPAREN)
	case CmdInput:
		// avoid conflict with <<
		p.spaced(CMDIN)
		p.stmtJoin(x.Stmts)
		p.nonSpaced(RPAREN)
	case EvalStmt:
		p.spaced(EVAL, x.Stmt)
	case LetStmt:
		p.spaced(LET)
		p.compactArithm = true
		for _, n := range x.Exprs {
			p.spaced(n)
		}
		p.compactArithm = false
	}
}

func (p *printer) wordJoin(ws []Word) {
	for _, w := range ws {
		p.spaced(w)
	}
}

func (p *printer) stmtJoin(stmts []Stmt) {
	for i, s := range stmts {
		p.separate(s.Pos(), i > 0)
		p.node(s)
	}
}
<|endoftext|>