<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/lib/pq"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

const OPEN311_API_URI = "http://311api.cityofchicago.org/open311/v2/requests.json?extensions=true&page_size=500"

type Open311Request struct {
	Lat, Long                                                                                float64
	Ward, Police_district                                                                                int
	Service_request_id, Status, Service_name, Service_code, Agency_responsible, Address, Channel, Media_url string
	Requested_datetime, Updated_datetime                                                                    string // FIXME: should these be proper time objects?
	Extended_attributes                                                                                map[string]interface{}
	Notes                                                                                []map[string]interface{}
}

type Worker struct {
	Db        *sql.DB
	LastRunAt time.Time
}

var worker Worker
var sr_number string
var backfill bool
var backfill_date string

func init() {
	// open database
	db, err := sql.Open("postgres", "dbname=cwfy sslmode=disable")
	if err != nil {
		log.Fatal("Cannot open database connection", err)
	}
	worker.Db = db

	// fetch SR num from command line, if present
	flag.StringVar(&sr_number, "sr-number", "", "SR number to fetch")
	flag.BoolVar(&backfill, "backfill", false, "run in reverse and backfill data")
	flag.StringVar(&backfill_date, "backfill-from", "", "date to start backfilling data from. Use RFC3339 format. Default will be the time of the least recently updated SR in the database.")
}

func main() {
	defer worker.Db.Close()
	flag.Parse()

	if sr_number != "" {
		sr := fetchSingleRequest(sr_number)
		sr.Save()
		return
	}

	start_backfill_from := backfill_date
	for {
		switch {
		case backfill:
			requests := backFillRequests(start_backfill_from)
			for _, request := range requests {
				request.Save()
			}

			start_backfill_from = requests[len(requests)-1].Updated_datetime // FIXME: is it safe to assume the items are sorted?

		case time.Since(worker.LastRunAt) > (30 * time.Second):
			// load requests from open311
			for _, request := range fetchRequests() {
				request.Save()
			}
			worker.LastRunAt = time.Now()
		default:
			log.Print("sleeping for 10 seconds")
			time.Sleep(10 * time.Second)
		}
	}
}

func (req Open311Request) String() string {
	// pretty print SR information
	return fmt.Sprintf("%s: %s at %s %f,%f, last update %s", req.Service_request_id, req.Service_name, req.Address, req.Lat, req.Long, req.Updated_datetime)
}

func (req Open311Request) Save() (persisted bool) {
	// create or update a SR

	// open311 says we should always ignore a SR that does not have a SR# assigned
	if req.Service_request_id == "" {
		log.Printf("cowardly refusing to create a new SR record because of empty SR#. Request type is %s", req.Service_name)
		return false
	}

	persisted = false

	// find existing record if exists
	var existing_id int
	err := worker.Db.QueryRow("SELECT id FROM service_requests WHERE service_request_id = $1", req.Service_request_id).Scan(&existing_id)
	switch {
	case err == sql.ErrNoRows:
		// log.Printf("did not find existing record %s", req.Service_request_id)
	case err != nil:
		// log.Print("error searching for existing SR", err)
	default:
		persisted = true
		// log.Printf("found existing sr %s", req.Service_request_id)
	}

	var stmt *sql.Stmt

	if !persisted {
		// create new record
		stmt, err = worker.Db.Prepare("INSERT INTO service_requests(service_request_id," +
			"status, service_name, service_code, agency_responsible, " +
			"address, requested_datetime, updated_datetime, lat, long," +
			"ward, police_district, media_url, channel, duplicate, parent_service_request_id, closed_datetime) " +
			"VALUES ($1::varchar, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17); ")

		// "WHERE NOT EXISTS (SELECT 1 FROM service_requests WHERE service_request_id = $1);")

		if err != nil {
			log.Fatal("error preparing database insert statement", err)
		}

	} else {
		// update existing record
		stmt, err = worker.Db.Prepare("UPDATE service_requests SET " +
			"status = $2, service_name = $3, service_code = $4, agency_responsible = $5, " +
			"address = $6, requested_datetime = $7, updated_datetime = $8, lat = $9, long = $10," +
			"ward = $11, police_district = $12, media_url = $13, channel = $14, duplicate = $15, " +
			"parent_service_request_id = $16, updated_at = NOW(), closed_datetime = $17 WHERE service_request_id = $1;")

		if err != nil {
			log.Fatal("error preparing database update statement", err)
		}
	}

	tx, err := worker.Db.Begin()

	if err != nil {
		log.Fatal("error beginning transaction", err)
	}

	t := req.ExtractClosedDatetime()
	closed_time := pq.NullTime{Time: t, Valid: !t.IsZero()}

	_, err = tx.Stmt(stmt).Exec(req.Service_request_id,
		req.Status,
		req.Service_name,
		req.Service_code,
		req.Agency_responsible,
		req.Address,
		req.Requested_datetime,
		req.Updated_datetime,
		req.Lat,
		req.Long,
		req.Extended_attributes["ward"],
		req.Extended_attributes["police_district"],
		req.Media_url,
		req.Extended_attributes["channel"],
		req.Extended_attributes["duplicate"],
		req.Extended_attributes["parent_service_request_id"],
		closed_time,
	)

	if err != nil {
		log.Fatalf("could not update %s because %s", req.Service_request_id, err)
	} else {
		var verb string
		switch {
		case !persisted && closed_time.Time.IsZero():
			verb = "CREATED"
		case !persisted && !closed_time.Time.IsZero():
			verb = "CREATED/CLOSED"
		case persisted && closed_time.Time.IsZero():
			verb = "UPDATED"
		case persisted && !closed_time.Time.IsZero():
			verb = "UPDATED/CLOSED"
		}

		log.Printf("[%s] %s", verb, req)
		persisted = true
	}

	err = tx.Commit()
	if err != nil {
		log.Fatal("error closing transaction", err)
	}

	return persisted

	// calculate closed time if necessary

}

func (req Open311Request) ExtractClosedDatetime() time.Time {
	// given an extended_attributes JSON blob, pluck out the closed time, if present
	// req.PrintNotes()

	var closed_at time.Time
	for _, note := range req.Notes {
		if note["type"] == "closed" {
			parsed_date, err := time.Parse("2006-01-02T15:04:05-07:00", note["datetime"].(string))
			if err != nil {
				log.Print("error parsing date", err)
			}
			log.Printf("SR %s closed at: %s", req, parsed_date)
			closed_at = parsed_date
		}
	}

	return closed_at
}

func (req Open311Request) PrintNotes() {
	fmt.Printf("Notes for SR %s:\n", req.Service_request_id)

	for _, note := range req.Notes {
		fmt.Printf("%+v\n", note)
	}
}

func fetchSingleRequest(sr_number string) (request Open311Request) {
	// given an SR, fetch the record
	log.Printf("fetching single SR %s", sr_number)
	open311_api_endpoint := fmt.Sprintf("http://311api.cityofchicago.org/open311/v2/requests/%s.json?extensions=true", sr_number)

	log.Printf("fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	var requests []Open311Request

	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("error parsing JSON:", err)
	}

	log.Printf("received %d requests from Open311", len(requests))

	return requests[0]
}

func fetchRequests() (requests []Open311Request) {
	last_updated_at := time.Now()
	if err := worker.Db.QueryRow("SELECT MAX(updated_datetime) FROM service_requests;").Scan(&last_updated_at); err != nil {
		log.Print("[fetchRequests] error loading most recent SR, will fallback to current time: ", err)
	}

	log.Print("[fetchRequests] most recent SR timestamp ", last_updated_at.Format(time.RFC3339))

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_after=" + last_updated_at.Format(time.RFC3339)

	log.Printf("[fetchRequests] fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("[fetchRequests] error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[fetchRequests] error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[fetchRequests] error parsing JSON:", err)
	}

	log.Printf("[fetchRequests] received %d requests from Open311", len(requests))

	return requests
}

func backFillRequests(start_from string) (requests []Open311Request) {
	var fetch_from time.Time

	if start_from == "" {
		err := worker.Db.QueryRow("SELECT updated_datetime FROM service_requests ORDER BY updated_datetime ASC LIMIT 1").Scan(&fetch_from)
		if err != nil {
			log.Println("error fetching oldest SR:", err)
		}
		log.Printf("no start_from value provided, so falling back to oldest (by last update) SR in the database: %s", fetch_from)
	} else {
		t, err := time.Parse(time.RFC3339, start_from)
		if err != nil {
			log.Fatal("[backfill] error parsing date to start from", err)
		}
		fetch_from = t
	}

	formatted_date_string_with_tz := fetch_from.Format(time.RFC3339)

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_before=" + formatted_date_string_with_tz

	log.Printf("[backfill] fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("[backfill] error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[backfill] error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[backfill] error parsing JSON:", err)
	}

	log.Printf("[backfill] received %d requests from Open311", len(requests))

	return requests
}
<commit_msg>marshal the notes field to JSON and store in database<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/lib/pq"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

const OPEN311_API_URI = "http://311api.cityofchicago.org/open311/v2/requests.json?extensions=true&page_size=500"

type Open311Request struct {
	Lat, Long                                                                                float64
	Ward, Police_district                                                                                int
	Service_request_id, Status, Service_name, Service_code, Agency_responsible, Address, Channel, Media_url string
	Requested_datetime, Updated_datetime                                                                    string // FIXME: should these be proper time objects?
	Extended_attributes                                                                                map[string]interface{}
	Notes                                                                                []map[string]interface{}
}

type Worker struct {
	Db        *sql.DB
	LastRunAt time.Time
}

var worker Worker
var sr_number string
var backfill bool
var backfill_date string

func init() {
	// open database
	db, err := sql.Open("postgres", "dbname=cwfy sslmode=disable")
	if err != nil {
		log.Fatal("Cannot open database connection", err)
	}
	worker.Db = db

	// fetch SR num from command line, if present
	flag.StringVar(&sr_number, "sr-number", "", "SR number to fetch")
	flag.BoolVar(&backfill, "backfill", false, "run in reverse and backfill data")
	flag.StringVar(&backfill_date, "backfill-from", "", "date to start backfilling data from. Use RFC3339 format. Default will be the time of the least recently updated SR in the database.")
}

func main() {
	defer worker.Db.Close()
	flag.Parse()

	if sr_number != "" {
		sr := fetchSingleRequest(sr_number)
		sr.Save()
		return
	}

	start_backfill_from := backfill_date
	for {
		switch {
		case backfill:
			requests := backFillRequests(start_backfill_from)
			for _, request := range requests {
				request.Save()
			}

			start_backfill_from = requests[len(requests)-1].Updated_datetime // FIXME: is it safe to assume the items are sorted?

		case time.Since(worker.LastRunAt) > (30 * time.Second):
			// load requests from open311
			for _, request := range fetchRequests() {
				request.Save()
			}
			worker.LastRunAt = time.Now()
		default:
			log.Print("sleeping for 10 seconds")
			time.Sleep(10 * time.Second)
		}
	}
}

func (req Open311Request) String() string {
	// pretty print SR information
	return fmt.Sprintf("%s: %s at %s %f,%f, last update %s", req.Service_request_id, req.Service_name, req.Address, req.Lat, req.Long, req.Updated_datetime)
}

func (req Open311Request) Save() (persisted bool) {
	// create or update a SR

	// open311 says we should always ignore a SR that does not have a SR# assigned
	if req.Service_request_id == "" {
		log.Printf("cowardly refusing to create a new SR record because of empty SR#. Request type is %s", req.Service_name)
		return false
	}

	persisted = false

	// find existing record if exists
	var existing_id int
	err := worker.Db.QueryRow("SELECT id FROM service_requests WHERE service_request_id = $1", req.Service_request_id).Scan(&existing_id)
	switch {
	case err == sql.ErrNoRows:
		// log.Printf("did not find existing record %s", req.Service_request_id)
	case err != nil:
		// log.Print("error searching for existing SR", err)
	default:
		persisted = true
		// log.Printf("found existing sr %s", req.Service_request_id)
	}

	var stmt *sql.Stmt

	if !persisted {
		// create new record
		stmt, err = worker.Db.Prepare("INSERT INTO service_requests(service_request_id," +
			"status, service_name, service_code, agency_responsible, " +
			"address, requested_datetime, updated_datetime, lat, long," +
			"ward, police_district, media_url, channel, duplicate, parent_service_request_id, closed_datetime, notes) " +
			"VALUES ($1::varchar, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18); ")

		// "WHERE NOT EXISTS (SELECT 1 FROM service_requests WHERE service_request_id = $1);")

		if err != nil {
			log.Fatal("error preparing database insert statement", err)
		}

	} else {
		// update existing record
		stmt, err = worker.Db.Prepare("UPDATE service_requests SET " +
			"status = $2, service_name = $3, service_code = $4, agency_responsible = $5, " +
			"address = $6, requested_datetime = $7, updated_datetime = $8, lat = $9, long = $10," +
			"ward = $11, police_district = $12, media_url = $13, channel = $14, duplicate = $15, " +
			"parent_service_request_id = $16, updated_at = NOW(), closed_datetime = $17, notes = $18 WHERE service_request_id = $1;")

		if err != nil {
			log.Fatal("error preparing database update statement", err)
		}
	}

	tx, err := worker.Db.Begin()

	if err != nil {
		log.Fatal("error beginning transaction", err)
	}

	t := req.ExtractClosedDatetime()
	closed_time := pq.NullTime{Time: t, Valid: !t.IsZero()}
	notes_as_json, err := json.Marshal(req.Notes)
	if err != nil {
		log.Print("error marshaling notes to JSON: ", err)
	}
	_, err = tx.Stmt(stmt).Exec(req.Service_request_id,
		req.Status,
		req.Service_name,
		req.Service_code,
		req.Agency_responsible,
		req.Address,
		req.Requested_datetime,
		req.Updated_datetime,
		req.Lat,
		req.Long,
		req.Extended_attributes["ward"],
		req.Extended_attributes["police_district"],
		req.Media_url,
		req.Extended_attributes["channel"],
		req.Extended_attributes["duplicate"],
		req.Extended_attributes["parent_service_request_id"],
		closed_time,
		notes_as_json,
	)

	if err != nil {
		log.Fatalf("could not update %s because %s", req.Service_request_id, err)
	} else {
		var verb string
		switch {
		case !persisted && closed_time.Time.IsZero():
			verb = "CREATED"
		case !persisted && !closed_time.Time.IsZero():
			verb = "CREATED/CLOSED"
		case persisted && closed_time.Time.IsZero():
			verb = "UPDATED"
		case persisted && !closed_time.Time.IsZero():
			verb = "UPDATED/CLOSED"
		}

		log.Printf("[%s] %s", verb, req)
		persisted = true
	}

	err = tx.Commit()
	if err != nil {
		log.Fatal("error closing transaction", err)
	}

	return persisted

	// calculate closed time if necessary

}

func (req Open311Request) ExtractClosedDatetime() time.Time {
	// given an extended_attributes JSON blob, pluck out the closed time, if present
	// req.PrintNotes()

	var closed_at time.Time
	for _, note := range req.Notes {
		if note["type"] == "closed" {
			parsed_date, err := time.Parse("2006-01-02T15:04:05-07:00", note["datetime"].(string))
			if err != nil {
				log.Print("error parsing date", err)
			}
			log.Printf("SR %s closed at: %s", req, parsed_date)
			closed_at = parsed_date
		}
	}

	return closed_at
}

func (req Open311Request) PrintNotes() {
	fmt.Printf("Notes for SR %s:\n", req.Service_request_id)

	for _, note := range req.Notes {
		fmt.Printf("%+v\n", note)
	}
}

func fetchSingleRequest(sr_number string) (request Open311Request) {
	// given an SR, fetch the record
	log.Printf("fetching single SR %s", sr_number)
	open311_api_endpoint := fmt.Sprintf("http://311api.cityofchicago.org/open311/v2/requests/%s.json?extensions=true", sr_number)

	log.Printf("fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	var requests []Open311Request

	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("error parsing JSON:", err)
	}

	log.Printf("received %d requests from Open311", len(requests))

	return requests[0]
}

func fetchRequests() (requests []Open311Request) {
	last_updated_at := time.Now()
	if err := worker.Db.QueryRow("SELECT MAX(updated_datetime) FROM service_requests;").Scan(&last_updated_at); err != nil {
		log.Print("[fetchRequests] error loading most recent SR, will fallback to current time: ", err)
	}

	log.Print("[fetchRequests] most recent SR timestamp ", last_updated_at.Format(time.RFC3339))

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_after=" + last_updated_at.Format(time.RFC3339)

	log.Printf("[fetchRequests] fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("[fetchRequests] error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[fetchRequests] error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[fetchRequests] error parsing JSON:", err)
	}

	log.Printf("[fetchRequests] received %d requests from Open311", len(requests))

	return requests
}

func backFillRequests(start_from string) (requests []Open311Request) {
	var fetch_from time.Time

	if start_from == "" {
		err := worker.Db.QueryRow("SELECT updated_datetime FROM service_requests ORDER BY updated_datetime ASC LIMIT 1").Scan(&fetch_from)
		if err != nil {
			log.Println("error fetching oldest SR:", err)
		}
		log.Printf("no start_from value provided, so falling back to oldest (by last update) SR in the database: %s", fetch_from)
	} else {
		t, err := time.Parse(time.RFC3339, start_from)
		if err != nil {
			log.Fatal("[backfill] error parsing date to start from", err)
		}
		fetch_from = t
	}

	formatted_date_string_with_tz := fetch_from.Format(time.RFC3339)

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_before=" + formatted_date_string_with_tz

	log.Printf("[backfill] fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	defer resp.Body.Close()

	if err != nil {
		log.Fatalln("[backfill] error fetching from Open311 endpoint", err)
	}

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[backfill] error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[backfill] error parsing JSON:", err)
	}

	log.Printf("[backfill] received %d requests from Open311", len(requests))

	return requests
}
<|endoftext|>


<commit_before>// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build appengine

package handlers

import (
	"appengine"
	"appengine/datastore"
	"appengine/memcache"
	"appengine/taskqueue"
	"code.google.com/p/mlab-ns2/gae/ns/rtt"
	"net"
	"net/http"
	"net/url"
	"time"
)

func init() {
	http.HandleFunc(rtt.URLTaskImportDay, processTaskRTTImportDay)
	http.HandleFunc(rtt.URLTaskImportPut, processTaskRTTCGPut)
}

// addTaskRTTImportDay adds a BigQuery import task into taskqueue for a
// specified date.
func addTaskRTTImportDay(w http.ResponseWriter, r *http.Request, t time.Time) {
	c := appengine.NewContext(r)

	date := t.Format(rtt.DateFormat)

	c.Infof("handlers: Submitting BQ import task for %s", date)

	values := make(url.Values)
	values.Add(rtt.FormKeyImportDate, date)
	task := taskqueue.NewPOSTTask(rtt.URLTaskImportDay, values)
	_, err := taskqueue.Add(c, task, rtt.TaskQueueNameImport)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		c.Errorf("handlers.addTaskRTTImportDay:taskqueue.Add: %s", err)
		return
	}
}

// processTaskRTTImportDay processes a taskqueue task for an import of BigQuery
// data for a specified date.
func processTaskRTTImportDay(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	dateStr := r.FormValue(rtt.FormKeyImportDate)
	t, err := time.Parse(rtt.DateFormat, dateStr)
	if err != nil {
		// Don't return HTTP error since incorrect date cannot be fixed.
		c.Errorf("handlers.processTaskRTTImportDay:time.Parse: %s", err)
		return
	}

	rtt.BQImportDay(w, r, t)
}

// processTaskRTTCGPut processes a taskqueue task for the putting of new
// ClientGroups into datastore.
func processTaskRTTCGPut(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	// Get memcache key to use from POST parameters
	dataKey := r.FormValue(rtt.FormKeyPutKey)
	var data []rtt.ClientGroup
	_, err := memcache.Gob.Get(c, dataKey, &data)
	if err != nil {
		// Don't return HTTP error since nothing can be done if data
		// is missing or corrupt. Just log to GAE to see how often this
		// happens.
		c.Errorf("handlers.processTaskRTTCGPut:memcache.Get: %s", err)
		return
	}

	// Create lists of keys to use in datastore.PutMulti
	parentKey := rtt.DatastoreParentKey(c)
	keys := make([]*datastore.Key, 0, len(data))
	var key *datastore.Key
	for _, cg := range data {
		key = datastore.NewKey(c, "ClientGroup", net.IP(cg.Prefix).String(), 0, parentKey)
		keys = append(keys, key)
	}

	// Put data into datastore
	_, err = datastore.PutMulti(c, keys, data)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		c.Errorf("handlers.processTaskRTTCGPut:datastore.PutMulti: %s", err)
		return
	}

	// Remove cached CGs
	if err := memcache.Delete(c, dataKey); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		c.Errorf("handlers.processTaskRTTCGPut:memcache.Delte: %s", err)
		return
	}

	c.Infof("handlers: %d ClientGroups were successfully put into datastore.", len(data))

	// Get which date this import is for
	dateStr := r.FormValue(rtt.FormKeyImportDate)
	t, err := time.Parse(rtt.DateFormat, dateStr)
	if err != nil {
		// Don't return HTTP error since incorrect date cannot be fixed.
		c.Errorf("handlers.processTaskRTTCGPut:time.Parse: %s", err)
		return
	}
	rtt.UpdateLastSuccessfulImportDate(c, t)
}
<commit_msg>handlers: Show date in logs for put tasks<commit_after>// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build appengine

package handlers

import (
	"appengine"
	"appengine/datastore"
	"appengine/memcache"
	"appengine/taskqueue"
	"code.google.com/p/mlab-ns2/gae/ns/rtt"
	"net"
	"net/http"
	"net/url"
	"time"
)

func init() {
	http.HandleFunc(rtt.URLTaskImportDay, processTaskRTTImportDay)
	http.HandleFunc(rtt.URLTaskImportPut, processTaskRTTCGPut)
}

// addTaskRTTImportDay adds a BigQuery import task into taskqueue for a
// specified date.
func addTaskRTTImportDay(w http.ResponseWriter, r *http.Request, t time.Time) {
	c := appengine.NewContext(r)

	date := t.Format(rtt.DateFormat)

	c.Infof("handlers: Submitting BQ import task for %s", date)

	values := make(url.Values)
	values.Add(rtt.FormKeyImportDate, date)
	task := taskqueue.NewPOSTTask(rtt.URLTaskImportDay, values)
	_, err := taskqueue.Add(c, task, rtt.TaskQueueNameImport)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		c.Errorf("handlers.addTaskRTTImportDay:taskqueue.Add: %s", err)
		return
	}
}

// processTaskRTTImportDay processes a taskqueue task for an import of BigQuery
// data for a specified date.
func processTaskRTTImportDay(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	dateStr := r.FormValue(rtt.FormKeyImportDate)
	t, err := time.Parse(rtt.DateFormat, dateStr)
	if err != nil {
		// Don't return HTTP error since incorrect date cannot be fixed.
		c.Errorf("handlers.processTaskRTTImportDay:time.Parse: %s", err)
		return
	}

	rtt.BQImportDay(w, r, t)
}

// processTaskRTTCGPut processes a taskqueue task for the putting of new
// ClientGroups into datastore.
func processTaskRTTCGPut(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	// Get memcache key to use from POST parameters
	dataKey := r.FormValue(rtt.FormKeyPutKey)
	var data []rtt.ClientGroup
	_, err := memcache.Gob.Get(c, dataKey, &data)
	if err != nil {
		// Don't return HTTP error since nothing can be done if data
		// is missing or corrupt. Just log to GAE to see how often this
		// happens.
		c.Errorf("handlers.processTaskRTTCGPut:memcache.Get: %s", err)
		return
	}

	// Create lists of keys to use in datastore.PutMulti
	parentKey := rtt.DatastoreParentKey(c)
	keys := make([]*datastore.Key, 0, len(data))
	var key *datastore.Key
	for _, cg := range data {
		key = datastore.NewKey(c, "ClientGroup", net.IP(cg.Prefix).String(), 0, parentKey)
		keys = append(keys, key)
	}

	// Put data into datastore
	_, err = datastore.PutMulti(c, keys, data)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		c.Errorf("handlers.processTaskRTTCGPut:datastore.PutMulti: %s", err)
		return
	}

	// Remove cached CGs
	if err := memcache.Delete(c, dataKey); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		c.Errorf("handlers.processTaskRTTCGPut:memcache.Delte: %s", err)
		return
	}

	dateStr := r.FormValue(rtt.FormKeyImportDate)
	c.Infof("handlers: %d ClientGroups were successfully put into datastore. (%s)", len(data), dateStr)

	// Get which date this import is for
	t, err := time.Parse(rtt.DateFormat, dateStr)
	if err != nil {
		// Don't return HTTP error since incorrect date cannot be fixed.
		c.Errorf("handlers.processTaskRTTCGPut:time.Parse: %s", err)
		return
	}
	rtt.UpdateLastSuccessfulImportDate(c, t)
}
<|endoftext|>


<commit_before>package core

import (
	"bytes"
	"crypto/sha256"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"log"
	"net/http"

	"github.com/kristjank/ark-go/arkcoin"
	"github.com/kristjank/ark-go/arkcoin/base58"
)

//TransactionType to make it more readable - enum
type TransactionType byte

const (
	SENDARK         = 0
	SECONDSIGNATURE = 1
	CREATEDELEGATE  = 2
	VOTE            = 3
	MULTISIGNATURE  = 4
)

//Transaction struct - represents structure of ARK.io blockchain transaction
//It is used to post transaction to mainnet and to receive results from arkapi
//Empty fields are emmited by default
type Transaction struct {
	ID                    string            `json:"id,omitempty"`
	Timestamp             int32             `json:"timestamp,omitempty"`
	RecipientID           string            `json:"recipientId,omitempty"`
	Amount                int64             `json:"amount,omitempty"`
	Asset                 map[string]string `json:"asset,omitempty"`
	Fee                   int64             `json:"fee,omitempty"`
	Type                  byte              `json:"type"`
	VendorField           string            `json:"vendorField,omitempty"`
	Signature             string            `json:"signature,omitempty"`
	SignSignature         string            `json:"signSignature,omitempty"`
	SenderPublicKey       string            `json:"senderPublicKey,omitempty"`
	SecondSenderPublicKey string            `json:"secondSenderPublicKey,omitempty"`
	RequesterPublicKey    string            `json:"requesterPublicKey,omitempty"`
	Blockid               string            `json:"blockid,omitempty"`
	Height                int               `json:"height,omitempty"`
	SenderID              string            `json:"senderId,omitempty"`
	Confirmations         int               `json:"confirmations,omitempty"`
}

//FromBytes implementation - deserialization of recveived tx
func fromBytes(txbytes []byte) Transaction {
	txReader := bytes.NewReader(txbytes)

	tx := Transaction{}

	//Get Transaction Type
	binary.Read(txReader, binary.LittleEndian, &tx.Type)

	//GetTimeStamp from 	binary.Write(txBuf, binary.LittleEndian, uint32(tx.Timestamp))
	binary.Read(txReader, binary.LittleEndian, &tx.Timestamp)

	//Get SenderPublicKey
	byteKey := make([]byte, 33)
	binary.Read(txReader, binary.LittleEndian, byteKey)
	tx.SenderPublicKey = hex.EncodeToString(byteKey)

	return tx
}

//ToBytes returns bytearray of the Transaction object to be signed and send to blockchain
func (tx *Transaction) toBytes(skipSignature, skipSecondSignature bool) []byte {
	txBuf := new(bytes.Buffer)
	binary.Write(txBuf, binary.LittleEndian, tx.Type)
	binary.Write(txBuf, binary.LittleEndian, uint32(tx.Timestamp))

	binary.Write(txBuf, binary.LittleEndian, quickHexDecode(tx.SenderPublicKey))

	if tx.RequesterPublicKey != "" {
		res, err := base58.Decode(tx.RequesterPublicKey)
		if err != nil {
			binary.Write(txBuf, binary.LittleEndian, res)
		}
	}

	if tx.RecipientID != "" {
		res, err := base58.Decode(tx.RecipientID)
		if err != nil {
			log.Fatal("Error converting Decoding b58 ", err.Error())
		}
		binary.Write(txBuf, binary.LittleEndian, res)
	} else {
		binary.Write(txBuf, binary.LittleEndian, make([]byte, 21))
	}

	if tx.VendorField != "" {
		vendorBytes := []byte(tx.VendorField)
		if len(vendorBytes) < 65 {
			binary.Write(txBuf, binary.LittleEndian, vendorBytes)

			bs := make([]byte, 64-len(vendorBytes))
			binary.Write(txBuf, binary.LittleEndian, bs)
		}
	} else {
		binary.Write(txBuf, binary.LittleEndian, make([]byte, 64))
	}

	binary.Write(txBuf, binary.LittleEndian, uint64(tx.Amount))
	binary.Write(txBuf, binary.LittleEndian, uint64(tx.Fee))

	switch tx.Type {
	case SECONDSIGNATURE:
		binary.Write(txBuf, binary.LittleEndian, quickHexDecode(tx.Asset["signature"]))
	case CREATEDELEGATE:
		usernameBytes := []byte(tx.Asset["username"])
		binary.Write(txBuf, binary.LittleEndian, usernameBytes)
	case VOTE:
		voteBytes := []byte(tx.Asset["votes"])
		binary.Write(txBuf, binary.LittleEndian, voteBytes)
	}

	if !skipSignature && len(tx.Signature) > 0 {
		binary.Write(txBuf, binary.LittleEndian, quickHexDecode(tx.Signature))
	}

	if !skipSecondSignature && len(tx.SignSignature) > 0 {
		binary.Write(txBuf, binary.LittleEndian, quickHexDecode(tx.SignSignature))
	}

	return txBuf.Bytes()
}

//CreateTransaction creates and returns new Transaction struct...
func CreateTransaction(recipientID string, satoshiAmount int64, vendorField, passphrase, secondPassphrase string) *Transaction {
	tx := Transaction{
		Type:        SENDARK,
		RecipientID: recipientID,
		Amount:      satoshiAmount,
		Fee:         EnvironmentParams.Fees.Send,
		VendorField: vendorField,
	}

	tx.Timestamp = GetTime() //1
	tx.sign(passphrase)

	if len(secondPassphrase) > 0 {
		tx.secondSign(secondPassphrase)
	}

	tx.getID() //calculates id of transaction
	return &tx
}

//CreateVote transaction used to vote for a chosen Delegate
//if updown value = "+" vot is given to the specified PublicKey
//if updown value = "-" vot is taken from the specified PublicKey
func CreateVote(updown, delegatePubKey, passphrase, secondPassphrase string) *Transaction {
	tx := Transaction{
		Type:        VOTE,
		Fee:         EnvironmentParams.Fees.Vote,
		VendorField: "Delegate vote transaction",
		Asset:       make(map[string]string),
	}
	key := arkcoin.NewPrivateKeyFromPassword(passphrase, arkcoin.ActiveCoinConfig)
	tx.RecipientID = key.PublicKey.Address()

	tx.Asset["votes"] = updown + delegatePubKey
	tx.Timestamp = GetTime() //1
	tx.sign(passphrase)

	if len(secondPassphrase) > 0 {
		tx.secondSign(secondPassphrase)
	}

	tx.getID() //calculates id of transaction
	return &tx
}

//CreateDelegate creates and returns new Transaction struct...
func CreateDelegate(username, passphrase, secondPassphrase string) *Transaction {
	tx := Transaction{
		Type:        CREATEDELEGATE,
		Fee:         EnvironmentParams.Fees.Delegate,
		VendorField: "Create delegate tx",
		Asset:       make(map[string]string),
	}
	tx.Asset["username"] = username
	tx.Timestamp = GetTime() //1
	tx.sign(passphrase)

	if len(secondPassphrase) > 0 {
		tx.secondSign(secondPassphrase)
	}

	tx.getID() //calculates id of transaction
	return &tx
}

//CreateSecondSignature creates and returns new Transaction struct...
func CreateSecondSignature(passphrase, secondPassphrase string) *Transaction {
	tx := Transaction{
		Type:        SECONDSIGNATURE,
		Fee:         EnvironmentParams.Fees.SecondSignature,
		VendorField: "Create second signature",
		Asset:       make(map[string]string),
	}

	key := arkcoin.NewPrivateKeyFromPassword(secondPassphrase, arkcoin.ActiveCoinConfig)
	tx.Asset["signature"] = hex.EncodeToString(key.PublicKey.Serialize())
	tx.Timestamp = GetTime() //1
	tx.sign(passphrase)

	tx.getID() //calculates id of transaction
	return &tx
}

//Sign the Transaction
func (tx *Transaction) sign(passphrase string) {
	key := arkcoin.NewPrivateKeyFromPassword(passphrase, arkcoin.ActiveCoinConfig)

	tx.SenderPublicKey = hex.EncodeToString(key.PublicKey.Serialize())

	trHashBytes := sha256.New()
	trHashBytes.Write(tx.toBytes(true, true))

	sig, err := key.Sign(trHashBytes.Sum(nil))
	if err == nil {
		tx.Signature = hex.EncodeToString(sig)
	}
}

//SecondSign the Transaction
func (tx *Transaction) secondSign(passphrase string) {
	key := arkcoin.NewPrivateKeyFromPassword(passphrase, arkcoin.ActiveCoinConfig)

	tx.SecondSenderPublicKey = hex.EncodeToString(key.PublicKey.Serialize())
	trHashBytes := sha256.New()
	trHashBytes.Write(tx.toBytes(false, true))

	sig, err := key.Sign(trHashBytes.Sum(nil))
	if err == nil {
		tx.SignSignature = hex.EncodeToString(sig)
	}
}

//GetID returns calculated ID of trancation - hashed s256
func (tx *Transaction) getID() {
	trHashBytes := sha256.New()
	trHashBytes.Write(tx.toBytes(false, false))

	tx.ID = hex.EncodeToString(trHashBytes.Sum(nil))
}

//ToJSON converts transaction object to JSON string
func (tx *Transaction) ToJSON() string {
	txJSON, err := json.Marshal(tx)
	if err != nil {
		log.Fatal(err.Error())
	}
	return string(txJSON)
}

func quickHexDecode(data string) []byte {
	res, err := hex.DecodeString(data)
	if err != nil {
		log.Fatal(err.Error())
	}
	return res
}

//Verify function verifies if tx is validly signed
//if return == nill verification was succesfull
func (tx *Transaction) Verify() error {
	key, err := arkcoin.NewPublicKey(quickHexDecode(tx.SenderPublicKey), arkcoin.ActiveCoinConfig)
	if err != nil {
		log.Fatal(err.Error())
	}
	trHashBytes := sha256.New()
	trHashBytes.Write(tx.toBytes(true, true))
	return key.Verify(quickHexDecode(tx.Signature), trHashBytes.Sum(nil))

}

//SecondVerify function verifies if tx is validly signed
//if return == nill verification was succesfull
func (tx *Transaction) SecondVerify() error {
	key, err := arkcoin.NewPublicKey(quickHexDecode(tx.SecondSenderPublicKey), arkcoin.ActiveCoinConfig)
	if err != nil {
		log.Fatal(err.Error())
	}
	trHashBytes := sha256.New()
	trHashBytes.Write(tx.toBytes(false, true))
	return key.Verify(quickHexDecode(tx.SignSignature), trHashBytes.Sum(nil))
}

//PostTransactionResponse structure for call /peer/list
type PostTransactionResponse struct {
	Success        bool     `json:"success"`
	Message        string   `json:"message"`
	Error          string   `json:"error"`
	TransactionIDs []string `json:"transactionIds"`
}

//TransactionPayload - list of tx to send to network
type TransactionPayload struct {
	Transactions []*Transaction `json:"transactions"`
}

//TransactionQueryParams for returing filtered list of transactions
type TransactionQueryParams struct {
	ID          string          `url:"id,omitempty"`
	BlockID     string          `url:"blockId,omitempty"`
	SenderID    string          `url:"senderId,omitempty"`
	RecipientID string          `url:"recipientId,omitempty"`
	Limit       int             `url:"limit,omitempty"`
	Offset      int             `url:"offset,omitempty"`
	OrderBy     string          `url:"orderBy,omitempty"` //"Name of column to order. After column name must go 'desc' or 'asc' to choose order type, prefix for column name is t_. Example: orderBy=t_timestamp:desc (String)"
	Type        TransactionType `url:"type,omitempty"`
}

//TransactionResponse structure holds parsed jsong reply from ark-node
//when calling list methods the Transactions [] has results
//when calling get methods the transaction object (Single) has results
type TransactionResponse struct {
	Success           bool          `json:"success"`
	Transactions      []Transaction `json:"transactions"`
	SingleTransaction Transaction   `json:"transaction"`
	Count             string        `json:"count"`
	Error             string        `json:"error"`
}

//PostTransaction to selected ARKNetwork
func (s *ArkClient) PostTransaction(payload TransactionPayload) (PostTransactionResponse, *http.Response, error) {
	respTr := new(PostTransactionResponse)
	errTr := new(ArkApiResponseError)

	/*var payload transactionPayload
	payload.Transactions = append(payload.Transactions, tx)
	*/
	resp, err := s.sling.New().Post("peer/transactions").BodyJSON(payload).Receive(respTr, errTr)

	if err == nil {
		err = errTr
	}

	return *respTr, resp, err
}

//ListTransaction function returns list of peers from ArkNode
func (s *ArkClient) ListTransaction(params TransactionQueryParams) (TransactionResponse, *http.Response, error) {
	transactionResponse := new(TransactionResponse)
	transactionResponseErr := new(ArkApiResponseError)
	resp, err := s.sling.New().Get("api/transactions").QueryStruct(&params).Receive(transactionResponse, transactionResponseErr)
	if err == nil {
		err = transactionResponseErr
	}

	return *transactionResponse, resp, err
}

//ListTransactionUnconfirmed function returns list of peers from ArkNode
func (s *ArkClient) ListTransactionUnconfirmed(params TransactionQueryParams) (TransactionResponse, *http.Response, error) {
	transactionResponse := new(TransactionResponse)
	transactionResponseErr := new(ArkApiResponseError)
	resp, err := s.sling.New().Get("api/transactions/unconfirmed").QueryStruct(&params).Receive(transactionResponse, transactionResponseErr)
	if err == nil {
		err = transactionResponseErr
	}

	return *transactionResponse, resp, err
}

//GetTransaction function returns list of peers from ArkNode
func (s *ArkClient) GetTransaction(params TransactionQueryParams) (TransactionResponse, *http.Response, error) {
	transactionResponse := new(TransactionResponse)
	transactionResponseErr := new(ArkApiResponseError)
	resp, err := s.sling.New().Get("api/transactions/get").QueryStruct(&params).Receive(transactionResponse, transactionResponseErr)
	if err == nil {
		err = transactionResponseErr
	}

	return *transactionResponse, resp, err
}

//GetTransactionUnconfirmed function returns list of peers from ArkNode
func (s *ArkClient) GetTransactionUnconfirmed(params TransactionQueryParams) (TransactionResponse, *http.Response, error) {
	transactionResponse := new(TransactionResponse)
	transactionResponseErr := new(ArkApiResponseError)
	resp, err := s.sling.New().Get("api/transactions/unconfirmed/get").QueryStruct(&params).Receive(transactionResponse, transactionResponseErr)
	if err == nil {
		err = transactionResponseErr
	}

	return *transactionResponse, resp, err
}
<commit_msg>Added relay send method<commit_after>package core

import (
	"bytes"
	"crypto/sha256"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"log"
	"net/http"

	"github.com/kristjank/ark-go/arkcoin"
	"github.com/kristjank/ark-go/arkcoin/base58"
	"github.com/kristjank/goark-node/base/model"
)

//TransactionType to make it more readable - enum
type TransactionType byte

const (
	SENDARK         = 0
	SECONDSIGNATURE = 1
	CREATEDELEGATE  = 2
	VOTE            = 3
	MULTISIGNATURE  = 4
)

//Transaction struct - represents structure of ARK.io blockchain transaction
//It is used to post transaction to mainnet and to receive results from arkapi
//Empty fields are emmited by default
type Transaction struct {
	ID                    string            `json:"id,omitempty"`
	Timestamp             int32             `json:"timestamp,omitempty"`
	RecipientID           string            `json:"recipientId,omitempty"`
	Amount                int64             `json:"amount,omitempty"`
	Asset                 map[string]string `json:"asset,omitempty"`
	Fee                   int64             `json:"fee,omitempty"`
	Type                  byte              `json:"type"`
	VendorField           string            `json:"vendorField,omitempty"`
	Signature             string            `json:"signature,omitempty"`
	SignSignature         string            `json:"signSignature,omitempty"`
	SenderPublicKey       string            `json:"senderPublicKey,omitempty"`
	SecondSenderPublicKey string            `json:"secondSenderPublicKey,omitempty"`
	RequesterPublicKey    string            `json:"requesterPublicKey,omitempty"`
	Blockid               string            `json:"blockid,omitempty"`
	Height                int               `json:"height,omitempty"`
	SenderID              string            `json:"senderId,omitempty"`
	Confirmations         int               `json:"confirmations,omitempty"`
}

//FromBytes implementation - deserialization of recveived tx
func fromBytes(txbytes []byte) Transaction {
	txReader := bytes.NewReader(txbytes)

	tx := Transaction{}

	//Get Transaction Type
	binary.Read(txReader, binary.LittleEndian, &tx.Type)

	//GetTimeStamp from 	binary.Write(txBuf, binary.LittleEndian, uint32(tx.Timestamp))
	binary.Read(txReader, binary.LittleEndian, &tx.Timestamp)

	//Get SenderPublicKey
	byteKey := make([]byte, 33)
	binary.Read(txReader, binary.LittleEndian, byteKey)
	tx.SenderPublicKey = hex.EncodeToString(byteKey)

	return tx
}

//ToBytes returns bytearray of the Transaction object to be signed and send to blockchain
func (tx *Transaction) toBytes(skipSignature, skipSecondSignature bool) []byte {
	txBuf := new(bytes.Buffer)
	binary.Write(txBuf, binary.LittleEndian, tx.Type)
	binary.Write(txBuf, binary.LittleEndian, uint32(tx.Timestamp))

	binary.Write(txBuf, binary.LittleEndian, quickHexDecode(tx.SenderPublicKey))

	if tx.RequesterPublicKey != "" {
		res, err := base58.Decode(tx.RequesterPublicKey)
		if err != nil {
			binary.Write(txBuf, binary.LittleEndian, res)
		}
	}

	if tx.RecipientID != "" {
		res, err := base58.Decode(tx.RecipientID)
		if err != nil {
			log.Fatal("Error converting Decoding b58 ", err.Error())
		}
		binary.Write(txBuf, binary.LittleEndian, res)
	} else {
		binary.Write(txBuf, binary.LittleEndian, make([]byte, 21))
	}

	if tx.VendorField != "" {
		vendorBytes := []byte(tx.VendorField)
		if len(vendorBytes) < 65 {
			binary.Write(txBuf, binary.LittleEndian, vendorBytes)

			bs := make([]byte, 64-len(vendorBytes))
			binary.Write(txBuf, binary.LittleEndian, bs)
		}
	} else {
		binary.Write(txBuf, binary.LittleEndian, make([]byte, 64))
	}

	binary.Write(txBuf, binary.LittleEndian, uint64(tx.Amount))
	binary.Write(txBuf, binary.LittleEndian, uint64(tx.Fee))

	switch tx.Type {
	case SECONDSIGNATURE:
		binary.Write(txBuf, binary.LittleEndian, quickHexDecode(tx.Asset["signature"]))
	case CREATEDELEGATE:
		usernameBytes := []byte(tx.Asset["username"])
		binary.Write(txBuf, binary.LittleEndian, usernameBytes)
	case VOTE:
		voteBytes := []byte(tx.Asset["votes"])
		binary.Write(txBuf, binary.LittleEndian, voteBytes)
	}

	if !skipSignature && len(tx.Signature) > 0 {
		binary.Write(txBuf, binary.LittleEndian, quickHexDecode(tx.Signature))
	}

	if !skipSecondSignature && len(tx.SignSignature) > 0 {
		binary.Write(txBuf, binary.LittleEndian, quickHexDecode(tx.SignSignature))
	}

	return txBuf.Bytes()
}

//CreateTransaction creates and returns new Transaction struct...
func CreateTransaction(recipientID string, satoshiAmount int64, vendorField, passphrase, secondPassphrase string) *Transaction {
	tx := Transaction{
		Type:        SENDARK,
		RecipientID: recipientID,
		Amount:      satoshiAmount,
		Fee:         EnvironmentParams.Fees.Send,
		VendorField: vendorField,
	}

	tx.Timestamp = GetTime() //1
	tx.sign(passphrase)

	if len(secondPassphrase) > 0 {
		tx.secondSign(secondPassphrase)
	}

	tx.getID() //calculates id of transaction
	return &tx
}

//CreateVote transaction used to vote for a chosen Delegate
//if updown value = "+" vot is given to the specified PublicKey
//if updown value = "-" vot is taken from the specified PublicKey
func CreateVote(updown, delegatePubKey, passphrase, secondPassphrase string) *Transaction {
	tx := Transaction{
		Type:        VOTE,
		Fee:         EnvironmentParams.Fees.Vote,
		VendorField: "Delegate vote transaction",
		Asset:       make(map[string]string),
	}
	key := arkcoin.NewPrivateKeyFromPassword(passphrase, arkcoin.ActiveCoinConfig)
	tx.RecipientID = key.PublicKey.Address()

	tx.Asset["votes"] = updown + delegatePubKey
	tx.Timestamp = GetTime() //1
	tx.sign(passphrase)

	if len(secondPassphrase) > 0 {
		tx.secondSign(secondPassphrase)
	}

	tx.getID() //calculates id of transaction
	return &tx
}

//CreateDelegate creates and returns new Transaction struct...
func CreateDelegate(username, passphrase, secondPassphrase string) *Transaction {
	tx := Transaction{
		Type:        CREATEDELEGATE,
		Fee:         EnvironmentParams.Fees.Delegate,
		VendorField: "Create delegate tx",
		Asset:       make(map[string]string),
	}
	tx.Asset["username"] = username
	tx.Timestamp = GetTime() //1
	tx.sign(passphrase)

	if len(secondPassphrase) > 0 {
		tx.secondSign(secondPassphrase)
	}

	tx.getID() //calculates id of transaction
	return &tx
}

//CreateSecondSignature creates and returns new Transaction struct...
func CreateSecondSignature(passphrase, secondPassphrase string) *Transaction {
	tx := Transaction{
		Type:        SECONDSIGNATURE,
		Fee:         EnvironmentParams.Fees.SecondSignature,
		VendorField: "Create second signature",
		Asset:       make(map[string]string),
	}

	key := arkcoin.NewPrivateKeyFromPassword(secondPassphrase, arkcoin.ActiveCoinConfig)
	tx.Asset["signature"] = hex.EncodeToString(key.PublicKey.Serialize())
	tx.Timestamp = GetTime() //1
	tx.sign(passphrase)

	tx.getID() //calculates id of transaction
	return &tx
}

//Sign the Transaction
func (tx *Transaction) sign(passphrase string) {
	key := arkcoin.NewPrivateKeyFromPassword(passphrase, arkcoin.ActiveCoinConfig)

	tx.SenderPublicKey = hex.EncodeToString(key.PublicKey.Serialize())

	trHashBytes := sha256.New()
	trHashBytes.Write(tx.toBytes(true, true))

	sig, err := key.Sign(trHashBytes.Sum(nil))
	if err == nil {
		tx.Signature = hex.EncodeToString(sig)
	}
}

//SecondSign the Transaction
func (tx *Transaction) secondSign(passphrase string) {
	key := arkcoin.NewPrivateKeyFromPassword(passphrase, arkcoin.ActiveCoinConfig)

	tx.SecondSenderPublicKey = hex.EncodeToString(key.PublicKey.Serialize())
	trHashBytes := sha256.New()
	trHashBytes.Write(tx.toBytes(false, true))

	sig, err := key.Sign(trHashBytes.Sum(nil))
	if err == nil {
		tx.SignSignature = hex.EncodeToString(sig)
	}
}

//GetID returns calculated ID of trancation - hashed s256
func (tx *Transaction) getID() {
	trHashBytes := sha256.New()
	trHashBytes.Write(tx.toBytes(false, false))

	tx.ID = hex.EncodeToString(trHashBytes.Sum(nil))
}

//ToJSON converts transaction object to JSON string
func (tx *Transaction) ToJSON() string {
	txJSON, err := json.Marshal(tx)
	if err != nil {
		log.Fatal(err.Error())
	}
	return string(txJSON)
}

func quickHexDecode(data string) []byte {
	res, err := hex.DecodeString(data)
	if err != nil {
		log.Fatal(err.Error())
	}
	return res
}

//Verify function verifies if tx is validly signed
//if return == nill verification was succesfull
func (tx *Transaction) Verify() error {
	key, err := arkcoin.NewPublicKey(quickHexDecode(tx.SenderPublicKey), arkcoin.ActiveCoinConfig)
	if err != nil {
		log.Fatal(err.Error())
	}
	trHashBytes := sha256.New()
	trHashBytes.Write(tx.toBytes(true, true))
	return key.Verify(quickHexDecode(tx.Signature), trHashBytes.Sum(nil))

}

//SecondVerify function verifies if tx is validly signed
//if return == nill verification was succesfull
func (tx *Transaction) SecondVerify() error {
	key, err := arkcoin.NewPublicKey(quickHexDecode(tx.SecondSenderPublicKey), arkcoin.ActiveCoinConfig)
	if err != nil {
		log.Fatal(err.Error())
	}
	trHashBytes := sha256.New()
	trHashBytes.Write(tx.toBytes(false, true))
	return key.Verify(quickHexDecode(tx.SignSignature), trHashBytes.Sum(nil))
}

//PostTransactionResponse structure for call /peer/list
type PostTransactionResponse struct {
	Success        bool     `json:"success"`
	Message        string   `json:"message"`
	Error          string   `json:"error"`
	TransactionIDs []string `json:"transactionIds"`
}

//TransactionPayload - list of tx to send to network
type TransactionPayload struct {
	Transactions []*Transaction `json:"transactions"`
}

//TransactionQueryParams for returing filtered list of transactions
type TransactionQueryParams struct {
	ID          string          `url:"id,omitempty"`
	BlockID     string          `url:"blockId,omitempty"`
	SenderID    string          `url:"senderId,omitempty"`
	RecipientID string          `url:"recipientId,omitempty"`
	Limit       int             `url:"limit,omitempty"`
	Offset      int             `url:"offset,omitempty"`
	OrderBy     string          `url:"orderBy,omitempty"` //"Name of column to order. After column name must go 'desc' or 'asc' to choose order type, prefix for column name is t_. Example: orderBy=t_timestamp:desc (String)"
	Type        TransactionType `url:"type,omitempty"`
}

//TransactionResponse structure holds parsed jsong reply from ark-node
//when calling list methods the Transactions [] has results
//when calling get methods the transaction object (Single) has results
type TransactionResponse struct {
	Success           bool          `json:"success"`
	Transactions      []Transaction `json:"transactions"`
	SingleTransaction Transaction   `json:"transaction"`
	Count             string        `json:"count"`
	Error             string        `json:"error"`
}

//PostTransaction to selected ARKNetwork
func (s *ArkClient) PostTransaction(payload TransactionPayload) (PostTransactionResponse, *http.Response, error) {
	respTr := new(PostTransactionResponse)
	errTr := new(ArkApiResponseError)

	/*var payload transactionPayload
	payload.Transactions = append(payload.Transactions, tx)
	*/
	resp, err := s.sling.New().Post("peer/transactions").BodyJSON(payload).Receive(respTr, errTr)

	if err == nil {
		err = errTr
	}

	return *respTr, resp, err
}

//RelayNodeTransaction2Nodes to selected ARKNetwork
//different structure types - packagea
//Call only from GOARK-NODE
func (s *ArkClient) RelayNodeTransaction2Nodes(payload model.TransactionPayload) (model.PostTransactionResponse, *http.Response, error) {
	respTr := new(model.PostTransactionResponse)
	errTr := new(ArkApiResponseError)

	/*var payload transactionPayload
	payload.Transactions = append(payload.Transactions, tx)
	*/
	resp, err := s.sling.New().Post("peer/transactions").BodyJSON(payload).Receive(respTr, errTr)

	if err == nil {
		err = errTr
	}

	return *respTr, resp, err
}

//ListTransaction function returns list of peers from ArkNode
func (s *ArkClient) ListTransaction(params TransactionQueryParams) (TransactionResponse, *http.Response, error) {
	transactionResponse := new(TransactionResponse)
	transactionResponseErr := new(ArkApiResponseError)
	resp, err := s.sling.New().Get("api/transactions").QueryStruct(&params).Receive(transactionResponse, transactionResponseErr)
	if err == nil {
		err = transactionResponseErr
	}

	return *transactionResponse, resp, err
}

//ListTransactionUnconfirmed function returns list of peers from ArkNode
func (s *ArkClient) ListTransactionUnconfirmed(params TransactionQueryParams) (TransactionResponse, *http.Response, error) {
	transactionResponse := new(TransactionResponse)
	transactionResponseErr := new(ArkApiResponseError)
	resp, err := s.sling.New().Get("api/transactions/unconfirmed").QueryStruct(&params).Receive(transactionResponse, transactionResponseErr)
	if err == nil {
		err = transactionResponseErr
	}

	return *transactionResponse, resp, err
}

//GetTransaction function returns list of peers from ArkNode
func (s *ArkClient) GetTransaction(params TransactionQueryParams) (TransactionResponse, *http.Response, error) {
	transactionResponse := new(TransactionResponse)
	transactionResponseErr := new(ArkApiResponseError)
	resp, err := s.sling.New().Get("api/transactions/get").QueryStruct(&params).Receive(transactionResponse, transactionResponseErr)
	if err == nil {
		err = transactionResponseErr
	}

	return *transactionResponse, resp, err
}

//GetTransactionUnconfirmed function returns list of peers from ArkNode
func (s *ArkClient) GetTransactionUnconfirmed(params TransactionQueryParams) (TransactionResponse, *http.Response, error) {
	transactionResponse := new(TransactionResponse)
	transactionResponseErr := new(ArkApiResponseError)
	resp, err := s.sling.New().Get("api/transactions/unconfirmed/get").QueryStruct(&params).Receive(transactionResponse, transactionResponseErr)
	if err == nil {
		err = transactionResponseErr
	}

	return *transactionResponse, resp, err
}
<|endoftext|>


<commit_before>package wrappers

import (
	"bytes"
	"encoding/base64"
	"encoding/binary"
	"errors"
	"fmt"
	"time"
)

// TimeWrapper is a time.Time wrapper that provides xml marshaling and unmarshaling
type TimeWrapper struct {
	Formatted bool      // True for Kdbx v3.1 (formatted as RFC3339)
	Time      time.Time // Time value
}

// Limit of int64 for time.Add
const intLimit int64 = 9000000000

type TimeOption func(*TimeWrapper)

func WithKDBX4Formatting(t *TimeWrapper) {
	WithFormatted(false)(t)
}

func WithFormatted(formatted bool) TimeOption {
	return func(t *TimeWrapper) {
		t.Formatted = formatted
	}
}

// Now returns a TimeWrapper instance with the current time in UTC
func Now(options ...TimeOption) TimeWrapper {
	t := TimeWrapper{
		Formatted: true,
		Time:      time.Now().In(time.UTC),
	}

	for _, option := range options {
		option(&t)
	}

	return t
}

// MarshalText marshals time into an RFC3339 compliant value in UTC (Kdbx v3.1)
// On Kdbx v4 it calculates the timestamp subtracting seconds from the time date and encode it with base64
func (tw TimeWrapper) MarshalText() ([]byte, error) {
	t := time.Time(tw.Time).In(time.UTC)
	if y := t.Year(); y < 0 || y >= 10000 {
		return nil, ErrYearOutsideOfRange
	}

	var ret []byte
	if tw.Formatted {
		// Kdbx v3.1
		b := make([]byte, 0, len(time.RFC3339))
		ret = t.AppendFormat(b, time.RFC3339)
	} else {
		// Kdbx v4 - Count since year 1
		total := float64(0)
		zero := time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC)

		temp := t
		for {
			diff := temp.Sub(zero)
			if diff.Seconds() == 0 {
				break
			}

			total = total + diff.Seconds()
			temp = temp.Add(-diff)
		}

		buf := make([]byte, 8)
		binary.LittleEndian.PutUint64(buf, uint64(total))
		ret = make([]byte, base64.StdEncoding.EncodedLen(len(buf)))
		base64.StdEncoding.Encode(ret, buf)
	}
	return ret, nil
}

// UnmarshalText take a string of format time.RFC3339 and marshals it into the TimeWrapper value (Kdbx v3.1)
// On Kdbx v4 it calculates the time with given seconds via data byte array (base64 encoded)
func (tw *TimeWrapper) UnmarshalText(data []byte) error {
	var formatted bool
	// Check for RFC string (KDBX 3.1), if it fail try with KDBX 4
	t, err := time.Parse(time.RFC3339, string(data))
	if err != nil {
		// KDBX v4
		// In version 4 the time is a base64 timestamp of seconds passed since 1/1/0001
		var buf int64

		decoded := make([]byte, base64.StdEncoding.DecodedLen(len(data)))
		_, err = base64.StdEncoding.Decode(decoded, data)
		if err != nil {
			return err
		}
		if err := binary.Read(bytes.NewReader(decoded), binary.LittleEndian, &buf); err != nil {
			return err
		}

		// Count since year 1
		t = time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC)
		for {
			if buf < intLimit {
				t = t.Add(time.Duration(buf) * time.Second)
				break
			} else {
				t = t.Add(time.Duration(intLimit) * time.Second)
				buf -= intLimit
			}
		}
		formatted = false
	} else {
		formatted = true
	}
	*tw = TimeWrapper{
		Formatted: formatted,
		Time:      t,
	}
	return nil
}

func (tw TimeWrapper) String() string {
	return fmt.Sprintf(
		"Formatted: %v, Time: %v",
		tw.Formatted,
		tw.Time,
	)
}

// ErrYearOutsideOfRange is the error returned when the year is outside 0 and 9999
var ErrYearOutsideOfRange = errors.New("Wrappers.Time.MarshalText: year outside of range [0,9999]")
<commit_msg>Improve timewrapper documentation<commit_after>package wrappers

import (
	"bytes"
	"encoding/base64"
	"encoding/binary"
	"errors"
	"fmt"
	"time"
)

// TimeWrapper is a time.Time wrapper that provides xml marshaling and unmarshaling
type TimeWrapper struct {
	Formatted bool      // True for Kdbx v3.1 (formatted as RFC3339)
	Time      time.Time // Time value
}

// Limit of int64 for time.Add
// time.Duration allows for a maximum of ~290 years to be covered with 1 duration.
// https://golang.org/pkg/time/#Duration
// In order to cover the distance in seconds from 01.01.0001 to the timestamp, we have
// to go stetp by step.
const intLimit int64 = 9000000000

type TimeOption func(*TimeWrapper)

func WithKDBX4Formatting(t *TimeWrapper) {
	WithFormatted(false)(t)
}

func WithFormatted(formatted bool) TimeOption {
	return func(t *TimeWrapper) {
		t.Formatted = formatted
	}
}

// Now returns a TimeWrapper instance with the current time in UTC
func Now(options ...TimeOption) TimeWrapper {
	t := TimeWrapper{
		Formatted: true,
		Time:      time.Now().In(time.UTC),
	}

	for _, option := range options {
		option(&t)
	}

	return t
}

// MarshalText marshals time into an RFC3339 compliant value in UTC (Kdbx v3.1)
// On Kdbx v4 it calculates the timestamp subtracting seconds from the time date and encode it with base64
func (tw TimeWrapper) MarshalText() ([]byte, error) {
	t := time.Time(tw.Time).In(time.UTC)
	if y := t.Year(); y < 0 || y >= 10000 {
		return nil, ErrYearOutsideOfRange
	}

	var ret []byte
	if tw.Formatted {
		// Kdbx v3.1
		b := make([]byte, 0, len(time.RFC3339))
		ret = t.AppendFormat(b, time.RFC3339)
	} else {
		// Kdbx v4 - Count since year 1
		total := float64(0)
		// Uses a zero value instead of `intLimit` here because we need a `time.Duration` value
		// and therefore are using `(*time).Sub`
		zero := time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC)

		temp := t
		for {
			diff := temp.Sub(zero)
			if diff.Seconds() == 0 {
				break
			}

			total = total + diff.Seconds()
			temp = temp.Add(-diff)
		}

		buf := make([]byte, 8)
		binary.LittleEndian.PutUint64(buf, uint64(total))
		ret = make([]byte, base64.StdEncoding.EncodedLen(len(buf)))
		base64.StdEncoding.Encode(ret, buf)
	}
	return ret, nil
}

// UnmarshalText take a string of format time.RFC3339 and marshals it into the TimeWrapper value (Kdbx v3.1)
// On Kdbx v4 it calculates the time with given seconds via data byte array (base64 encoded)
func (tw *TimeWrapper) UnmarshalText(data []byte) error {
	var formatted bool
	// Check for RFC string (KDBX 3.1), if it fail try with KDBX 4
	t, err := time.Parse(time.RFC3339, string(data))
	if err != nil {
		// KDBX v4
		// In version 4 the time is a base64 timestamp of seconds passed since 1/1/0001
		var buf int64

		decoded := make([]byte, base64.StdEncoding.DecodedLen(len(data)))
		_, err = base64.StdEncoding.Decode(decoded, data)
		if err != nil {
			return err
		}
		if err := binary.Read(bytes.NewReader(decoded), binary.LittleEndian, &buf); err != nil {
			return err
		}

		// Count since year 1
		t = time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC)
		for {
			if buf < intLimit {
				t = t.Add(time.Duration(buf) * time.Second)
				break
			} else {
				t = t.Add(time.Duration(intLimit) * time.Second)
				buf -= intLimit
			}
		}
		formatted = false
	} else {
		formatted = true
	}
	*tw = TimeWrapper{
		Formatted: formatted,
		Time:      t,
	}
	return nil
}

func (tw TimeWrapper) String() string {
	return fmt.Sprintf(
		"Formatted: %v, Time: %v",
		tw.Formatted,
		tw.Time,
	)
}

// ErrYearOutsideOfRange is the error returned when the year is outside 0 and 9999
var ErrYearOutsideOfRange = errors.New("Wrappers.Time.MarshalText: year outside of range [0,9999]")
<|endoftext|>


<commit_before>package gumble_ffmpeg

import (
	"encoding/binary"
	"errors"
	"io"
	"os/exec"
	"strconv"
	"time"

	"github.com/layeh/gumble/gumble"
)

type Stream struct {
	Done func()

	client     *gumble.Client
	cmd        *exec.Cmd
	pipe       io.ReadCloser
	sourceStop chan bool
	volume     float32
}

func New(client *gumble.Client) (*Stream, error) {
	stream := &Stream{
		client: client,
		volume: 1.0,
	}
	return stream, nil
}

func (s *Stream) Play(file string) error {
	if s.sourceStop != nil {
		return errors.New("already playing")
	}
	s.cmd = exec.Command("ffmpeg", "-i", file, "-ac", "1", "-ar", strconv.Itoa(gumble.AudioSampleRate), "-f", "s16le", "-")
	if pipe, err := s.cmd.StdoutPipe(); err != nil {
		s.cmd = nil
		return nil
	} else {
		s.pipe = pipe
	}
	s.sourceStop = make(chan bool)
	go s.sourceRoutine()
	s.cmd.Start()
	return nil
}

func (s *Stream) IsPlaying() bool {
	return s.sourceStop != nil
}

func (s *Stream) Stop() error {
	if s.sourceStop != nil {
		close(s.sourceStop)
	}
	return nil
}

func (s *Stream) Volume() float32 {
	return s.volume
}

func (s *Stream) SetVolume(volume float32) {
	s.volume = volume
}

func (s *Stream) sourceRoutine() {
	ticker := time.NewTicker(10 * time.Millisecond)
	defer ticker.Stop()

	defer func() {
		s.cmd.Process.Kill()
		s.cmd.Wait()
		s.cmd = nil
		s.sourceStop = nil
		if done := s.Done; done != nil {
			done()
		}
	}()

	stop := s.sourceStop
	int16Buffer := make([]int16, gumble.AudioDefaultFrameSize)
	byteBuffer := make([]byte, gumble.AudioDefaultFrameSize*2)

	for {
		select {
		case <-stop:
			return
		case <-ticker.C:
			if _, err := io.ReadFull(s.pipe, byteBuffer); err != nil {
				return
			}
			for i := range int16Buffer {
				float := float32(int16(binary.LittleEndian.Uint16(byteBuffer[i*2 : (i+1)*2])))
				int16Buffer[i] = int16(s.volume * float)
			}
			s.client.Send(gumble.AudioBuffer(int16Buffer))
		}
	}
}
<commit_msg>gumble_ffmpeg: properly return errors from Stream.Play<commit_after>package gumble_ffmpeg

import (
	"encoding/binary"
	"errors"
	"io"
	"os/exec"
	"strconv"
	"time"

	"github.com/layeh/gumble/gumble"
)

type Stream struct {
	Done func()

	client     *gumble.Client
	cmd        *exec.Cmd
	pipe       io.ReadCloser
	sourceStop chan bool
	volume     float32
}

func New(client *gumble.Client) (*Stream, error) {
	stream := &Stream{
		client: client,
		volume: 1.0,
	}
	return stream, nil
}

func (s *Stream) Play(file string) error {
	if s.sourceStop != nil {
		return errors.New("already playing")
	}
	s.cmd = exec.Command("ffmpeg", "-i", file, "-ac", "1", "-ar", strconv.Itoa(gumble.AudioSampleRate), "-f", "s16le", "-")
	if pipe, err := s.cmd.StdoutPipe(); err != nil {
		s.cmd = nil
		return err
	} else {
		s.pipe = pipe
	}
	if err := s.cmd.Start(); err != nil {
		return err
	}
	s.sourceStop = make(chan bool)
	go s.sourceRoutine()
	return nil
}

func (s *Stream) IsPlaying() bool {
	return s.sourceStop != nil
}

func (s *Stream) Stop() error {
	if s.sourceStop != nil {
		close(s.sourceStop)
	}
	return nil
}

func (s *Stream) Volume() float32 {
	return s.volume
}

func (s *Stream) SetVolume(volume float32) {
	s.volume = volume
}

func (s *Stream) sourceRoutine() {
	ticker := time.NewTicker(10 * time.Millisecond)
	defer ticker.Stop()

	defer func() {
		s.cmd.Process.Kill()
		s.cmd.Wait()
		s.cmd = nil
		s.sourceStop = nil
		if done := s.Done; done != nil {
			done()
		}
	}()

	stop := s.sourceStop
	int16Buffer := make([]int16, gumble.AudioDefaultFrameSize)
	byteBuffer := make([]byte, gumble.AudioDefaultFrameSize*2)

	for {
		select {
		case <-stop:
			return
		case <-ticker.C:
			if _, err := io.ReadFull(s.pipe, byteBuffer); err != nil {
				return
			}
			for i := range int16Buffer {
				float := float32(int16(binary.LittleEndian.Uint16(byteBuffer[i*2 : (i+1)*2])))
				int16Buffer[i] = int16(s.volume * float)
			}
			s.client.Send(gumble.AudioBuffer(int16Buffer))
		}
	}
}
<|endoftext|>


<commit_before>package util

import (
	"fmt"
	"github.com/Shopify/go-lua"
)

// DeepPush will put any basic Go type on the lua stack. If the value
// contains a map or a slice, it will recursively push those values as
// tables on the Lua stack.
//
// Supported types are:
//
//    | Go                       | Lua
//    |-------------------------------------------------------------------------
//    | nil                      | nil
//    | bool                     | bool
//    | string                   | string
//    | any int                  | number (float64)
//    | any float                | number (float64)
//    | any complex              | number (real value as float64)
//    |------------------------------------------------------------------------
//    | map[string]interface{}   | table, child `interface{}` recursively
//    |                          | resolved if childs are of a type
//    |                          | in the first part of this table
//    |------------------------------------------------------------------------
//
// For string, int, float64, interface{}.
//
//    | Go                       | Lua
//    |-------------------------------------------------------------------------
//    | []                       | table with array properties
//    | [][]                     | 2d table with array properties
//    | [][][]                   | 3d table with array properties
//
// ...where interface{} is not resolved past a fourth level.  This means
//
//    var arr [][][]interface{} = {{{"string", 1, 0.65, true}, {}}, {{}, {}}}
//
// ...will give a Lua table such as:
//
//    {{{string, number, number, true}, {}}, {{}, {}}}
//
// ...but this will not work:
//
//    var arr [][][][]interface{} = {{{{"string", 1, 0.65, true}, {}}, {{}, {}}}}
//
// ... because the fourth level is not of a basic Go type in the first table.
func DeepPush(l *lua.State, v interface{}) int {
	forwardOnType(l, v)
	return 1
}

func forwardOnType(l *lua.State, val interface{}) {

	switch val := val.(type) {
	case nil:
		lua.PushNil(l)

	case bool:
		lua.PushBoolean(l, val)

	case string:
		lua.PushString(l, val)

	case uint8:
		lua.PushNumber(l, float64(val))
	case uint16:
		lua.PushNumber(l, float64(val))
	case uint32:
		lua.PushNumber(l, float64(val))
	case uint64:
		lua.PushNumber(l, float64(val))
	case uint:
		lua.PushNumber(l, float64(val))

	case int8:
		lua.PushNumber(l, float64(val))
	case int16:
		lua.PushNumber(l, float64(val))
	case int32:
		lua.PushNumber(l, float64(val))
	case int64:
		lua.PushNumber(l, float64(val))
	case int:
		lua.PushNumber(l, float64(val))

	case float32:
		lua.PushNumber(l, float64(val))
	case float64:
		lua.PushNumber(l, val)

	case complex64:
		forwardOnType(l, []float32{real(val), imag(val)})
	case complex128:
		forwardOnType(l, []float64{real(val), imag(val)})

	case map[string]interface{}:
		lua.CreateTable(l, 0, len(val))
		recurseOnMap(l, val)

	case []int:
		recurseOnFuncArray(l, func(i int) interface{} { return val[i] }, len(val))
	case [][]int:
		recurseOnFuncArray(l, func(i int) interface{} { return val[i] }, len(val))
	case [][][]int:
		recurseOnFuncArray(l, func(i int) interface{} { return val[i] }, len(val))

	case []float64:
		recurseOnFuncArray(l, func(i int) interface{} { return val[i] }, len(val))
	case [][]float64:
		recurseOnFuncArray(l, func(i int) interface{} { return val[i] }, len(val))
	case [][][]float64:
		recurseOnFuncArray(l, func(i int) interface{} { return val[i] }, len(val))

	case []string:
		recurseOnFuncArray(l, func(i int) interface{} { return val[i] }, len(val))
	case [][]string:
		recurseOnFuncArray(l, func(i int) interface{} { return val[i] }, len(val))
	case [][][]string:
		recurseOnFuncArray(l, func(i int) interface{} { return val[i] }, len(val))

	case []interface{}:
		recurseOnFuncArray(l, func(i int) interface{} { return val[i] }, len(val))
	case [][]interface{}:
		recurseOnFuncArray(l, func(i int) interface{} { return val[i] }, len(val))
	case [][][]interface{}:
		recurseOnFuncArray(l, func(i int) interface{} { return val[i] }, len(val))

	default:
		lua.Errorf(l, fmt.Sprintf("contains unsupported type: %T", val))
		panic("unreachable")
	}
}

func recurseOnMap(l *lua.State, input map[string]interface{}) {
	// -1 is a table
	for key, val := range input {
		forwardOnType(l, key)
		forwardOnType(l, val)
		// -1: something, -2: key, -3: table
		lua.RawSet(l, -3)
	}
}

// the hack of using a func(int)interface{} makes it that it is valid for any
// combination of []something
func recurseOnFuncArray(l *lua.State, input func(int) interface{}, n int) {
	lua.CreateTable(l, n, 0)
	for i := 0; i < n; i++ {
		forwardOnType(l, input(i))
		lua.RawSetInt(l, -2, i+1)
	}
}
<commit_msg>Do the `reflect` dance.<commit_after>package util

import (
	"fmt"
	"github.com/Shopify/go-lua"
	"reflect"
)

// DeepPush will put any basic Go type on the lua stack. If the value
// contains a map or a slice, it will recursively push those values as
// tables on the Lua stack.
//
// Supported types are:
//
//    | Go                       | Lua
//    |-------------------------------------------------------------------------
//    | nil                      | nil
//    | bool                     | bool
//    | string                   | string
//    | any int                  | number (float64)
//    | any float                | number (float64)
//    | any complex              | number (real value as float64)
//    |                          |
//    | map[t]t                  | table, key and val `t` recursively
//    |                          | resolved
//    |                          |
//    | []t                      | table with array properties, with `t`
//    |                          | values recursively resolved
func DeepPush(l *lua.State, v interface{}) int {
	forwardOnType(l, v)
	return 1
}

func forwardOnType(l *lua.State, val interface{}) {

	switch val := val.(type) {
	case nil:
		lua.PushNil(l)

	case bool:
		lua.PushBoolean(l, val)

	case string:
		lua.PushString(l, val)

	case uint8:
		lua.PushNumber(l, float64(val))
	case uint16:
		lua.PushNumber(l, float64(val))
	case uint32:
		lua.PushNumber(l, float64(val))
	case uint64:
		lua.PushNumber(l, float64(val))
	case uint:
		lua.PushNumber(l, float64(val))

	case int8:
		lua.PushNumber(l, float64(val))
	case int16:
		lua.PushNumber(l, float64(val))
	case int32:
		lua.PushNumber(l, float64(val))
	case int64:
		lua.PushNumber(l, float64(val))
	case int:
		lua.PushNumber(l, float64(val))

	case float32:
		lua.PushNumber(l, float64(val))
	case float64:
		lua.PushNumber(l, val)

	case complex64:
		forwardOnType(l, []float32{real(val), imag(val)})
	case complex128:
		forwardOnType(l, []float64{real(val), imag(val)})

	default:
		forwardOnReflect(l, val)
	}
}

func forwardOnReflect(l *lua.State, val interface{}) {

	switch v := reflect.ValueOf(val); v.Kind() {

	case reflect.Array:
		recurseOnFuncSlice(l, func(i int) interface{} { return v.Index(i).Interface() }, v.Len())

	case reflect.Slice:
		recurseOnFuncSlice(l, func(i int) interface{} { return v.Index(i).Interface() }, v.Len())

	case reflect.Map:
		lua.CreateTable(l, 0, v.Len())
		for _, key := range v.MapKeys() {
			mapKey := key.Interface()
			mapVal := v.MapIndex(key).Interface()
			forwardOnType(l, mapKey)
			forwardOnType(l, mapVal)
			lua.RawSet(l, -3)
		}

	default:
		lua.Errorf(l, fmt.Sprintf("contains unsupported type: %T", val))
		panic("unreachable")
	}

}

// the hack of using a func(int)interface{} makes it that it is valid for any
// type of slice
func recurseOnFuncSlice(l *lua.State, input func(int) interface{}, n int) {
	lua.CreateTable(l, n, 0)
	for i := 0; i < n; i++ {
		forwardOnType(l, input(i))
		lua.RawSetInt(l, -2, i+1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The casbin Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package util

import (
	"log"
	"testing"
)

func testFixAttribute(t *testing.T, s string, res string) {
	myRes := FixAttribute(s)
	log.Printf("%s: %s", s, myRes)

	if myRes != res {
		t.Errorf("%s: %s, supposed to be %s", s, myRes, res)
	}
}

func TestFixAttribute(t *testing.T) {
	testFixAttribute(t, "r.sub.domain", "subAttr(r.sub, \"domain\")")
	testFixAttribute(t, "r.obj.role", "objAttr(r.obj, \"role\")")
	testFixAttribute(t, "r.act.url", "actAttr(r.act, \"url\")")
	testFixAttribute(t, "p.sub.domain", "subAttr(p.sub, \"domain\")")
	testFixAttribute(t, "p.obj.role", "objAttr(p.obj, \"role\")")
	testFixAttribute(t, "p.act.url", "actAttr(p.act, \"url\")")
}
<commit_msg>Add tests for util.<commit_after>// Copyright 2017 The casbin Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package util

import (
	"log"
	"testing"
)

func testFixAttribute(t *testing.T, s string, res string) {
	myRes := FixAttribute(s)
	log.Printf("%s: %s", s, myRes)

	if myRes != res {
		t.Errorf("%s: %s, supposed to be %s", s, myRes, res)
	}
}

func TestFixAttribute(t *testing.T) {
	testFixAttribute(t, "r.sub.domain", "subAttr(r.sub, \"domain\")")
	testFixAttribute(t, "r.obj.role", "objAttr(r.obj, \"role\")")
	testFixAttribute(t, "r.act.url", "actAttr(r.act, \"url\")")
	testFixAttribute(t, "p.sub.domain", "subAttr(p.sub, \"domain\")")
	testFixAttribute(t, "p.obj.role", "objAttr(p.obj, \"role\")")
	testFixAttribute(t, "p.act.url", "actAttr(p.act, \"url\")")
}

func testArrayEquals(t *testing.T, a []string, b []string, res bool) {
	myRes := ArrayEquals(a, b)
	log.Printf("%s == %s: %t", a, b, myRes)

	if myRes != res {
		t.Errorf("%s == %s: %t, supposed to be %t", a, b, myRes, res)
	}
}

func TestArrayEquals(t *testing.T) {
	testArrayEquals(t, []string{"a", "b", "c"}, []string{"a", "b", "c"}, true)
	testArrayEquals(t, []string{"a", "b", "c"}, []string{"a", "b"}, false)
	testArrayEquals(t, []string{"a", "b", "c"}, []string{"a", "c", "b"}, false)
	testArrayEquals(t, []string{"a", "b", "c"}, []string{}, false)
}

func testArray2DEquals(t *testing.T, a [][]string, b [][]string, res bool) {
	myRes := Array2DEquals(a, b)
	log.Printf("%s == %s: %t", a, b, myRes)

	if myRes != res {
		t.Errorf("%s == %s: %t, supposed to be %t", a, b, myRes, res)
	}
}

func TestArray2DEquals(t *testing.T) {
	testArray2DEquals(t, [][]string{{"a", "b", "c"}, {"1", "2", "3"}}, [][]string{{"a", "b", "c"}, {"1", "2", "3"}}, true)
	testArray2DEquals(t, [][]string{{"a", "b", "c"}, {"1", "2", "3"}}, [][]string{{"a", "b", "c"}}, false)
	testArray2DEquals(t, [][]string{{"a", "b", "c"}, {"1", "2", "3"}}, [][]string{{"a", "b", "c"}, {"1", "2"}}, false)
	testArray2DEquals(t, [][]string{{"a", "b", "c"}, {"1", "2", "3"}}, [][]string{{"1", "2", "3"}, {"a", "b", "c"}}, false)
	testArray2DEquals(t, [][]string{{"a", "b", "c"}, {"1", "2", "3"}}, [][]string{}, false)
}

func testSetEquals(t *testing.T, a []string, b []string, res bool) {
	myRes := SetEquals(a, b)
	log.Printf("%s == %s: %t", a, b, myRes)

	if myRes != res {
		t.Errorf("%s == %s: %t, supposed to be %t", a, b, myRes, res)
	}
}

func TestSetEquals(t *testing.T) {
	testSetEquals(t, []string{"a", "b", "c"}, []string{"a", "b", "c"}, true)
	testSetEquals(t, []string{"a", "b", "c"}, []string{"a", "b"}, false)
	testSetEquals(t, []string{"a", "b", "c"}, []string{"a", "c", "b"}, true)
	testSetEquals(t, []string{"a", "b", "c"}, []string{}, false)
}
<|endoftext|>


<commit_before>package coremain

const (
	coreName    = "CoreDNS"
	coreVersion = "009"

	serverType = "dns"
)
<commit_msg>Release 010<commit_after>package coremain

const (
	coreName    = "CoreDNS"
	coreVersion = "010"

	serverType = "dns"
)
<|endoftext|>


<commit_before>package coremain

// Various CoreDNS constants.
const (
	CoreVersion = "1.8.4"
	coreName    = "CoreDNS"
	serverType  = "dns"
)
<commit_msg>Update version string to 1.8.5 (#4855)<commit_after>package coremain

// Various CoreDNS constants.
const (
	CoreVersion = "1.8.5"
	coreName    = "CoreDNS"
	serverType  = "dns"
)
<|endoftext|>