<commit_before>package main

import (
	"encoding/csv"
	"fmt"
	"strconv"
	"log"
	"os"
	"strings"
	"time"
)

const (
	SELLWIRE_TIMESTAMP_FORMAT = "2006-01-02 15:04:05"
	SELLWIRE_TRANSACTION_COLUMN_STATUS=16
	SELLWIRE_TRANSACTION_COLUMN_TRANSACTION_ID=2
	SELLWIRE_TRANSACTION_COLUMN_TIMESTAMP=12
	SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_NAME=3
	SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_TAX=6
	SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_AMOUNT=7
	SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_IS_EU=9
	SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_COUNTRY_CODE=10
	SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_TAX_NUMBER=11
	PAYPAL_DATE_OUTPUT_FORMAT = "02.01.2006"
	STRIPE_TRANSFER_DATE_FORMAT = "2006-01-02 15:04"
	STRIPE_TRANSFER_COLUMN_STATUS=3
	STRIPE_TRANSFER_COLUMN_DATE=0
	STRIPE_TRANSFER_COLUMN_TRANSFER_ID=1
	STRIPE_TRANSFER_COLUMND_AMOUNT=5
	STRIPE_PAYMENT_COLUMN_PAYMENT_ID=0
	STRIPE_PAYMENT_COLUMN_PAYMENT_DATE=2
	STRIPE_PAYMENT_COLUMN_TRANSFER_ID=45
	STRIPE_PAYMENT_COLUMN_STATUS=12
)

type TransactionType string

const (
	TransactionTypePaypal TransactionType = "paypal"
	TransactionTypeStripe TransactionType = "stripe"
)

type Amount struct {
	Dollars int64
	Cents int64
}

func (a Amount) IsZero() bool {
	return a.Dollars == 0 && a.Cents == 0
}

func (a Amount) ToStringGermany() string {
	if a.Dollars == 0 && a.Cents == 0 {
		return ""
	}
	return fmt.Sprintf("%d,%02d",a.Dollars, a.Cents)
}

// VAT % of total amount (including VAT)
func (a Amount) VATPercentOfAsStringGermany(other Amount) string {
	totalCentsA := a.Dollars * 100 + a.Cents
	totalCentsB := other.Dollars * 100 + other.Cents - totalCentsA
	percentage := float64(totalCentsA) * 100.0 / float64(totalCentsB)
	percentageRounded := int64(percentage + 0.5)
	if percentageRounded == 0 {
		return ""
	}
	return fmt.Sprintf("%d%%", percentageRounded)
}

type SellwireTransaction struct {
	TransactionType TransactionType
	TransactionId string
	Timestamp time.Time
	CustomerName string
	Amount Amount
	TaxAmount Amount
	IsEU bool
	IsPrivate bool
	CountryCode string
	TaxNumber string
	IsRefund bool
}

type StripeTransfer struct {
	TransferId string
	Date time.Time
	Amount Amount
	Status string
}

var transactions []SellwireTransaction
var stripeTransfersByTransactionId map[string]StripeTransfer
var vatCorrections map[string]string // from broken number to corrected number (including country prefix)

func main() {
	importVatCorrections()
	importSellwireTransactions()
	importStripeTransferMap()
	outputPaypalTransactions()
	outputStripeTransactions()
}

func importSellwireTransactions() {
	sellwireOrdersFile, err := os.Open("input/SellwireOrders.csv")
	if err != nil {
		log.Fatal(err)
	}
	r := csv.NewReader(sellwireOrdersFile)

	records, err := r.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	for _, record := range records[1:] {
		status := record[SELLWIRE_TRANSACTION_COLUMN_STATUS]
		if status != "complete" && status != "refunded" {
			continue
		}
		timestampStr := record[SELLWIRE_TRANSACTION_COLUMN_TIMESTAMP]
		timestamp, err := time.Parse(SELLWIRE_TIMESTAMP_FORMAT, timestampStr)
		if err != nil {
			log.Fatal(err)
		}

		countryCode := record[SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_COUNTRY_CODE]
		if countryCode == "" {
			countryCode = "US"
		}
		taxNumber := strings.ToUpper(record[SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_TAX_NUMBER])

		if taxNumber != "" && !strings.HasPrefix(taxNumber, countryCode) {
			taxNumber = fmt.Sprintf("%s%s", countryCode, taxNumber)
		}

		if correction, ok := vatCorrections[taxNumber]; ok {
			taxNumber = correction
		}

		isEU, _ := strconv.ParseBool(record[SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_IS_EU])

		amount := parseAmount(record[SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_AMOUNT])
		taxAmount := parseAmount(record[SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_TAX])

		transactionId := record[SELLWIRE_TRANSACTION_COLUMN_TRANSACTION_ID]

		transactionType := TransactionTypePaypal
		if strings.HasPrefix(transactionId, "ch_") {
			transactionType = TransactionTypeStripe
		}

		if amount.IsZero() {
			continue
		}
		sellwireRecord := SellwireTransaction{
			TransactionType: transactionType,
			TransactionId: transactionId,
			Timestamp: timestamp,
			CustomerName: strings.Title(strings.ToLower(record[SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_NAME])),
			Amount: amount,
			TaxAmount: taxAmount,
			IsEU: isEU,
			IsPrivate: isEU && taxNumber == "",
			CountryCode: countryCode,
			TaxNumber: taxNumber,
			IsRefund: status == "refunded",
		}

		transactions = append(transactions, sellwireRecord)
	}
}

func parseAmount(amountStr string) Amount {
			var amount Amount
			amountStrStripped := strings.Replace(amountStr, ",", "", -1)
			amountParts := strings.Split(amountStrStripped, ".")
			if len(amountParts) > 2 {
				log.Fatalf("Invalid amount found: %s", amountStr)
			}
			if len(amountParts) > 0 {
				dollars, err := strconv.ParseInt(amountParts[0], 10, 64)
				if err != nil {
					log.Fatal(err)
				}
				amount.Dollars = dollars
				if len(amountParts) > 1 {
					cents, err  := strconv.ParseInt(amountParts[1], 10, 64)
					if cents < 10 { // 9.8 is 9 dollar 80 cents and not 9 dollar 8 cents
						cents *= 10
					}
					if err != nil {
						log.Fatal(err)
					}
					amount.Cents = cents
				}
			}
			return amount
		}

func importStripeTransferMap() {
	stripeTransfersByTransferId := make(map[string]StripeTransfer)

	stripeTransfersFile, err := os.Open("input/payouts.csv")
	if err != nil {
		log.Fatal(err)
	}
	r := csv.NewReader(stripeTransfersFile)

	transferRecords, err := r.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	for _, record := range transferRecords[1:] {
		status := record[STRIPE_TRANSFER_COLUMN_STATUS]

		dateStr := record[STRIPE_TRANSFER_COLUMN_DATE]
		date, err := time.Parse(STRIPE_TRANSFER_DATE_FORMAT, dateStr)
		if err != nil {
			log.Fatal(err)
		}

		var amount Amount
		amountStr := record[STRIPE_TRANSFER_COLUMND_AMOUNT]
		amountStrStripped := strings.Replace(amountStr, ".", "", -1)
		amountParts := strings.Split(amountStrStripped, ",")
		if len(amountParts) > 2 {
			log.Fatalf("Invalid amount found: %s", amountStr)
		}
		if len(amountParts) > 0 {
			dollars, err := strconv.ParseInt(amountParts[0], 10, 64)
			if err != nil {
				log.Fatal(err)
			}
			amount.Dollars = dollars
			if len(amountParts) > 1 {
				cents, err  := strconv.ParseInt(amountParts[1], 10, 64)
				if err != nil {
					log.Fatal(err)
				}
				amount.Cents = cents
			}
		}

		transferId := record[STRIPE_TRANSFER_COLUMN_TRANSFER_ID]

		transferRecord := StripeTransfer{
			TransferId: transferId,
			Date: date,
			Amount: amount,
			Status: status,
		}

		stripeTransfersByTransferId[transferId] = transferRecord
	}

	stripePaymentsFile, err := os.Open("input/payments.csv")
	if err != nil {
		log.Fatal(err)
	}
	r2 := csv.NewReader(stripePaymentsFile)

	paymentRecords, err := r2.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	stripeTransfersByTransactionId = make(map[string]StripeTransfer)

	for _, record := range paymentRecords[1:] {
		paymentId := record[STRIPE_PAYMENT_COLUMN_PAYMENT_ID]
		transferId := record[STRIPE_PAYMENT_COLUMN_TRANSFER_ID]
		status := record[STRIPE_PAYMENT_COLUMN_STATUS]
		paymentDate := record[STRIPE_PAYMENT_COLUMN_PAYMENT_DATE]

		if status != "Paid" && status != "Refunded" {
			continue
		}

		transfer, ok := stripeTransfersByTransferId[transferId]
		if !ok {
			log.Printf("transfer id %s no transfer found for payment id %s from date %v", transferId, paymentId, paymentDate)
		}
		stripeTransfersByTransactionId[paymentId] = transfer
	}
}

func importVatCorrections() {
	vatCorrections = make(map[string]string)

	correctionsFile, err := os.Open("input/vat_corrections.csv")
	if err != nil {
		log.Fatal(err)
	}
	r := csv.NewReader(correctionsFile)

	corrections, err := r.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	for _, record := range corrections {
		vatCorrections[record[0]] = record[1]
	}
}

func outputPaypalTransactions() {
	paypalOutput := [][]string{
		{"Datum", "Kundenname", "Betrag USD", "VAT USD", "VAT", "Land", "EU", "Privat", "USt-ID", "Rückerstattet"},
	}

	for _, tx := range transactions {
		if tx.TransactionType != TransactionTypePaypal {
			continue
		}

		isEU := ""
		if tx.IsEU {
			isEU = "x"
		}

		isPrivate := ""
		if tx.IsPrivate {
			isPrivate = "x"
		}
		if !tx.IsEU {
			isPrivate = "-"
		}

		isRefund := ""
		if tx.IsRefund {
			isRefund = "x"
		}

		record := []string{
			tx.Timestamp.Format(PAYPAL_DATE_OUTPUT_FORMAT),
			tx.CustomerName,
			tx.Amount.ToStringGermany(),
			tx.TaxAmount.ToStringGermany(),
			tx.TaxAmount.VATPercentOfAsStringGermany(tx.Amount),
			tx.CountryCode,
			isEU,
			isPrivate,
			tx.TaxNumber,
			isRefund,
		}
		paypalOutput = append(paypalOutput, record)
	}

	outputFile, err := os.Create("output/Paypal.csv")
	if err != nil {
		log.Fatal(err)
	}

	w := csv.NewWriter(outputFile)
	w.WriteAll(paypalOutput)

	if err := w.Error(); err != nil {
		log.Fatalln("error writing paypal output csv: %v", err)
	}
}

func outputStripeTransactions() {
	stripeOutput := [][]string{
		{"Datum", "Kundenname", "Betrag USD", "VAT USD", "VAT", "Land", "EU", "Privat", "USt-ID", "Datum Transfer", "Gesamtbetrag Transfer EUR", "Rückerstattet"},
	}

	for _, tx := range transactions {
		if tx.TransactionType != TransactionTypeStripe {
			continue
		}

		isEU := ""
		if tx.IsEU {
			isEU = "x"
		}

		isPrivate := ""
		if tx.IsPrivate {
			isPrivate = "x"
		}
		if !tx.IsEU {
			isPrivate = "-"
		}

		isRefund := ""
		if tx.IsRefund {
			isRefund = "x"
		}

		transfer := stripeTransfersByTransactionId[tx.TransactionId]

		record := []string{
			tx.Timestamp.Format(PAYPAL_DATE_OUTPUT_FORMAT),
			tx.CustomerName,
			tx.Amount.ToStringGermany(),
			tx.TaxAmount.ToStringGermany(),
			tx.TaxAmount.VATPercentOfAsStringGermany(tx.Amount),
			tx.CountryCode,
			isEU,
			isPrivate,
			tx.TaxNumber,
			transfer.Date.Format(PAYPAL_DATE_OUTPUT_FORMAT),
			transfer.Amount.ToStringGermany(),
			isRefund,
		}
		stripeOutput = append(stripeOutput, record)
	}

	outputFile, err := os.Create("output/Stripe.csv")
	if err != nil {
		log.Fatal(err)
	}

	w := csv.NewWriter(outputFile)
	w.WriteAll(stripeOutput)

	if err := w.Error(); err != nil {
		log.Fatalln("error writing stripe output csv: %v", err)
	}
}
<commit_msg>First version of EDD/Stripe only accounting<commit_after>package main

import (
	"encoding/csv"
	"fmt"
	"strconv"
	"log"
	"os"
	"strings"
	"time"
)

const (
	EDD_TIMESTAMP_FORMAT = "2006-01-02 15:04:05"
	EDD_PAYMENT_COLUMN_STATUS=25
	EDD_PAYMENT_COLUMN_TIMESTAMP=20
	EDD_PAYMENT_COLUMN_CUSTOMER_EMAIL=2
	EDD_PAYMENT_COLUMN_CUSTOMER_FIRST_NAME=4
	EDD_PAYMENT_COLUMN_CUSTOMER_LAST_NAME=5
	EDD_PAYMENT_COLUMN_CUSTOMER_TAX=15
	EDD_PAYMENT_COLUMN_CUSTOMER_AMOUNT=14
	EDD_PAYMENT_COLUMN_CUSTOMER_COUNTRY_CODE=10
	EDD_PAYMENT_COLUMN_CUSTOMER_TAX_NUMBER=27
	EDD_PAYMENT_COLUMN_VAT_RATE=29

	STRIPE_DATE_FORMAT = "2006-01-02 15:04"
	STRIPE_TRANSFER_COLUMN_STATUS=3
	STRIPE_TRANSFER_COLUMN_DATE=0
	STRIPE_TRANSFER_COLUMN_TRANSFER_ID=1
	STRIPE_TRANSFER_COLUMND_AMOUNT=5
	STRIPE_PAYMENT_COLUMN_PAYMENT_ID=0
	STRIPE_PAYMENT_COLUMN_PAYMENT_CUSTOMER_ID=15
	STRIPE_PAYMENT_COLUMN_PAYMENT_DATE=2
	STRIPE_PAYMENT_COLUMN_TRANSFER_ID=45
	STRIPE_PAYMENT_COLUMN_STATUS=12

	REPORT_DATE_OUTPUT_FORMAT = "02.01.2006"
)

type Amount struct {
	Dollars int64
	Cents int64
}

func (a Amount) IsZero() bool {
	return a.Dollars == 0 && a.Cents == 0
}

func (a Amount) ToStringGermany() string {
	if a.Dollars == 0 && a.Cents == 0 {
		return ""
	}
	return fmt.Sprintf("%d,%02d",a.Dollars, a.Cents)
}

// VAT % of total amount (including VAT)
func (a Amount) VATPercentOfAsStringGermany(other Amount) string {
	totalCentsA := a.Dollars * 100 + a.Cents
	totalCentsB := other.Dollars * 100 + other.Cents - totalCentsA
	percentage := float64(totalCentsA) * 100.0 / float64(totalCentsB)
	percentageRounded := int64(percentage + 0.5)
	if percentageRounded == 0 {
		return ""
	}
	return fmt.Sprintf("%d%%", percentageRounded)
}

type EddPayment struct {
	Timestamp time.Time
	CustomerEmail string
	CustomerName string
	Amount Amount
	TaxAmount Amount
	IsEU bool
	IsPrivate bool
	CountryCode string
	TaxNumber string
	IsRefund bool
}

type StripeTransfer struct {
	TransferId string
	Date time.Time
	Amount Amount
	Status string
}

var payments []EddPayment
var stripeTransfersByTransactionKey map[string]StripeTransfer

func main() {
	importEddPayments()
	importStripeTransferMap()
	outputStripeTransactions()
}

func importEddPayments() {
	eddPaymentsFile, err := os.Open("input/edd-export-payments.csv")
	if err != nil {
		log.Fatal(err)
	}
	r := csv.NewReader(eddPaymentsFile)

	records, err := r.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	for _, record := range records[1:] {
		status := record[EDD_PAYMENT_COLUMN_STATUS]
		if status != "complete" && status != "refunded" {
			continue
		}

		timestampStr := record[EDD_PAYMENT_COLUMN_TIMESTAMP]
		timestamp, err := time.Parse(EDD_TIMESTAMP_FORMAT, timestampStr)
		if err != nil {
			log.Fatal(err)
		}

		customerEmail := record[EDD_PAYMENT_COLUMN_CUSTOMER_EMAIL]

		countryCode := record[EDD_PAYMENT_COLUMN_CUSTOMER_COUNTRY_CODE]
		taxNumber := strings.ToUpper(record[EDD_PAYMENT_COLUMN_CUSTOMER_TAX_NUMBER])
		isEU := record[EDD_PAYMENT_COLUMN_VAT_RATE] != "??"
		isRefund := status == "refunded"
		isPrivate := isEU && taxNumber == ""

		amount := parseAmount(record[EDD_PAYMENT_COLUMN_CUSTOMER_AMOUNT])
		taxAmount := parseAmount(record[EDD_PAYMENT_COLUMN_CUSTOMER_TAX])

		customerFirstName := strings.Title(strings.ToLower(record[EDD_PAYMENT_COLUMN_CUSTOMER_FIRST_NAME]))
		customerLastName := strings.Title(strings.ToLower(record[EDD_PAYMENT_COLUMN_CUSTOMER_LAST_NAME]))
		customerName := customerFirstName
		if customerLastName != "" {
			customerName = customerName + " " + customerLastName
		}

		eddPayment := EddPayment{
			Timestamp: timestamp,
			CustomerEmail: customerEmail,
			CustomerName: customerName,
			Amount: amount,
			TaxAmount: taxAmount,
			IsEU: isEU,
			IsPrivate: isPrivate,
			CountryCode: countryCode,
			TaxNumber: taxNumber,
			IsRefund: isRefund,
		}

		payments = append(payments, eddPayment)
	}
}

func parseAmount(amountStr string) Amount {
			var amount Amount
			amountStrStripped := strings.Replace(amountStr, ",", "", -1)
			amountParts := strings.Split(amountStrStripped, ".")
			if len(amountParts) > 2 {
				log.Fatalf("Invalid amount found: %s", amountStr)
			}
			if len(amountParts) > 0 {
				dollars, err := strconv.ParseInt(amountParts[0], 10, 64)
				if err != nil {
					log.Fatal(err)
				}
				amount.Dollars = dollars
				if len(amountParts) > 1 {
					cents, err  := strconv.ParseInt(amountParts[1], 10, 64)
					if cents < 10 { // 9.8 is 9 dollar 80 cents and not 9 dollar 8 cents
						cents *= 10
					}
					if err != nil {
						log.Fatal(err)
					}
					amount.Cents = cents
				}
			}
			return amount
		}

func importStripeTransferMap() {
	stripeTransfersByTransferId := make(map[string]StripeTransfer)

	stripeTransfersFile, err := os.Open("input/payouts.csv")
	if err != nil {
		log.Fatal(err)
	}
	r := csv.NewReader(stripeTransfersFile)

	transferRecords, err := r.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	for _, record := range transferRecords[1:] {
		status := record[STRIPE_TRANSFER_COLUMN_STATUS]

		dateStr := record[STRIPE_TRANSFER_COLUMN_DATE]
		date, err := time.Parse(STRIPE_DATE_FORMAT, dateStr)
		if err != nil {
			log.Fatal(err)
		}

		var amount Amount
		amountStr := record[STRIPE_TRANSFER_COLUMND_AMOUNT]
		amountStrStripped := strings.Replace(amountStr, ".", "", -1)
		amountParts := strings.Split(amountStrStripped, ",")
		if len(amountParts) > 2 {
			log.Fatalf("Invalid amount found: %s", amountStr)
		}
		if len(amountParts) > 0 {
			dollars, err := strconv.ParseInt(amountParts[0], 10, 64)
			if err != nil {
				log.Fatal(err)
			}
			amount.Dollars = dollars
			if len(amountParts) > 1 {
				cents, err  := strconv.ParseInt(amountParts[1], 10, 64)
				if err != nil {
					log.Fatal(err)
				}
				amount.Cents = cents
			}
		}

		transferId := record[STRIPE_TRANSFER_COLUMN_TRANSFER_ID]

		transferRecord := StripeTransfer{
			TransferId: transferId,
			Date: date,
			Amount: amount,
			Status: status,
		}

		stripeTransfersByTransferId[transferId] = transferRecord
	}

	stripePaymentsFile, err := os.Open("input/payments.csv")
	if err != nil {
		log.Fatal(err)
	}
	r2 := csv.NewReader(stripePaymentsFile)

	paymentRecords, err := r2.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	stripeTransfersByTransactionKey = make(map[string]StripeTransfer)

	for _, record := range paymentRecords[1:] {
		paymentId := record[STRIPE_PAYMENT_COLUMN_PAYMENT_ID]
		customerId := record[STRIPE_PAYMENT_COLUMN_PAYMENT_CUSTOMER_ID]
		transferId := record[STRIPE_PAYMENT_COLUMN_TRANSFER_ID]
		status := record[STRIPE_PAYMENT_COLUMN_STATUS]
		paymentDateStr := record[STRIPE_PAYMENT_COLUMN_PAYMENT_DATE]
		paymentDate, err := time.Parse(STRIPE_DATE_FORMAT, paymentDateStr)
		if err != nil {
			log.Fatal(err)
		}

		if status != "Paid" && status != "Refunded" {
			continue
		}

		transfer, ok := stripeTransfersByTransferId[transferId]
		if !ok {
			log.Printf("transfer id %s no transfer found for payment id %s from date %v", transferId, paymentId, paymentDate)
		}

		transactionKey := transactionKeyFromCustomerIdAndTimestamp(customerId, paymentDate)
		log.Printf("Transaction key of payout: %v", transactionKey)
		stripeTransfersByTransactionKey[transactionKey] = transfer
	}
}

// payment id can currently not be used for lookup as edd does not export it
// instead we use a combination of customer id and the timestamp rounded to the hour
// (for now, can be improved later when problems occur)
func transactionKeyFromCustomerIdAndTimestamp(customerId string, timestamp time.Time) string {
	return fmt.Sprintf("%s_%d", customerId, timestamp.Truncate(60 * time.Minute).Unix())
}

func outputStripeTransactions() {
	stripeOutput := [][]string{
		{"Datum", "Kundenname", "Betrag USD", "VAT USD", "VAT", "Land", "EU", "Privat", "USt-ID", "Datum Transfer", "Gesamtbetrag Transfer EUR", "Rückerstattet"},
	}

	for _, tx := range payments {
		isEU := ""
		if tx.IsEU {
			isEU = "x"
		}

		isPrivate := ""
		if tx.IsPrivate {
			isPrivate = "x"
		}
		if !tx.IsEU {
			isPrivate = "-"
		}

		isRefund := ""
		if tx.IsRefund {
			isRefund = "x"
		}

		// use customer email as id because real id is just an internal number
		transactionKey := transactionKeyFromCustomerIdAndTimestamp(tx.CustomerEmail, tx.Timestamp)
		log.Printf("Transaction key of payment: %v", transactionKey)
		transfer, ok := stripeTransfersByTransactionKey[transactionKey]
		if !ok {
			log.Printf("no transfer found for payment %+v", tx)
			continue
		}

		record := []string{
			tx.Timestamp.Format(REPORT_DATE_OUTPUT_FORMAT),
			tx.CustomerName,
			tx.Amount.ToStringGermany(),
			tx.TaxAmount.ToStringGermany(),
			tx.TaxAmount.VATPercentOfAsStringGermany(tx.Amount),
			tx.CountryCode,
			isEU,
			isPrivate,
			tx.TaxNumber,
			transfer.Date.Format(REPORT_DATE_OUTPUT_FORMAT),
			transfer.Amount.ToStringGermany(),
			isRefund,
		}
		stripeOutput = append(stripeOutput, record)
	}

	outputFile, err := os.Create("output/Stripe.csv")
	if err != nil {
		log.Fatal(err)
	}

	w := csv.NewWriter(outputFile)
	w.WriteAll(stripeOutput)

	if err := w.Error(); err != nil {
		log.Fatalln("error writing stripe output csv: %v", err)
	}
}
<|endoftext|>


<commit_before>// Please use library.
package main

import (
	_ "github.com/winlinvip/go-fdkaac/dec"
)

func main() {
	return
}
<commit_msg>update main<commit_after>// Please use library.
package main

import (
	_ "github.com/winlinvip/go-fdkaac/dec"
	_ "github.com/winlinvip/go-fdkaac/enc"
)

func main() {
	return
}
<|endoftext|>


<commit_before>// +build example

package main

import (
        "fmt"
        "io/ioutil"
        "log"
        "net/url"
        "os"
        "os/exec"
        "path/filepath"
        "regexp"
        "strings"
        "sync"
        "time"

        "github.com/aws/aws-sdk-go/aws"
        "github.com/aws/aws-sdk-go/aws/session"
        "github.com/aws/aws-sdk-go/service/s3"
        "github.com/urfave/cli"

        "k8s.io/client-go/kubernetes"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        apiv1 "k8s.io/api/core/v1"
        k8sErrors "k8s.io/apimachinery/pkg/api/errors"
        rest "k8s.io/client-go/rest"
        discovery "k8s.io/apimachinery/pkg/version"
        wait "k8s.io/apimachinery/pkg/util/wait"

)

// Lists all objects in a bucket using pagination
//
// Usage:

const (
        // High enough QPS to fit all expected use cases. QPS=0 is not set here, because
        // client code is overriding it.
        defaultQPS = 1e6
        // High enough Burst to fit all expected use cases. Burst=0 is not set here, because
        // client code is overriding it.
        defaultBurst = 1e6
)

var secret_hash map[string]map[string][]byte

type S3Address struct {
        Bucket string
        Prefix string
}

func parseURL(path string) S3Address {
        var s S3Address

        if !strings.HasPrefix(path, "s3://") {
                path = fmt.Sprintf("s3://%s", path)
        }

        u, err := url.Parse(path)
        if err != nil {
                log.Fatal("Could not parse URL due to error: ", err)
        }

        if u.Scheme != "" && u.Scheme != "s3" {
                log.Fatal("Invalid URI scheme must be one of s3/NONE")
        }

        if strings.HasPrefix(u.Path, "/") {
                s.Prefix = u.Path[1:]
        } else {
                s.Prefix = u.Path
        }

        s.Bucket = u.Host
        return s
}

// listObjects <bucket>
func main() {
        app := cli.NewApp()
        app.Name = "s3sync"
        app.Version = "0.0.5"

        cli.VersionFlag = cli.BoolFlag{
                Name:  "version, V",
                Usage: "print version number",
        }

        app.Flags = []cli.Flag{
                cli.StringFlag{
                        Name:   "exec-on-change",
                        Usage:  "run a command in case of changes after sync",
                        EnvVar: "EXEC_ON_CHANGE,ON_CHANGE_EXEC,ON_CHANGE",
                },
                cli.StringFlag{
                        Name:   "match-regexp",
                        Usage:  "match s3 bucket contents with following regexp",
                        EnvVar: "MATCH_REGEXP",
                },
                cli.StringFlag{
                        Name:   "filter-out-regexp",
                        Usage:  "filter out s3 bucket with following regexp",
                        EnvVar: "FILTER_OUT_REGEXP",
                },
                cli.StringFlag{
                        Name:   "create-k8s-secret",
                        Usage:  "If true, will create secret on Kubernetes cluster instead of local file",
                        EnvVar: "CREATE_K8S_SECRET",
                },
                cli.StringFlag{
                        Name:   "k8s-secret-namespace",
                        Usage:  "Kubernetes cluster Namesace Name for secrets",
                        EnvVar: "K8S_SECRET_NAMESPACE",
                },
                cli.StringFlag{
                        Name:   "k8s-custom-label-name",
                        Usage:  "Kubernetes label name that will be assign to secret",
                        EnvVar: "K8S_CUSTOM_LABEL_NAME",
                },
                cli.StringFlag{
                        Name:   "k8s-custom-label-value",
                        Usage:  "Kubernetes label value that will be assign to secret",
                        EnvVar: "K8S_CUSTOM_LABEL_VALUE",
                },
                cli.StringFlag{
                        Name:   "verbose",
                        Usage:  "Verbose flag",
                        EnvVar: "VERBOSE",
                },
        }

        app.Commands = []cli.Command{
                {
                        Name:   "sync",
                        Usage:  "Synchronize a directory tree to S3 -- LOCAL_DIR s3://BUCKET[/PREFIX] or s3://BUCKET[/PREFIX] LOCAL_DIR",
                        Action: CmdSync,
                        Flags:  app.Flags,
                },
        }
        app.Run(os.Args)
}

func CmdSync(c *cli.Context) error {
        sess, err := session.NewSession()

        var wg sync.WaitGroup

        if err != nil {
                fmt.Println("failed to create session,", err)
                return err
        }

        svc := s3.New(sess)
        s3_path := c.Args().Get(0)
        local_path := c.Args().Get(1)

        u := parseURL(s3_path)

        params := &s3.ListObjectsV2Input{
                Bucket:     aws.String(u.Bucket), // Required
                FetchOwner: aws.Bool(true),
                Prefix:     aws.String(u.Prefix),
        }

        i := 0
        var ActionRequired bool
        secret_hash = make(map[string]map[string][]byte)

        err = svc.ListObjectsV2Pages(params,
                func(p *s3.ListObjectsV2Output, last bool) (shouldContinue bool) {
                        i++
                        for _, obj := range p.Contents {
                                // make magic here
                                var filter_out bool
                                matched := true

                                if c.GlobalString("match-regexp") != "" {
                                        matched, _ = regexp.MatchString(c.GlobalString("match-regexp"), *obj.Key)
                                }

                                if c.GlobalString("filter-out-regexp") != "" {
                                        filter_out, _ = regexp.MatchString(c.GlobalString("filter-out-regexp"), *obj.Key)
                                }

                                if matched && filter_out != true {
                                        if c.GlobalString("create-k8s-secret") == "true" {
                                                saveFileToSecretMapFromS3(svc, u.Bucket, *obj, filepath.Base(*obj.Key), &wg)
                                        } else {
                                                filename := filepath.Join(local_path, filepath.Base(*obj.Key))
                                                fileInfo, err := os.Stat(filename)

                                                if os.IsNotExist(err) {
                                                        fmt.Println("file: ", filename, "does not exists, copying from s3")
                                                        go copyFileFromS3(svc, u.Bucket, *obj, filename, &wg)
                                                        ActionRequired = true
                                                } else {
                                                        if fileInfo.ModTime().UTC() != *obj.LastModified {
                                                                fmt.Println("modification time of file:", filename, "does not match one from s3, ordering copy")
                                                                go copyFileFromS3(svc, u.Bucket, *obj, filename, &wg)
                                                                ActionRequired = true
                                                        }
                                                }
                                        }
                                }
                        }
                        wg.Wait()
                        return true
                })

        if c.GlobalString("create-k8s-secret") == "true" {
                secret_namespace := "default"

                if c.GlobalString("k8s-secret-namespace") != "" {
                        secret_namespace = c.GlobalString("k8s-secret-namespace")
                }

                err = saveSecretMapToK8s(secret_hash,secret_namespace,c.GlobalString("k8s-custom-label-name"),c.GlobalString("k8s-custom-label-value"),c.GlobalString("verbose"))
        }

        if err != nil {
                // Print the error, cast err to awserr.Error to get the Code and
                // Message from an error.
                fmt.Println(err.Error())
                return err
        }

        if ActionRequired {
                action := fmt.Sprintf("%v", c.GlobalString("exec-on-change"))
                fmt.Println("Executing command", action)

                cmd := exec.Command("/bin/sh", "-c", action)
                cmd.Stdout = os.Stdout
                cmd.Stderr = os.Stderr
                err = cmd.Run()
                if err != nil {
                        fmt.Println("could not execute due to error:", err)
                }
        }
        return nil

}

func copyFileFromS3(s *s3.S3, bucket string, obj s3.Object, filename string, wg *sync.WaitGroup) error {
        defer wg.Done()
        wg.Add(1)
        params := &s3.GetObjectInput{
                Bucket: aws.String(bucket),   // Required
                Key:    aws.String(*obj.Key), // Required
        }

        resp, err := s.GetObject(params)
        if err != nil {
                return err
        }

        data, err := ioutil.ReadAll(resp.Body)
        if err != nil {
                fmt.Println("Could not download file from s3 due to:", err)
                return err
        }

        err = ioutil.WriteFile(filename, data, 0666)
        if err != nil {
                fmt.Println("Could not write file from s3 due to:", err)
                return err
        }
        return os.Chtimes(filename, *obj.LastModified, *obj.LastModified)
}

func createApiserverClient() (*kubernetes.Clientset, error) {
        cfg, err := rest.InClusterConfig()
        if err != nil {
                fmt.Println("could not execute due to error:", err)
                return nil, err
        }

        cfg.QPS = defaultQPS
        cfg.Burst = defaultBurst
        cfg.ContentType = "application/vnd.kubernetes.protobuf"

        client, err := kubernetes.NewForConfig(cfg)
        if err != nil {
                fmt.Println("could not execute due to error:", err)
                return nil, err
        }

        var v *discovery.Info

        // In some environments is possible the client cannot connect the API server in the first request
        // https://github.com/kubernetes/ingress-nginx/issues/1968
        defaultRetry := wait.Backoff{
                Steps:    10,
                Duration: 1 * time.Second,
                Factor:   1.5,
                Jitter:   0.1,
        }

        var lastErr error
        retries := 0
        err = wait.ExponentialBackoff(defaultRetry, func() (bool, error) {
                v, err = client.Discovery().ServerVersion()
                if err == nil {
                        return true, nil
                }

                lastErr = err
                retries++
                return false, nil
        })

        // err is not null only if there was a timeout in the exponential backoff (ErrWaitTimeout)
        if err != nil {
                return nil, lastErr
        }

        return client, nil
}

func EnsureSecret(secret *apiv1.Secret, verbose string) (*apiv1.Secret, error) {
        kubeClient, err := createApiserverClient()
        s, err := kubeClient.CoreV1().Secrets(secret.Namespace).Create(secret)
        if err != nil {
                if k8sErrors.IsAlreadyExists(err) {
                        if verbose == "true" {
                                fmt.Println("Secret" ,secret.Name," already exist in namespace ",secret.Namespace,", updating")
                        }
                        return kubeClient.CoreV1().Secrets(secret.Namespace).Update(secret)
                }
                fmt.Println("could not execute due to error:", err)
                return nil, err
        }
        fmt.Println("Secret" ,secret.Name," created in namespace ",secret.Namespace)
        return s, nil
}

func saveFileToSecretMapFromS3(s *s3.S3, bucket string, obj s3.Object, filename string, wg *sync.WaitGroup) (error) {
        defer wg.Done()
        wg.Add(1)
        params := &s3.GetObjectInput{
                Bucket: aws.String(bucket),   // Required
                Key:    aws.String(*obj.Key), // Required
        }
        resp, err := s.GetObject(params)
        if err != nil {
                return err
        }

        data, err := ioutil.ReadAll(resp.Body)
        if err != nil {
                fmt.Println("Could not download file from s3 due to:", err)
                return err
        }

        r_secret_name, err := regexp.Compile("^(.+)\\.(?:key|crt)$")

        match_key, err := regexp.MatchString("^(.+)\\.key$", filename)
        match_cert, err := regexp.MatchString("^(.+)\\.crt$", filename)

        if match_key {
                if secret_hash[r_secret_name.FindStringSubmatch(filename)[1]] == nil {
                        secret_hash[r_secret_name.FindStringSubmatch(filename)[1]] = make(map[string][]byte)
                }
                secret_hash[r_secret_name.FindStringSubmatch(filename)[1]]["key"] = data
        }
        if match_cert {
                if secret_hash[r_secret_name.FindStringSubmatch(filename)[1]] == nil {
                        secret_hash[r_secret_name.FindStringSubmatch(filename)[1]] = make(map[string][]byte)
                }
                secret_hash[r_secret_name.FindStringSubmatch(filename)[1]]["cert"] = data
        }

        return err
}

func saveSecretMapToK8s(sh map[string]map[string][]byte, namespace string, label_name string, label_value string, verbose string) (err error) {
        fmt.Println("Start creating secrets in Kubernetes")

        for secret_domain := range sh {
                var err_message string
                if sh[secret_domain]["cert"] == nil {
                        err_message = "ERROR: " + secret_domain + " do not have cert file"
                        fmt.Println(err_message)
                } else if sh[secret_domain]["key"] == nil {
                        err_message = "ERROR: " + secret_domain + " do not have key file"
                        fmt.Println(err_message)
                } else {
                        var labels map[string]string
                        labels = make (map[string]string)
                        labels["cert_domain"] = secret_domain
                        labels["created_by"] = "s3sync"
                        if label_name != "" && label_value != "" {
                                labels[label_name] = label_value
                        }

                        _, err = EnsureSecret(&apiv1.Secret{
                                ObjectMeta: metav1.ObjectMeta{
                                        Name:      secret_domain,
                                        Namespace: namespace,
                                        Labels: labels,
                                },
                                Data: map[string][]byte{
                                        apiv1.TLSCertKey:       sh[secret_domain]["cert"],
                                        apiv1.TLSPrivateKeyKey: sh[secret_domain]["key"],
                                },
                        }, verbose)

                        if err != nil {
                                fmt.Println(err.Error())
                                return err
                        }
                }
        }
        return nil
}
<commit_msg>add verbose for start creating secrets<commit_after>// +build example

package main

import (
        "fmt"
        "io/ioutil"
        "log"
        "net/url"
        "os"
        "os/exec"
        "path/filepath"
        "regexp"
        "strings"
        "sync"
        "time"

        "github.com/aws/aws-sdk-go/aws"
        "github.com/aws/aws-sdk-go/aws/session"
        "github.com/aws/aws-sdk-go/service/s3"
        "github.com/urfave/cli"

        "k8s.io/client-go/kubernetes"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        apiv1 "k8s.io/api/core/v1"
        k8sErrors "k8s.io/apimachinery/pkg/api/errors"
        rest "k8s.io/client-go/rest"
        discovery "k8s.io/apimachinery/pkg/version"
        wait "k8s.io/apimachinery/pkg/util/wait"

)

// Lists all objects in a bucket using pagination
//
// Usage:

const (
        // High enough QPS to fit all expected use cases. QPS=0 is not set here, because
        // client code is overriding it.
        defaultQPS = 1e6
        // High enough Burst to fit all expected use cases. Burst=0 is not set here, because
        // client code is overriding it.
        defaultBurst = 1e6
)

var secret_hash map[string]map[string][]byte

type S3Address struct {
        Bucket string
        Prefix string
}

func parseURL(path string) S3Address {
        var s S3Address

        if !strings.HasPrefix(path, "s3://") {
                path = fmt.Sprintf("s3://%s", path)
        }

        u, err := url.Parse(path)
        if err != nil {
                log.Fatal("Could not parse URL due to error: ", err)
        }

        if u.Scheme != "" && u.Scheme != "s3" {
                log.Fatal("Invalid URI scheme must be one of s3/NONE")
        }

        if strings.HasPrefix(u.Path, "/") {
                s.Prefix = u.Path[1:]
        } else {
                s.Prefix = u.Path
        }

        s.Bucket = u.Host
        return s
}

// listObjects <bucket>
func main() {
        app := cli.NewApp()
        app.Name = "s3sync"
        app.Version = "0.0.5"

        cli.VersionFlag = cli.BoolFlag{
                Name:  "version, V",
                Usage: "print version number",
        }

        app.Flags = []cli.Flag{
                cli.StringFlag{
                        Name:   "exec-on-change",
                        Usage:  "run a command in case of changes after sync",
                        EnvVar: "EXEC_ON_CHANGE,ON_CHANGE_EXEC,ON_CHANGE",
                },
                cli.StringFlag{
                        Name:   "match-regexp",
                        Usage:  "match s3 bucket contents with following regexp",
                        EnvVar: "MATCH_REGEXP",
                },
                cli.StringFlag{
                        Name:   "filter-out-regexp",
                        Usage:  "filter out s3 bucket with following regexp",
                        EnvVar: "FILTER_OUT_REGEXP",
                },
                cli.StringFlag{
                        Name:   "create-k8s-secret",
                        Usage:  "If true, will create secret on Kubernetes cluster instead of local file",
                        EnvVar: "CREATE_K8S_SECRET",
                },
                cli.StringFlag{
                        Name:   "k8s-secret-namespace",
                        Usage:  "Kubernetes cluster Namesace Name for secrets",
                        EnvVar: "K8S_SECRET_NAMESPACE",
                },
                cli.StringFlag{
                        Name:   "k8s-custom-label-name",
                        Usage:  "Kubernetes label name that will be assign to secret",
                        EnvVar: "K8S_CUSTOM_LABEL_NAME",
                },
                cli.StringFlag{
                        Name:   "k8s-custom-label-value",
                        Usage:  "Kubernetes label value that will be assign to secret",
                        EnvVar: "K8S_CUSTOM_LABEL_VALUE",
                },
                cli.StringFlag{
                        Name:   "verbose",
                        Usage:  "Verbose flag",
                        EnvVar: "VERBOSE",
                },
        }

        app.Commands = []cli.Command{
                {
                        Name:   "sync",
                        Usage:  "Synchronize a directory tree to S3 -- LOCAL_DIR s3://BUCKET[/PREFIX] or s3://BUCKET[/PREFIX] LOCAL_DIR",
                        Action: CmdSync,
                        Flags:  app.Flags,
                },
        }
        app.Run(os.Args)
}

func CmdSync(c *cli.Context) error {
        sess, err := session.NewSession()

        var wg sync.WaitGroup

        if err != nil {
                fmt.Println("failed to create session,", err)
                return err
        }

        svc := s3.New(sess)
        s3_path := c.Args().Get(0)
        local_path := c.Args().Get(1)

        u := parseURL(s3_path)

        params := &s3.ListObjectsV2Input{
                Bucket:     aws.String(u.Bucket), // Required
                FetchOwner: aws.Bool(true),
                Prefix:     aws.String(u.Prefix),
        }

        i := 0
        var ActionRequired bool
        secret_hash = make(map[string]map[string][]byte)

        err = svc.ListObjectsV2Pages(params,
                func(p *s3.ListObjectsV2Output, last bool) (shouldContinue bool) {
                        i++
                        for _, obj := range p.Contents {
                                // make magic here
                                var filter_out bool
                                matched := true

                                if c.GlobalString("match-regexp") != "" {
                                        matched, _ = regexp.MatchString(c.GlobalString("match-regexp"), *obj.Key)
                                }

                                if c.GlobalString("filter-out-regexp") != "" {
                                        filter_out, _ = regexp.MatchString(c.GlobalString("filter-out-regexp"), *obj.Key)
                                }

                                if matched && filter_out != true {
                                        if c.GlobalString("create-k8s-secret") == "true" {
                                                saveFileToSecretMapFromS3(svc, u.Bucket, *obj, filepath.Base(*obj.Key), &wg)
                                        } else {
                                                filename := filepath.Join(local_path, filepath.Base(*obj.Key))
                                                fileInfo, err := os.Stat(filename)

                                                if os.IsNotExist(err) {
                                                        fmt.Println("file: ", filename, "does not exists, copying from s3")
                                                        go copyFileFromS3(svc, u.Bucket, *obj, filename, &wg)
                                                        ActionRequired = true
                                                } else {
                                                        if fileInfo.ModTime().UTC() != *obj.LastModified {
                                                                fmt.Println("modification time of file:", filename, "does not match one from s3, ordering copy")
                                                                go copyFileFromS3(svc, u.Bucket, *obj, filename, &wg)
                                                                ActionRequired = true
                                                        }
                                                }
                                        }
                                }
                        }
                        wg.Wait()
                        return true
                })

        if c.GlobalString("create-k8s-secret") == "true" {
                secret_namespace := "default"

                if c.GlobalString("k8s-secret-namespace") != "" {
                        secret_namespace = c.GlobalString("k8s-secret-namespace")
                }

                err = saveSecretMapToK8s(secret_hash,secret_namespace,c.GlobalString("k8s-custom-label-name"),c.GlobalString("k8s-custom-label-value"),c.GlobalString("verbose"))
        }

        if err != nil {
                // Print the error, cast err to awserr.Error to get the Code and
                // Message from an error.
                fmt.Println(err.Error())
                return err
        }

        if ActionRequired {
                action := fmt.Sprintf("%v", c.GlobalString("exec-on-change"))
                fmt.Println("Executing command", action)

                cmd := exec.Command("/bin/sh", "-c", action)
                cmd.Stdout = os.Stdout
                cmd.Stderr = os.Stderr
                err = cmd.Run()
                if err != nil {
                        fmt.Println("could not execute due to error:", err)
                }
        }
        return nil

}

func copyFileFromS3(s *s3.S3, bucket string, obj s3.Object, filename string, wg *sync.WaitGroup) error {
        defer wg.Done()
        wg.Add(1)
        params := &s3.GetObjectInput{
                Bucket: aws.String(bucket),   // Required
                Key:    aws.String(*obj.Key), // Required
        }

        resp, err := s.GetObject(params)
        if err != nil {
                return err
        }

        data, err := ioutil.ReadAll(resp.Body)
        if err != nil {
                fmt.Println("Could not download file from s3 due to:", err)
                return err
        }

        err = ioutil.WriteFile(filename, data, 0666)
        if err != nil {
                fmt.Println("Could not write file from s3 due to:", err)
                return err
        }
        return os.Chtimes(filename, *obj.LastModified, *obj.LastModified)
}

func createApiserverClient() (*kubernetes.Clientset, error) {
        cfg, err := rest.InClusterConfig()
        if err != nil {
                fmt.Println("could not execute due to error:", err)
                return nil, err
        }

        cfg.QPS = defaultQPS
        cfg.Burst = defaultBurst
        cfg.ContentType = "application/vnd.kubernetes.protobuf"

        client, err := kubernetes.NewForConfig(cfg)
        if err != nil {
                fmt.Println("could not execute due to error:", err)
                return nil, err
        }

        var v *discovery.Info

        // In some environments is possible the client cannot connect the API server in the first request
        // https://github.com/kubernetes/ingress-nginx/issues/1968
        defaultRetry := wait.Backoff{
                Steps:    10,
                Duration: 1 * time.Second,
                Factor:   1.5,
                Jitter:   0.1,
        }

        var lastErr error
        retries := 0
        err = wait.ExponentialBackoff(defaultRetry, func() (bool, error) {
                v, err = client.Discovery().ServerVersion()
                if err == nil {
                        return true, nil
                }

                lastErr = err
                retries++
                return false, nil
        })

        // err is not null only if there was a timeout in the exponential backoff (ErrWaitTimeout)
        if err != nil {
                return nil, lastErr
        }

        return client, nil
}

func EnsureSecret(secret *apiv1.Secret, verbose string) (*apiv1.Secret, error) {
        kubeClient, err := createApiserverClient()
        s, err := kubeClient.CoreV1().Secrets(secret.Namespace).Create(secret)
        if err != nil {
                if k8sErrors.IsAlreadyExists(err) {
                        if verbose == "true" {
                                fmt.Println("Secret" ,secret.Name," already exist in namespace ",secret.Namespace,", updating")
                        }
                        return kubeClient.CoreV1().Secrets(secret.Namespace).Update(secret)
                }
                fmt.Println("could not execute due to error:", err)
                return nil, err
        }
        if verbose == "true" {
                fmt.Println("Secret" ,secret.Name," created in namespace ",secret.Namespace)
        }
        return s, nil
}

func saveFileToSecretMapFromS3(s *s3.S3, bucket string, obj s3.Object, filename string, wg *sync.WaitGroup) (error) {
        defer wg.Done()
        wg.Add(1)
        params := &s3.GetObjectInput{
                Bucket: aws.String(bucket),   // Required
                Key:    aws.String(*obj.Key), // Required
        }
        resp, err := s.GetObject(params)
        if err != nil {
                return err
        }

        data, err := ioutil.ReadAll(resp.Body)
        if err != nil {
                fmt.Println("Could not download file from s3 due to:", err)
                return err
        }

        r_secret_name, err := regexp.Compile("^(.+)\\.(?:key|crt)$")

        match_key, err := regexp.MatchString("^(.+)\\.key$", filename)
        match_cert, err := regexp.MatchString("^(.+)\\.crt$", filename)

        if match_key {
                if secret_hash[r_secret_name.FindStringSubmatch(filename)[1]] == nil {
                        secret_hash[r_secret_name.FindStringSubmatch(filename)[1]] = make(map[string][]byte)
                }
                secret_hash[r_secret_name.FindStringSubmatch(filename)[1]]["key"] = data
        }
        if match_cert {
                if secret_hash[r_secret_name.FindStringSubmatch(filename)[1]] == nil {
                        secret_hash[r_secret_name.FindStringSubmatch(filename)[1]] = make(map[string][]byte)
                }
                secret_hash[r_secret_name.FindStringSubmatch(filename)[1]]["cert"] = data
        }

        return err
}

func saveSecretMapToK8s(sh map[string]map[string][]byte, namespace string, label_name string, label_value string, verbose string) (err error) {
        if verbose == "true" {
                fmt.Println("Start creating secrets in Kubernetes")
        }

        for secret_domain := range sh {
                var err_message string
                if sh[secret_domain]["cert"] == nil {
                        err_message = "ERROR: " + secret_domain + " do not have cert file"
                        fmt.Println(err_message)
                } else if sh[secret_domain]["key"] == nil {
                        err_message = "ERROR: " + secret_domain + " do not have key file"
                        fmt.Println(err_message)
                } else {
                        var labels map[string]string
                        labels = make (map[string]string)
                        labels["cert_domain"] = secret_domain
                        labels["created_by"] = "s3sync"
                        if label_name != "" && label_value != "" {
                                labels[label_name] = label_value
                        }

                        _, err = EnsureSecret(&apiv1.Secret{
                                ObjectMeta: metav1.ObjectMeta{
                                        Name:      secret_domain,
                                        Namespace: namespace,
                                        Labels: labels,
                                },
                                Data: map[string][]byte{
                                        apiv1.TLSCertKey:       sh[secret_domain]["cert"],
                                        apiv1.TLSPrivateKeyKey: sh[secret_domain]["key"],
                                },
                        }, verbose)

                        if err != nil {
                                fmt.Println(err.Error())
                                return err
                        }
                }
        }
        return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"
	"runtime"
	"GoOnchain/net"
	"GoOnchain/net/httpjsonrpc"
	"GoOnchain/common/log"
	"GoOnchain/core/ledger"
	"GoOnchain/core/transaction"
	"GoOnchain/core/store"
	"GoOnchain/client"
)

const (
	// The number of the CPU cores for parallel optimization,TODO set from config file
	NCPU	 = 4
)

func init() {
	runtime.GOMAXPROCS(NCPU)
	var path string = "./Log/"
	log.CreatePrintLog(path)
}

func main() {
	fmt.Println("//**************************************************************************")
	fmt.Println("//*** 0. Client Set                                                      ***")
	fmt.Println("//**************************************************************************")
	ledger.DefaultLedger = new(ledger.Ledger)
	ledger.DefaultLedger.Store = store.NewLedgerStore()
	ledger.DefaultLedger.Store.InitLedgerStore(ledger.DefaultLedger)
	transaction.TxStore =ledger.DefaultLedger.Store
	fmt.Println("  Client set completed. Test Start...")

	fmt.Println("//**************************************************************************")
	fmt.Println("//*** 1. BlockChain init                                                 ***")
	fmt.Println("//**************************************************************************")
	//blockchain :=
	fmt.Println("  BlockChain generate completed. Func test Start...")
	ledger.DefaultLedger.Blockchain = ledger.NewBlockchainWithGenesisBlock()

	fmt.Println("//**************************************************************************")
	fmt.Println("//*** 2. Generate Account                                                ***")
	fmt.Println("//**************************************************************************")
	user, _:= client.NewAccount([]byte{})
	admin, _:= client.NewAccount([]byte{})
	userpubkey, _:= user.PublicKey.EncodePoint(true)
	fmt.Printf( "user.PrivateKey: %x user.PrivateKey Len: %d\n", user.PrivateKey, len(user.PrivateKey) )
	fmt.Printf( "user.PublicKey: %x user.PublicKey Len: %d\n", userpubkey, len(userpubkey) )
	fmt.Printf( "admin.PrivateKey: %x admin.PrivateKey Len: %d\n", admin.PrivateKey, len(admin.PrivateKey) )

	time.Sleep(2 * time.Second)
	net.StartProtocol()

	go httpjsonrpc.StartServer()

	time.Sleep(2 * time.Second)
	httpjsonrpc.StartClient()

	// Modules start sample
	//ledger.Start(net.NetToLedgerCh <-chan *Msg, net.LedgerToNetCh chan<- *Msg)
	//consensus.Start(net.NetToConsensusCh <-chan *Msg, net.ConsensusToNetCh chan<- *Msg)

	for {
		time.Sleep(2 * time.Second)
	}
}
<commit_msg>fix compile error for main.go<commit_after>package main

import (
	"fmt"
	"time"
	"runtime"
	"GoOnchain/net"
	"GoOnchain/net/httpjsonrpc"
	"GoOnchain/common/log"
	"GoOnchain/core/ledger"
	"GoOnchain/core/transaction"
	"GoOnchain/core/store"
	"GoOnchain/client"
)

const (
	// The number of the CPU cores for parallel optimization,TODO set from config file
	NCPU	 = 4
)

func init() {
	runtime.GOMAXPROCS(NCPU)
	var path string = "./Log/"
	log.CreatePrintLog(path)
}

func main() {
	fmt.Println("//**************************************************************************")
	fmt.Println("//*** 0. Client Set                                                      ***")
	fmt.Println("//**************************************************************************")
	ledger.DefaultLedger = new(ledger.Ledger)
	ledger.DefaultLedger.Store = store.NewLedgerStore()
	ledger.DefaultLedger.Store.InitLedgerStore(ledger.DefaultLedger)
	transaction.TxStore =ledger.DefaultLedger.Store
	fmt.Println("  Client set completed. Test Start...")

	fmt.Println("//**************************************************************************")
	fmt.Println("//*** 1. BlockChain init                                                 ***")
	fmt.Println("//**************************************************************************")
	//blockchain :=
	fmt.Println("  BlockChain generate completed. Func test Start...")
	ledger.DefaultLedger.Blockchain, _ = ledger.NewBlockchainWithGenesisBlock()

	fmt.Println("//**************************************************************************")
	fmt.Println("//*** 2. Generate Account                                                ***")
	fmt.Println("//**************************************************************************")
	user, _:= client.NewAccount([]byte{})
	admin, _:= client.NewAccount([]byte{})
	userpubkey, _:= user.PublicKey.EncodePoint(true)
	fmt.Printf( "user.PrivateKey: %x user.PrivateKey Len: %d\n", user.PrivateKey, len(user.PrivateKey) )
	fmt.Printf( "user.PublicKey: %x user.PublicKey Len: %d\n", userpubkey, len(userpubkey) )
	fmt.Printf( "admin.PrivateKey: %x admin.PrivateKey Len: %d\n", admin.PrivateKey, len(admin.PrivateKey) )

	time.Sleep(2 * time.Second)
	net.StartProtocol()

	go httpjsonrpc.StartServer()

	time.Sleep(2 * time.Second)
	httpjsonrpc.StartClient()

	// Modules start sample
	//ledger.Start(net.NetToLedgerCh <-chan *Msg, net.LedgerToNetCh chan<- *Msg)
	//consensus.Start(net.NetToConsensusCh <-chan *Msg, net.ConsensusToNetCh chan<- *Msg)

	for {
		time.Sleep(2 * time.Second)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"path/filepath"

	"github.com/NSkelsey/btcbuilder"
	"github.com/NSkelsey/protocol/ahimsa"
	"github.com/NSkelsey/watchtower"
	"github.com/conformal/btcnet"
	"github.com/conformal/btcrpcclient"
	"github.com/conformal/btcscript"
	"github.com/conformal/btcutil"
	"github.com/conformal/btcwire"
	"github.com/jessevdk/go-flags"
)

var (
	appDataDir        = btcutil.AppDataDir("ahimsa", false)
	defaultConfigFile = filepath.Join(appDataDir, "ahimsa.conf")
	defaultDbName     = filepath.Join(appDataDir, "pubrecord.db")
	defaultBlockDir   = filepath.Join(btcutil.AppDataDir(".bitcoin", false), "testnet3/blocks")
	defaultNetwork    = "TestNet3"
	defaultNodeAddr   = "127.0.0.1:18333"
	defaultRPCAddr    = "127.0.0.1:18332"
	debug             = false
	// Sane defaults for a linux based OS
	cfg = &config{
		ConfigFile: defaultConfigFile,
		BlockDir:   defaultBlockDir,
		DbFile:     defaultDbName,
		NodeAddr:   defaultNodeAddr,
		NetName:    defaultNetwork,
		RPCAddr:    defaultRPCAddr,
		Rebuild:    false,
	}
)

// Application globals
var activeNetParams *btcnet.Params
var logger *log.Logger = log.New(os.Stdout, "", log.Ltime)

type config struct {
	ConfigFile  string `short:"C" long:"configfile" description:"Path to configuration file"`
	BlockDir    string `long:"blockdir" description:"Path to bitcoin blockdir"`
	DbFile      string `long:"dbname" description:"Name of the database file"`
	Rebuild     bool   `long:"rebuild" description:"Flag to rebuild the pubrecord db"`
	RPCAddr     string `long:"rpcaddr" description:"Address of bitcoin rpc endpoint to use"`
	RPCUser     string `long:"rpcuser" description:"RPC username"`
	RPCPassword string `long:"rpcpassword" description:"RPC password"`
	NodeAddr    string `long:"nodeaddr" description:"Address + port of the bitcoin node to connect to"`
	NetName     string `short:"n" long:"network" description:"The name of the network to use"`
	Debug       bool   `shodt:"d" long:"debug" description:"Debug flag for verbose error logging"`
	PrintHelp   bool   `short:"h" long:"help" description:"Prints out this message"`
}

func main() {
	// Parse command line args first then use file args
	parser := flags.NewParser(cfg, flags.None)
	_, err := parser.Parse()
	if err != nil {
		parser.WriteHelp(os.Stdout)
		logger.Fatal(err)
	}

	if cfg.PrintHelp {
		parser.WriteHelp(os.Stdout)
		os.Exit(0)
	}

	// Check to see if application files exist and create them if not
	_, err = os.Stat(appDataDir)
	if err != nil {
		makeDataDir()
	}

	err = flags.NewIniParser(parser).ParseFile(cfg.ConfigFile)
	if err != nil {
		logger.Println("No config file provided, using command line params")
	}

	activeNetParams, err = btcbuilder.NetParamsFromStr(cfg.NetName)
	if err != nil {
		logger.Fatal(err)
	}

	// Configure debug logger for verbose output
	if debug {
		logger = log.New(os.Stdout, "DEBUG\t", log.Ltime|log.Llongfile)
	}

	// Configure and create a RPC client
	connCfg := &btcrpcclient.ConnConfig{
		Host:         cfg.RPCAddr,
		User:         cfg.RPCUser,
		Pass:         cfg.RPCPassword,
		HttpPostMode: true,
		DisableTLS:   true,
	}
	rpcclient, err := btcrpcclient.New(connCfg, nil)
	if err != nil {
		logger.Fatal(err)
	}
	// Test rpc connection
	if err := rpcclient.Ping(); err != nil {
		logger.Println(err)
		msg := `
Connecting to the Bitcoin via RPC failed!! This may have been caused by one of the following:
1. Bitcoind is not running
2. The RPC server is not activated (server=1)
3. rpcuser and rpcpassword were not set
4. You are using Testnet3 settings for a Mainnet server or vice versa.
`
		fmt.Printf(msg)
		os.Exit(1)
	}

	rpcSubChan := make(chan *TxReq)

	// start a rpc command handler
	go authorlookup(rpcclient, rpcSubChan)

	fmt.Println(getBanner())
	// Load the db and find its current chain height
	db := loadDb(rpcclient)

	if err != nil {
		logger.Fatal(err)
	}
	curH := db.CurrentHeight()
	actualH, err := rpcclient.GetBlockCount()
	if err != nil {
		logger.Fatal(err)
	}
	println("Db Height:", curH)

	// Configure the live network feed
	towerCfg := watchtower.TowerCfg{
		Addr:        cfg.NodeAddr,
		Net:         activeNetParams.Net,
		StartHeight: int(db.CurrentHeight()),
		Logger:      logger,
	}

	// If the database reports a height lower than the current height reported by
	// the bitcoin node but is within 500 blocks we can avoid redownloading the
	// whole chain. This is done at the network level with a getblocks msg for
	// any blocks we are missing. This is a relatively simple optimization and it
	// gives us 3 days of wiggle room before the whole chain must be validated
	// again.
	if actualH-curH > 0 {
		getblocks, err := makeBlockMsg(db)
		if err != nil {
			logger.Fatal(err)
		}
		towerCfg.ToSend = []btcwire.Message{getblocks}
	}

	// Start a watchtower instance and listen for new blocks
	txParser := txClosure(db, rpcSubChan)
	blockParser := blockClosure(db)

	watchtower.Create(towerCfg, txParser, blockParser)
}

func makeDataDir() {
	// Creates the application data dir initializing it with a config file that
	// is empty.

	// create dir
	perms := os.ModeDir | 0700
	if err := os.Mkdir(appDataDir, perms); err != nil {
		logger.Fatal(err)
	}

	// touch config file
	f, err := os.Create(cfg.ConfigFile)
	if err != nil {
		logger.Fatal(err)
	}
	if err := f.Close(); err != nil {
		logger.Fatal(err)
	}

	// touch db file
	f, err = os.Create(cfg.DbFile)
	if err != nil {
		logger.Fatal(err)
	}
	if err := f.Close(); err != nil {
		logger.Fatal(err)
	}
}

func loadDb(client *btcrpcclient.Client) *LiteDb {
	// Load the db from the file specified in config and get it to a usuable state
	// where ahimsad can add blocks from the network
	db, err := LoadDb(cfg.DbFile)
	if err != nil {
		logger.Fatal(err)
	}

	actualH, err := client.GetBlockCount()
	if err != nil {
		logger.Fatal(err)
	}

	curH := db.CurrentHeight()

	println("Database hieghts:", curH, actualH)
	// Fudge factor
	if curH < actualH-499 || cfg.Rebuild {
		println("Creating DB")
		// init db
		db, err = InitDb(cfg.DbFile)
		if err != nil {
			logger.Fatal(err)
		}

		// get the tip of the longest valid chain
		tip, err := runBlockScan(cfg.BlockDir, db)
		if err != nil {
			logger.Fatal(err)
		}

		genBlk := walkBackwards(tip)
		err = storeChainBulletins(genBlk, db, client)
		if err != nil {
			logger.Fatal(err)
		}

	}
	return db
}

func storeChainBulletins(genBlock *Block, db *LiteDb, client *btcrpcclient.Client) error {
	// Stores all of the Bulletins we found in the blockchain into the sqlite db.
	// This is done iteratively and is not optimized in any way. We log errors as
	// we encounter them.
	chainHeight := genBlock.depth
	blks := make([]*Block, 0, 1000)
	var blk *Block = genBlock
	for {
		// Walk forwards through the blocks
		if blk.NextBlock == nil {
			break
		}

		if len(blks) >= 1000 {
			err := db.BatchInsertBH(blks, chainHeight)
			if err != nil {
				return err
			}
			blks = make([]*Block, 0, 1000)
		}
		blks = append(blks, blk)

		blk = blk.NextBlock
	}

	// Insert remaining blocks
	err := db.BatchInsertBH(blks, chainHeight)
	if err != nil {
		return err
	}

	// We are now back at the tip
	//var tip *Block = blk
	for {
		// walk backwards through blocks
		if blk.Hash == genesisHash {
			break
		}

		var bh *btcwire.BlockHeader
		bh = btcBHFromBH(*blk.Head)

		blockhash, _ := bh.BlockSha()
		for _, tx := range blk.RelTxs {
			// Get author of bulletin via RPC call
			authOutpoint := tx.TxIn[0].PreviousOutPoint
			asyncRes := client.GetRawTransactionAsync(&authOutpoint.Hash)
			authorTx, err := asyncRes.Receive()
			if err != nil {
				return err
			}
			// This pubkeyscript defines the author of the post
			relScript := authorTx.MsgTx().TxOut[authOutpoint.Index].PkScript

			scriptClass, addrs, _, err := btcscript.ExtractPkScriptAddrs(relScript, activeNetParams)
			if err != nil {
				return err
			}
			if scriptClass != btcscript.PubKeyHashTy {
				return fmt.Errorf("Author script is not p2pkh")
			}
			// We know that the returned value is a P2PKH; therefore it must have
			// one address which is the author of the attached bulletin
			author := addrs[0].String()

			bltn, err := ahimsa.NewBulletin(tx, author, &blockhash)
			if err != nil {
				return err
			}
			if err := db.storeBulletin(bltn); err != nil {
				return err
			}
		}

		blk = blk.PrevBlock
	}
	return nil
}
<commit_msg>typo, clearer logging<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"path/filepath"

	"github.com/NSkelsey/btcbuilder"
	"github.com/NSkelsey/protocol/ahimsa"
	"github.com/NSkelsey/watchtower"
	"github.com/conformal/btcnet"
	"github.com/conformal/btcrpcclient"
	"github.com/conformal/btcscript"
	"github.com/conformal/btcutil"
	"github.com/conformal/btcwire"
	"github.com/jessevdk/go-flags"
)

var (
	appDataDir        = btcutil.AppDataDir("ahimsa", false)
	defaultConfigFile = filepath.Join(appDataDir, "ahimsa.conf")
	defaultDbName     = filepath.Join(appDataDir, "pubrecord.db")
	defaultBlockDir   = filepath.Join(btcutil.AppDataDir(".bitcoin", false), "testnet3/blocks")
	defaultNetwork    = "TestNet3"
	defaultNodeAddr   = "127.0.0.1:18333"
	defaultRPCAddr    = "127.0.0.1:18332"
	debug             = false
	// Sane defaults for a linux based OS
	cfg = &config{
		ConfigFile: defaultConfigFile,
		BlockDir:   defaultBlockDir,
		DbFile:     defaultDbName,
		NodeAddr:   defaultNodeAddr,
		NetName:    defaultNetwork,
		RPCAddr:    defaultRPCAddr,
		Rebuild:    false,
	}
)

// Application globals
var activeNetParams *btcnet.Params
var logger *log.Logger = log.New(os.Stdout, "", log.Ltime)

type config struct {
	ConfigFile  string `short:"C" long:"configfile" description:"Path to configuration file"`
	BlockDir    string `long:"blockdir" description:"Path to bitcoin blockdir"`
	DbFile      string `long:"dbname" description:"Name of the database file"`
	Rebuild     bool   `long:"rebuild" description:"Flag to rebuild the pubrecord db"`
	RPCAddr     string `long:"rpcaddr" description:"Address of bitcoin rpc endpoint to use"`
	RPCUser     string `long:"rpcuser" description:"RPC username"`
	RPCPassword string `long:"rpcpassword" description:"RPC password"`
	NodeAddr    string `long:"nodeaddr" description:"Address + port of the bitcoin node to connect to"`
	NetName     string `short:"n" long:"network" description:"The name of the network to use"`
	Debug       bool   `shodt:"d" long:"debug" description:"Debug flag for verbose error logging"`
	PrintHelp   bool   `short:"h" long:"help" description:"Prints out this message"`
}

func main() {
	// Parse command line args first then use file args
	parser := flags.NewParser(cfg, flags.None)
	_, err := parser.Parse()
	if err != nil {
		parser.WriteHelp(os.Stdout)
		logger.Fatal(err)
	}

	if cfg.PrintHelp {
		parser.WriteHelp(os.Stdout)
		os.Exit(0)
	}

	// Check to see if application files exist and create them if not
	_, err = os.Stat(appDataDir)
	if err != nil {
		makeDataDir()
	}

	err = flags.NewIniParser(parser).ParseFile(cfg.ConfigFile)
	if err != nil {
		logger.Println("No config file provided, using command line params")
	}

	activeNetParams, err = btcbuilder.NetParamsFromStr(cfg.NetName)
	if err != nil {
		logger.Fatal(err)
	}

	// Configure debug logger for verbose output
	if debug {
		logger = log.New(os.Stdout, "DEBUG\t", log.Ltime|log.Llongfile)
	}

	// Configure and create a RPC client
	connCfg := &btcrpcclient.ConnConfig{
		Host:         cfg.RPCAddr,
		User:         cfg.RPCUser,
		Pass:         cfg.RPCPassword,
		HttpPostMode: true,
		DisableTLS:   true,
	}
	rpcclient, err := btcrpcclient.New(connCfg, nil)
	if err != nil {
		logger.Fatal(err)
	}
	// Test rpc connection
	if err := rpcclient.Ping(); err != nil {
		logger.Println(err)
		msg := `
Connecting to the Bitcoin via RPC failed!! This may have been caused by one of the following:
1. Bitcoind is not running
2. The RPC server is not activated (server=1)
3. rpcuser and rpcpassword were not set
4. You are using Testnet3 settings for a Mainnet server or vice versa.
`
		fmt.Printf(msg)
		os.Exit(1)
	}

	rpcSubChan := make(chan *TxReq)

	// start a rpc command handler
	go authorlookup(rpcclient, rpcSubChan)

	fmt.Println(getBanner())
	// Load the db and find its current chain height
	db := loadDb(rpcclient)

	if err != nil {
		logger.Fatal(err)
	}
	curH := db.CurrentHeight()
	actualH, err := rpcclient.GetBlockCount()
	if err != nil {
		logger.Fatal(err)
	}

	// Configure the live network feed
	towerCfg := watchtower.TowerCfg{
		Addr:        cfg.NodeAddr,
		Net:         activeNetParams.Net,
		StartHeight: int(db.CurrentHeight()),
		Logger:      logger,
	}

	// If the database reports a height lower than the current height reported by
	// the bitcoin node but is within 500 blocks we can avoid redownloading the
	// whole chain. This is done at the network level with a getblocks msg for
	// any blocks we are missing. This is a relatively simple optimization and it
	// gives us 3 days of wiggle room before the whole chain must be validated
	// again.
	if actualH-curH > 0 {
		getblocks, err := makeBlockMsg(db)
		if err != nil {
			logger.Fatal(err)
		}
		towerCfg.ToSend = []btcwire.Message{getblocks}
	}

	// Start a watchtower instance and listen for new blocks
	txParser := txClosure(db, rpcSubChan)
	blockParser := blockClosure(db)

	watchtower.Create(towerCfg, txParser, blockParser)
}

func makeDataDir() {
	// Creates the application data dir initializing it with a config file that
	// is empty.

	// create dir
	perms := os.ModeDir | 0700
	if err := os.Mkdir(appDataDir, perms); err != nil {
		logger.Fatal(err)
	}

	// touch config file
	f, err := os.Create(cfg.ConfigFile)
	if err != nil {
		logger.Fatal(err)
	}
	if err := f.Close(); err != nil {
		logger.Fatal(err)
	}

	// touch db file
	f, err = os.Create(cfg.DbFile)
	if err != nil {
		logger.Fatal(err)
	}
	if err := f.Close(); err != nil {
		logger.Fatal(err)
	}
}

func loadDb(client *btcrpcclient.Client) *LiteDb {
	// Load the db from the file specified in config and get it to a usuable state
	// where ahimsad can add blocks from the network
	db, err := LoadDb(cfg.DbFile)
	if err != nil {
		logger.Fatal(err)
	}

	actualH, err := client.GetBlockCount()
	if err != nil {
		logger.Fatal(err)
	}

	curH := db.CurrentHeight()

	fmt.Printf("Block database heights [ahimsad: %d, bitcoind: %d]\n", curH, actualH)
	// Fudge factor
	if curH < actualH-499 || cfg.Rebuild {
		println("Creating DB")
		// init db
		db, err = InitDb(cfg.DbFile)
		if err != nil {
			logger.Fatal(err)
		}

		// get the tip of the longest valid chain
		tip, err := runBlockScan(cfg.BlockDir, db)
		if err != nil {
			logger.Fatal(err)
		}

		genBlk := walkBackwards(tip)
		err = storeChainBulletins(genBlk, db, client)
		if err != nil {
			logger.Fatal(err)
		}

	}
	return db
}

func storeChainBulletins(genBlock *Block, db *LiteDb, client *btcrpcclient.Client) error {
	// Stores all of the Bulletins we found in the blockchain into the sqlite db.
	// This is done iteratively and is not optimized in any way. We log errors as
	// we encounter them.
	chainHeight := genBlock.depth
	blks := make([]*Block, 0, 1000)
	var blk *Block = genBlock
	for {
		// Walk forwards through the blocks
		if blk.NextBlock == nil {
			break
		}

		if len(blks) >= 1000 {
			err := db.BatchInsertBH(blks, chainHeight)
			if err != nil {
				return err
			}
			blks = make([]*Block, 0, 1000)
		}
		blks = append(blks, blk)

		blk = blk.NextBlock
	}

	// Insert remaining blocks
	err := db.BatchInsertBH(blks, chainHeight)
	if err != nil {
		return err
	}

	// We are now back at the tip
	//var tip *Block = blk
	for {
		// walk backwards through blocks
		if blk.Hash == genesisHash {
			break
		}

		var bh *btcwire.BlockHeader
		bh = btcBHFromBH(*blk.Head)

		blockhash, _ := bh.BlockSha()
		for _, tx := range blk.RelTxs {
			// Get author of bulletin via RPC call
			authOutpoint := tx.TxIn[0].PreviousOutPoint
			asyncRes := client.GetRawTransactionAsync(&authOutpoint.Hash)
			authorTx, err := asyncRes.Receive()
			if err != nil {
				return err
			}
			// This pubkeyscript defines the author of the post
			relScript := authorTx.MsgTx().TxOut[authOutpoint.Index].PkScript

			scriptClass, addrs, _, err := btcscript.ExtractPkScriptAddrs(relScript, activeNetParams)
			if err != nil {
				return err
			}
			if scriptClass != btcscript.PubKeyHashTy {
				return fmt.Errorf("Author script is not p2pkh")
			}
			// We know that the returned value is a P2PKH; therefore it must have
			// one address which is the author of the attached bulletin
			author := addrs[0].String()

			bltn, err := ahimsa.NewBulletin(tx, author, &blockhash)
			if err != nil {
				return err
			}
			if err := db.storeBulletin(bltn); err != nil {
				return err
			}
		}

		blk = blk.PrevBlock
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/garyburd/redigo/redis"
)

var (
	Trace               *log.Logger
	Info                *log.Logger
	Warning             *log.Logger
	Error               *log.Logger
	Status              *RedisStatus
	CmdLimit            int
	Frequency           int
	MonitorSampleLength int
	SlowlogSize         int
	RedisHost           string
	RedisPort           int
)

func init() {

	Trace = log.New(os.Stdout,
		"TRACE: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Info = log.New(os.Stdout,
		"INFO: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Warning = log.New(os.Stderr,
		"WARNING: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Error = log.New(os.Stderr,
		"ERROR: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Status = &RedisStatus{Info: make(map[string]interface{}), Slowlog: make([]string, 0), MonitorSample: make([]*MonitorCmd, 0)}
	flag.IntVar(&CmdLimit, "cmdlimit", 100, "number of commands the monitor will store")
	flag.IntVar(&Frequency, "frequency", 10000, "Number of miliseconds to delay between samples info, slowlog")
	flag.IntVar(&MonitorSampleLength, "monsamplen", 1000, "Length of miliseconds that the monitor is sampled (0 will be coninuous however this is very costly to performance)")
	flag.IntVar(&SlowlogSize, "slogsize", 10, "slowlog size")
	flag.StringVar(&RedisHost, "h", "127.0.0.1", "redis host ")
	flag.IntVar(&RedisPort, "p", 6379, "redis port")
}

type CommandStats struct {
	CommandCounts map[string]int64 `json:"command_counts"`
	BadCommands   map[string]int64 `json:"bad_commands"`
}
type RedisStatus struct {
	CommandStats  CommandStats           `json:"stats"`
	Slowlog       []string               `json:"slowlog"`
	Info          map[string]interface{} `json:"info"`
	MonitorSample []*MonitorCmd          `json:"monitor_sample"`
	stats         *Stats
}

func rcon() (redis.Conn, error) {
	return redis.Dial("tcp", fmt.Sprintf("%s:%d", RedisHost, RedisPort))
}

func main() {
	go func() {
		for {
			stat := &RedisStatus{Info: make(map[string]interface{}), Slowlog: make([]string, 0), MonitorSample: make([]*MonitorCmd, 0), CommandStats: CommandStats{CommandCounts: make(map[string]int64), BadCommands: make(map[string]int64)}, stats: NewStats()}
			c, err := rcon()
			if err != nil {
				Error.Printf("Failed to make connection %s", err)
				continue
			}
			go SampleMonitor(stat)
			_, err = SampleInfo(c, stat)
			if err != nil {
				Error.Println(err)
			}
			_, err = SampleSlowlog(c, stat)
			if err != nil {
				Error.Printf("Error with slowlog %s", err)
			}
			Status = stat
			c.Close()
			time.Sleep(time.Duration(Frequency) * time.Second)
		}
	}()

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		enc := json.NewEncoder(w)
		// TODO: Fix this just added before hack session timed out
		Status.CommandStats.CommandCounts = Status.stats.Counts()
		Status.CommandStats.BadCommands = Status.stats.BadCmds()
		enc.Encode(Status)
	})
	log.Fatal(http.ListenAndServe(":8000", nil))
}
<commit_msg>Don't rest stats every frequency<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/garyburd/redigo/redis"
)

var (
	Trace               *log.Logger
	Info                *log.Logger
	Warning             *log.Logger
	Error               *log.Logger
	Status              *RedisStatus
	CmdLimit            int
	Frequency           int
	MonitorSampleLength int
	SlowlogSize         int
	RedisHost           string
	RedisPort           int
)

func init() {

	Trace = log.New(os.Stdout,
		"TRACE: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Info = log.New(os.Stdout,
		"INFO: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Warning = log.New(os.Stderr,
		"WARNING: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Error = log.New(os.Stderr,
		"ERROR: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Status := &RedisStatus{Info: make(map[string]interface{}), Slowlog: make([]string, 0), MonitorSample: make([]*MonitorCmd, 0), CommandStats: CommandStats{CommandCounts: make(map[string]int64), BadCommands: make(map[string]int64)}, stats: NewStats()}
	flag.IntVar(&CmdLimit, "cmdlimit", 100, "number of commands the monitor will store")
	flag.IntVar(&Frequency, "frequency", 10000, "Number of miliseconds to delay between samples info, slowlog")
	flag.IntVar(&MonitorSampleLength, "monsamplen", 1000, "Length of miliseconds that the monitor is sampled (0 will be coninuous however this is very costly to performance)")
	flag.IntVar(&SlowlogSize, "slogsize", 10, "slowlog size")
	flag.StringVar(&RedisHost, "h", "127.0.0.1", "redis host ")
	flag.IntVar(&RedisPort, "p", 6379, "redis port")
}

type CommandStats struct {
	CommandCounts map[string]int64 `json:"command_counts"`
	BadCommands   map[string]int64 `json:"bad_commands"`
}
type RedisStatus struct {
	CommandStats  CommandStats           `json:"stats"`
	Slowlog       []string               `json:"slowlog"`
	Info          map[string]interface{} `json:"info"`
	MonitorSample []*MonitorCmd          `json:"monitor_sample"`
	stats         *Stats
}

func rcon() (redis.Conn, error) {
	return redis.Dial("tcp", fmt.Sprintf("%s:%d", RedisHost, RedisPort))
}

func main() {
	go func() {
		for {
			c, err := rcon()
			if err != nil {
				Error.Printf("Failed to make connection %s", err)
				continue
			}
			go SampleMonitor(Status)
			_, err = SampleInfo(c, Status)
			if err != nil {
				Error.Println(err)
			}
			_, err = SampleSlowlog(c, Status)
			if err != nil {
				Error.Printf("Error with slowlog %s", err)
			}
			c.Close()
			time.Sleep(time.Duration(Frequency) * time.Second)
		}
	}()

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		enc := json.NewEncoder(w)
		// TODO: Fix this just added before hack session timed out
		Status.CommandStats.CommandCounts = Status.stats.Counts()
		Status.CommandStats.BadCommands = Status.stats.BadCmds()
		enc.Encode(Status)
	})
	log.Fatal(http.ListenAndServe(":8000", nil))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/rand"
	"encoding/binary"
	"flag"
	"fmt"
	"golang.org/x/crypto/nacl/box"
	"golang.org/x/crypto/nacl/secretbox"
	"io"
	"log"
	"net"
	"os"
)

// header encodes encrypted message length and nonce information.
// It is sent unencrypted at the start of the message.
// The length is used to ensure we read enough data to decrypt the
// fix length message.
type header struct {
	Length uint64
	Nonce  [24]byte
}

type secureReader struct {
	src io.Reader
	key *[32]byte
	buf []byte
}

type secureWriter struct {
	dst io.Writer
	key *[32]byte
}

func (p secureReader) Read(b []byte) (int, error) {
	// Check to see if there are still remnants of the last message to
	// read
	if p.buf == nil {
		// Read header from stream
		var h header
		if err := binary.Read(p.src, binary.LittleEndian, &h); err != nil {
			return 0, err
		}

		// Allocate a buffer to contain the encrypted and decrypted message
		p.buf = make([]byte, h.Length-secretbox.Overhead)
		tmp := make([]byte, h.Length)

		// Read the encrypted message
		if _, err := io.ReadFull(p.src, tmp); err != nil {
			return 0, err
		}

		// Decrypt message and check it is authentic
		_, auth := secretbox.Open(p.buf[:0], tmp[:], &h.Nonce, p.key)
		if !auth {
			return 0, fmt.Errorf("Message failed authentication")
		}
	}

	// Copy the result into the output buffer, leaving a partial result
	// in the buffer if needed
	size := copy(b, p.buf)
	if size == len(p.buf) {
		p.buf = nil
	} else {
		p.buf = p.buf[size:]
	}
	return size, nil
}

func (p secureWriter) Write(b []byte) (int, error) {
	// Encode header containing length and randomly generated nonce
	var h header
	h.Length = uint64(len(b) + secretbox.Overhead)
	if _, err := io.ReadFull(rand.Reader, h.Nonce[:]); err != nil {
		return 0, err
	}

	// Write header out
	hbuf := new(bytes.Buffer)
	binary.Write(hbuf, binary.LittleEndian, &h)
	if _, err := p.dst.Write(hbuf.Bytes()); err != nil {
		return 0, err
	}

	// Encrypt and send the message
	tmp := make([]byte, h.Length)
	secretbox.Seal(tmp[:0], b, &h.Nonce, p.key)
	return p.dst.Write(tmp)
}

// NewSecureReader instantiates a new SecureReader
func NewSecureReader(r io.Reader, priv, pub *[32]byte) io.Reader {
	sr := secureReader{src: r, key: new([32]byte), buf: nil}
	box.Precompute(sr.key, pub, priv)
	return sr
}

// NewSecureWriter instantiates a new SecureWriter
func NewSecureWriter(w io.Writer, priv, pub *[32]byte) io.Writer {
	sw := secureWriter{dst: w, key: new([32]byte)}
	box.Precompute(sw.key, pub, priv)
	return sw
}

// Generate a public/private key pair.
// Swap public keys over ReadWriter.
func swapKeys(rw io.ReadWriter) (priv, peer *[32]byte, err error) {
	// Generate our public/private key pair
	pub, priv, err := box.GenerateKey(rand.Reader)
	if err != nil {
		return nil, nil, err
	}

	// Send our public key
	senderr := make(chan error)
	go func() {
		_, err := rw.Write(pub[:])
		senderr <- err
	}()

	// Receive their public key
	peer = new([32]byte)
	_, err = io.ReadFull(rw, peer[:])
	if err != nil {
		return nil, nil, err
	}

	// Wait for our send to complete
	if err = <-senderr; err != nil {
		return nil, nil, err
	}

	return
}

// Dial generates a private/public key pair,
// connects to the server, perform the handshake
// and return a reader/writer.
func Dial(addr string) (io.ReadWriteCloser, error) {
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}

	priv, pub, err := swapKeys(conn)
	if err != nil {
		return nil, err
	}

	return struct {
		io.Reader
		io.Writer
		io.Closer
	}{
		NewSecureReader(conn, priv, pub),
		NewSecureWriter(conn, priv, pub),
		conn,
	}, nil
}

func connect(conn net.Conn) {
	defer conn.Close()

	priv, pub, err := swapKeys(conn)
	if err != nil {
		log.Fatal(err)
	}

	r := NewSecureReader(conn, priv, pub)
	w := NewSecureWriter(conn, priv, pub)

	buf := make([]byte, 2048)
	for {
		n, err := r.Read(buf)
		if err != nil && err != io.EOF {
			log.Fatal(err)
		}
		if n > 0 {
			if _, err := w.Write(buf[:n]); err != nil {
				log.Fatal(err)
			}
		}
		if err == io.EOF {
			return
		}
	}
}

// Serve starts a secure echo server on the given listener.
func Serve(l net.Listener) error {
	for {
		conn, err := l.Accept()
		if err != nil {
			return err
		}
		go connect(conn)
	}
}

func init() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

func main() {
	port := flag.Int("l", 0, "Listen mode. Specify port")
	flag.Parse()

	// Server mode
	if *port != 0 {
		l, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
		if err != nil {
			log.Fatal(err)
		}
		defer l.Close()
		log.Fatal(Serve(l))
	}

	// Client mode
	if len(os.Args) != 3 {
		log.Fatalf("Usage: %s <port> <message>", os.Args[0])
	}
	conn, err := Dial("localhost:" + os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	if _, err := conn.Write([]byte(os.Args[2])); err != nil {
		log.Fatal(err)
	}
	buf := make([]byte, len(os.Args[2]))
	n, err := conn.Read(buf)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s\n", buf[:n])
}
<commit_msg>Modify echo server to use io.Copy.<commit_after>package main

import (
	"bytes"
	"crypto/rand"
	"encoding/binary"
	"flag"
	"fmt"
	"golang.org/x/crypto/nacl/box"
	"golang.org/x/crypto/nacl/secretbox"
	"io"
	"log"
	"net"
	"os"
)

// header encodes encrypted message length and nonce information.
// It is sent unencrypted at the start of the message.
// The length is used to ensure we read enough data to decrypt the
// fix length message.
type header struct {
	Length uint64
	Nonce  [24]byte
}

type secureReader struct {
	src io.Reader
	key *[32]byte
	buf []byte
}

type secureWriter struct {
	dst io.Writer
	key *[32]byte
}

func (p secureReader) Read(b []byte) (int, error) {
	// Check to see if there are still remnants of the last message to
	// read
	if p.buf == nil {
		// Read header from stream
		var h header
		if err := binary.Read(p.src, binary.LittleEndian, &h); err != nil {
			return 0, err
		}

		// Allocate a buffer to contain the encrypted and decrypted message
		p.buf = make([]byte, h.Length-secretbox.Overhead)
		tmp := make([]byte, h.Length)

		// Read the encrypted message
		if _, err := io.ReadFull(p.src, tmp); err != nil {
			return 0, err
		}

		// Decrypt message and check it is authentic
		_, auth := secretbox.Open(p.buf[:0], tmp[:], &h.Nonce, p.key)
		if !auth {
			return 0, fmt.Errorf("Message failed authentication")
		}
	}

	// Copy the result into the output buffer, leaving a partial result
	// in the buffer if needed
	size := copy(b, p.buf)
	if size == len(p.buf) {
		p.buf = nil
	} else {
		p.buf = p.buf[size:]
	}
	return size, nil
}

func (p secureWriter) Write(b []byte) (int, error) {
	// Encode header containing length and randomly generated nonce
	var h header
	h.Length = uint64(len(b) + secretbox.Overhead)
	if _, err := io.ReadFull(rand.Reader, h.Nonce[:]); err != nil {
		return 0, err
	}

	// Write header out
	hbuf := new(bytes.Buffer)
	binary.Write(hbuf, binary.LittleEndian, &h)
	if _, err := p.dst.Write(hbuf.Bytes()); err != nil {
		return 0, err
	}

	// Encrypt and send the message
	tmp := make([]byte, h.Length)
	secretbox.Seal(tmp[:0], b, &h.Nonce, p.key)
	if _, err := p.dst.Write(tmp); err != nil {
		return 0, err
	}
	return len(b), nil
}

// NewSecureReader instantiates a new SecureReader
func NewSecureReader(r io.Reader, priv, pub *[32]byte) io.Reader {
	sr := secureReader{src: r, key: new([32]byte), buf: nil}
	box.Precompute(sr.key, pub, priv)
	return sr
}

// NewSecureWriter instantiates a new SecureWriter
func NewSecureWriter(w io.Writer, priv, pub *[32]byte) io.Writer {
	sw := secureWriter{dst: w, key: new([32]byte)}
	box.Precompute(sw.key, pub, priv)
	return sw
}

// Generate a public/private key pair.
// Swap public keys over ReadWriter.
func swapKeys(rw io.ReadWriter) (priv, peer *[32]byte, err error) {
	// Generate our public/private key pair
	pub, priv, err := box.GenerateKey(rand.Reader)
	if err != nil {
		return nil, nil, err
	}

	// Send our public key
	senderr := make(chan error)
	go func() {
		_, err := rw.Write(pub[:])
		senderr <- err
	}()

	// Receive their public key
	peer = new([32]byte)
	_, err = io.ReadFull(rw, peer[:])
	if err != nil {
		return nil, nil, err
	}

	// Wait for our send to complete
	if err = <-senderr; err != nil {
		return nil, nil, err
	}

	return
}

// Dial generates a private/public key pair,
// connects to the server, perform the handshake
// and return a reader/writer.
func Dial(addr string) (io.ReadWriteCloser, error) {
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}

	priv, pub, err := swapKeys(conn)
	if err != nil {
		return nil, err
	}

	return struct {
		io.Reader
		io.Writer
		io.Closer
	}{
		NewSecureReader(conn, priv, pub),
		NewSecureWriter(conn, priv, pub),
		conn,
	}, nil
}

func connect(conn net.Conn) {
	defer conn.Close()

	priv, pub, err := swapKeys(conn)
	if err != nil {
		log.Fatal(err)
	}

	r := NewSecureReader(conn, priv, pub)
	w := NewSecureWriter(conn, priv, pub)

	if _, err := io.Copy(w, r); err != nil {
		log.Fatal(err)
	}
}

// Serve starts a secure echo server on the given listener.
func Serve(l net.Listener) error {
	for {
		conn, err := l.Accept()
		if err != nil {
			return err
		}
		go connect(conn)
	}
}

func init() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

func main() {
	port := flag.Int("l", 0, "Listen mode. Specify port")
	flag.Parse()

	// Server mode
	if *port != 0 {
		l, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
		if err != nil {
			log.Fatal(err)
		}
		defer l.Close()
		log.Fatal(Serve(l))
	}

	// Client mode
	if len(os.Args) != 3 {
		log.Fatalf("Usage: %s <port> <message>", os.Args[0])
	}
	conn, err := Dial("localhost:" + os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	if _, err := conn.Write([]byte(os.Args[2])); err != nil {
		log.Fatal(err)
	}
	buf := make([]byte, len(os.Args[2]))
	n, err := conn.Read(buf)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s\n", buf[:n])
}
<|endoftext|>