<commit_before>/*
Copyright 2013 Tamás Gulácsi

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/tgulacsi/goracle/oracle"
)

// TestGen tests the generation - for this, it needs a dsn with privileges
// if you get "ORA-01031: insufficient privileges", then you need
// GRANT CREATE PROCEDURE TO username;
func TestGenSimple(t *testing.T) {
	conn := getConnection(t)

	cu := conn.NewCursor()
	defer cu.Close()

	build(t)
	outFn := generateAndBuild(t, "SIMPLE_")

	for i, todo := range [][3]string{
		{"simple_char_in", `{"txt": "abraka dabra"}`, `{}`},
		{"simple_char_out", `{}`, `{"ret":"A"}`},
		{"simple_num_in", `{"num": 33}`, `{}`},
		{"simple_num_out", `{}`, `{"ret":0.6666666666666665}`},
		{"simple_date_in", `{"dat": "2013-12-25T21:15:00+01:00"}`, `{}`},
		{"simple_date_out", `{}`, `{"ret":"{{NOW}}"}`}, // 5.
		{"simple_char_in_char_ret", `{"txt": "abraka dabra"}`, `{"ret":"Typ=1 Len=12: 97,98,114,97,107,97,32,100,97,98,114,97"}`},
		{"simple_all_inout",
			`{"txt1": "abraka", "txt3": "A", "int1": -1, "int3": -2, "num1": 0.1, "num3": 0.3, "dt1": null, "dt3": "2014-01-03T00:00:00+02:00"}`,
			`{"txt2":"abraka#","int2":0,"num2":0.4333333333333333,"dt2":"0000-01-31T00:00:00+02:00","txt3":"A#","int3":-1,"num3":1.3,"dt3":"2014-02-03T00:00:00+01:00"}`},
		{"simple_nums_count", `{"nums":[1,2,3,4.4]}`, `{"ret":4}`},
		{"simple_sum_nums", `{"nums":[1,2,3.3]}`, `{"outnums":[2,4,6.6000000000000005],"ret":6.3}`},
	} {
		got := runTest(t, outFn, "-connect="+*flagConnect, "TST_oracall."+todo[0], todo[1])
		if strings.Index(todo[2], "{{NOW}}") >= 0 {
			todo[2] = strings.Replace(todo[2], "{{NOW}}", time.Now().Format(time.RFC3339), -1)
		}
		if strings.TrimSpace(got) != todo[2] {
			t.Errorf("%d. awaited\n\t%s\ngot\n\t%s", i, todo[2], got)
		}
	}
}

func TestGenRec(t *testing.T) {
	conn := getConnection(t)

	cu := conn.NewCursor()
	defer cu.Close()

	build(t)
	outFn := generateAndBuild(t, "REC_")

	for i, todo := range [][3]string{
		{"rec_in", `{"rec":{"num":33,"text":"xxx","dt":"2006-08-26T00:00:00Z01:00"}}`, `{"ret":"33;;xxx"}`},
		{"rec_tab_in", `{"tab":[{"num":1,"text":"A","dt":"2006-08-26T00:00:00Z01:00"},{"num":2,"text":"B"}]}`,
			`{"ret":"1;;A` + "\n" + `2;;B"}`},
	} {
		got := runTest(t, outFn, "-connect="+*flagConnect, "TST_oracall."+todo[0], todo[1])
		if strings.Index(todo[2], "{{NOW}}") >= 0 {
			todo[2] = strings.Replace(todo[2], "{{NOW}}", time.Now().Format(time.RFC3339), -1)
		}
		if strings.TrimSpace(got) != todo[2] {
			t.Errorf("%d. awaited\n\t%s\ngot\n\t%s", i, todo[2], got)
		}
	}
}

func createStoredProc(t *testing.T) {
	cu := conn.NewCursor()
	defer cu.Close()

	err := cu.Execute(`CREATE OR REPLACE PACKAGE TST_oracall AS
TYPE num_tab_typ IS TABLE OF NUMBER INDEX BY BINARY_INTEGER;

TYPE mix_rec_typ IS RECORD (num NUMBER, dt DATE, text VARCHAR2(1000));
TYPE mix_tab_typ IS TABLE OF mix_rec_typ INDEX BY BINARY_INTEGER;

PROCEDURE simple_char_in(txt IN VARCHAR2);
FUNCTION simple_char_out RETURN VARCHAR2;
PROCEDURE simple_num_in(num IN NUMBER);
FUNCTION simple_num_out RETURN NUMBER;
PROCEDURE simple_date_in(dat IN DATE);
FUNCTION simple_date_out RETURN DATE;
FUNCTION simple_char_in_char_ret(txt IN VARCHAR2) RETURN VARCHAR2;
PROCEDURE simple_all_inout(
    txt1 IN VARCHAR2, int1 IN PLS_INTEGER, num1 IN NUMBER, dt1 IN DATE,
    txt2 OUT VARCHAR2, int2 OUT PLS_INTEGER, num2 OUT NUMBER, dt2 OUT DATE,
    txt3 IN OUT VARCHAR2, int3 IN OUT PLS_INTEGER, num3 IN OUT NUMBER, dt3 IN OUT DATE);

FUNCTION simple_nums_count(nums IN num_tab_typ) RETURN PLS_INTEGER;
FUNCTION simple_sum_nums(nums IN num_tab_typ, outnums OUT num_tab_typ) RETURN NUMBER;

FUNCTION rec_in(rec IN mix_rec_typ) RETURN VARCHAR2;
FUNCTION rec_tab_in(tab IN mix_tab_typ) RETURN VARCHAR2;
END TST_oracall;
    `, nil, nil)
	if err != nil {
		t.Fatalf("error creating package head: %v", err)
	}
	if err = cu.Execute(`CREATE OR REPLACE PACKAGE BODY TST_oracall AS
PROCEDURE simple_char_in(txt IN VARCHAR2) IS
  v_txt VARCHAR2(1000) := SUBSTR(txt, 1, 100);
BEGIN NULL; END simple_char_in;
FUNCTION simple_char_out RETURN VArCHAR2 IS BEGIN RETURN('A'); END simple_char_out;

PROCEDURE simple_num_in(num IN NUMBER) IS
  v_num NUMBER := num;
BEGIN NULL; END simple_num_in;

PROCEDURE simple_date_in(dat IN DATE) IS
  v_dat DATE := dat;
BEGIN NULL; END simple_date_in;

FUNCTION simple_char_in_char_ret(txt IN VARCHAR2) RETURN VARCHAR2 IS
  v_txt CONSTANT VARCHAR2(4000) := SUBSTR(txt, 1, 4000);
  v_ret VARCHAR2(4000);
BEGIN
  SELECT DUMP(txt) INTO v_ret FROM DUAL;
  RETURN v_ret;
END simple_char_in_char_ret;

FUNCTION simple_date_out RETURN DATE IS BEGIN RETURN SYSDATE; END simple_date_out;
FUNCTION simple_num_out RETURN NUMBER IS BEGIN RETURN 2/3; END simple_num_out;

PROCEDURE simple_all_inout(
    txt1 IN VARCHAR2, int1 IN PLS_INTEGER, num1 IN NUMBER, dt1 IN DATE,
    txt2 OUT VARCHAR2, int2 OUT PLS_INTEGER, num2 OUT NUMBER, dt2 OUT DATE,
    txt3 IN OUT VARCHAR2,
    int3 IN OUT PLS_INTEGER, num3 IN OUT NUMBER, dt3 IN OUT DATE) IS
BEGIN
  txt2 := txt1||'#'; int2 := NVL(int1, 0) + 1;
  num2 := NVL(num1, 0) + 1/3; dt2 := ADD_MONTHS(NVL(dt1, SYSDATE), 1);
  txt3 := txt3||'#'; int3 := NVL(int3, 0) + 1;
  num3 := NVL(num3, 0) + 1; dt3 := ADD_MONTHS(NVL(dt3, SYSDATE), 1);
END simple_all_inout;

FUNCTION simple_nums_count(nums IN num_tab_typ) RETURN PLS_INTEGER IS
BEGIN
  RETURN nums.COUNT;
END simple_nums_count;

FUNCTION simple_sum_nums(nums IN num_tab_typ, outnums OUT num_tab_typ) RETURN NUMBER IS
  v_idx PLS_INTEGER;
  s NUMBER := 0;
BEGIN
  outnums.DELETE;
  v_idx := nums.FIRST;
  WHILE v_idx IS NOT NULL LOOP
    s := NVL(s, 0) + NVL(nums(v_idx), 0);
    outnums(v_idx) := NVL(nums(v_idx), 0) * 2;
    v_idx := nums.NEXT(v_idx);
  END LOOP;
  RETURN(s);
END simple_sum_nums;

FUNCTION rec_in(rec IN mix_rec_typ) RETURN VARCHAR2 IS
BEGIN
  RETURN rec.num||';"'||TO_CHAR(rec.dt, 'YYYY-MM-DD HH24:MI:SS')||'";"'||rec.text||'"';
END rec_in;

FUNCTION rec_tab_in(tab IN mix_tab_typ) RETURN VARCHAR2 IS
  i PLS_INTEGER;
  text VARCHAR2(32767);
BEGIN
  i := tab.FIRST;
  WHILE i IS NOT NULL LOOP
    text := text||CHR(10)||SUBSTR(
              tab(i).num||';"'||TO_CHAR(tab(i).dt, 'YYYY-MM-DD HH24:MI:SS')
              ||'";"'||tab(i).text||'"',
              1, GREATEST(0, 32767-LENGTH(text)-1));
    EXIT WHEN LENGTH(text) >= 32767;
    i := tab.NEXT(i);
  END LOOP;
  RETURN(text);
END rec_tab_in;

FUNCTION nums_count(nums IN num_tab_typ) RETURN PLS_INTEGER IS
BEGIN
  RETURN nums.COUNT;
END nums_count;

FUNCTION sum_nums(nums IN num_tab_typ, outnums OUT num_tab_typ) RETURN NUMBER IS
  v_idx PLS_INTEGER;
  s NUMBER := 0;
BEGIN
  outnums.DELETE;
  v_idx := nums.FIRST;
  WHILE v_idx IS NOT NULL LOOP
    s := NVL(s, 0) + NVL(nums(v_idx), 0);
    outnums(v_idx) := NVL(nums(v_idx), 0) * 2;
    v_idx := nums.NEXT(v_idx);
  END LOOP;
  RETURN(s);
END sum_nums;

END TST_oracall;
    `, nil, nil); err != nil {
		t.Fatalf("error creating package body: %v", err)
	}
	if err = cu.Execute("SELECT text FROM user_errors WHERE name = :1", []interface{}{"TST_ORACALL"}, nil); err != nil {
		t.Fatalf("error querying errors: %v", err)
	}
	rows, err := cu.FetchAll()
	if err != nil {
		t.Fatalf("error fetching errors: %v", err)
	}
	if len(rows) > 0 {
		errTexts := make([]string, len(rows))
		for i := range rows {
			errTexts[i] = rows[i][0].(string)
		}
		t.Fatalf("error with package: %s", strings.Join(errTexts, "\n"))
	}
}
func runCommand(t *testing.T, prog string, args ...string) {
	out, err := exec.Command(prog, args...).CombinedOutput()
	if err != nil {
		t.Errorf("error '%q %s': %v\n%s", prog, args, err, out)
		t.FailNow()
	} else {
		t.Logf("%q %s:\n%s", prog, args, out)
	}
}

func build(t *testing.T) {
	buildOnce.Do(func() {
		createStoredProc(t)
		runCommand(t, "go", "build")
	})
}

func generateAndBuild(t *testing.T, prefix string) (outFn string) {
	runCommand(t, "sh", "-c",
		"./oracall -F -connect='"+*flagConnect+
			"' TST_ORACALL."+strings.ToUpper(prefix)+"%"+
			" >./testdata/integration_test/generated_functions.go")

	if outFh, err := ioutil.TempFile("", "oracall-integration_test"); err != nil {
		t.Errorf("cannot create temp file: %v", err)
		t.FailNow()
	} else {
		outFn = outFh.Name()
		outFh.Close()
	}
	os.Remove(outFn)
	runCommand(t, "go", "build", "-o="+outFn, "./testdata/integration_test")
	return
}

var errBuf = bytes.NewBuffer(make([]byte, 0, 512))

func runTest(t *testing.T, prog string, args ...string) string {
	c := exec.Command(prog, args...)
	errBuf.Reset()
	c.Stderr = errBuf
	out, err := c.Output()
	if err != nil {
		t.Errorf("error '%q %s': %v\n%s", prog, args, err, errBuf)
		t.FailNow()
	} else {
		t.Logf("%q %s:\n%s\n%s", prog, args, out, errBuf)
	}
	return string(out)
}

//var dsn = flag.String("connect", "", "Oracle DSN (user/passw@sid)")
var dbg = flag.Bool("debug", false, "print debug messages?")
var buildOnce sync.Once

func init() {
	flag.Parse()
}

var conn oracle.Connection

func getConnection(t *testing.T) oracle.Connection {
	if conn.IsConnected() {
		return conn
	}

	if !(flagConnect != nil && *flagConnect != "") {
		t.Logf("cannot test connection without dsn!")
		t.FailNow()
	}
	user, passw, sid := oracle.SplitDSN(*flagConnect)
	var err error
	conn, err = oracle.NewConnection(user, passw, sid, false)
	if err != nil {
		log.Panicf("error creating connection to %s: %s", *flagConnect, err)
	}
	if err = conn.Connect(0, false); err != nil {
		log.Panicf("error connecting: %s", err)
	}
	return conn
}
<commit_msg>fix rec_tab_in test<commit_after>/*
Copyright 2013 Tamás Gulácsi

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/tgulacsi/goracle/oracle"
)

// TestGen tests the generation - for this, it needs a dsn with privileges
// if you get "ORA-01031: insufficient privileges", then you need
// GRANT CREATE PROCEDURE TO username;
func TestGenSimple(t *testing.T) {
	conn := getConnection(t)

	cu := conn.NewCursor()
	defer cu.Close()

	build(t)
	outFn := generateAndBuild(t, "SIMPLE_")

	for i, todo := range [][3]string{
		{"simple_char_in", `{"txt": "abraka dabra"}`, `{}`},
		{"simple_char_out", `{}`, `{"ret":"A"}`},
		{"simple_num_in", `{"num": 33}`, `{}`},
		{"simple_num_out", `{}`, `{"ret":0.6666666666666665}`},
		{"simple_date_in", `{"dat": "2013-12-25T21:15:00+01:00"}`, `{}`},
		{"simple_date_out", `{}`, `{"ret":"{{NOW}}"}`}, // 5.
		{"simple_char_in_char_ret", `{"txt": "abraka dabra"}`, `{"ret":"Typ=1 Len=12: 97,98,114,97,107,97,32,100,97,98,114,97"}`},
		{"simple_all_inout",
			`{"txt1": "abraka", "txt3": "A", "int1": -1, "int3": -2, "num1": 0.1, "num3": 0.3, "dt1": null, "dt3": "2014-01-03T00:00:00+02:00"}`,
			`{"txt2":"abraka#","int2":0,"num2":0.4333333333333333,"dt2":"0000-01-31T00:00:00+02:00","txt3":"A#","int3":-1,"num3":1.3,"dt3":"2014-02-03T00:00:00+01:00"}`},
		{"simple_nums_count", `{"nums":[1,2,3,4.4]}`, `{"ret":4}`},
		{"simple_sum_nums", `{"nums":[1,2,3.3]}`, `{"outnums":[2,4,6.6000000000000005],"ret":6.3}`},
	} {
		got := runTest(t, outFn, "-connect="+*flagConnect, "TST_oracall."+todo[0], todo[1])
		if strings.Index(todo[2], "{{NOW}}") >= 0 {
			todo[2] = strings.Replace(todo[2], "{{NOW}}", time.Now().Format(time.RFC3339), -1)
		}
		if strings.TrimSpace(got) != todo[2] {
			t.Errorf("%d. awaited\n\t%s\ngot\n\t%s", i, todo[2], got)
		}
	}
}

func TestGenRec(t *testing.T) {
	conn := getConnection(t)

	cu := conn.NewCursor()
	defer cu.Close()

	build(t)
	outFn := generateAndBuild(t, "REC_")

	for i, todo := range [][3]string{
		{"rec_in", `{"rec":{"num":33,"text":"xxx","dt":"2006-08-26T00:00:00+01:00"}}`,
			`{"ret":"33;\"2006-08-26 00:00:00\";\"xxx\""}`},
		{"rec_tab_in", `{"tab":[{"num":1,"text":"A","dt":"2006-08-26T00:00:00+01:00"},{"num":2,"text":"B"},{"num":3,"text":"C"}]}`,
			`{"ret":"\n1;\"2006-08-26 00:00:00\";\"A\"\n2;\"0001-01-01 00:00:00\";\"B\"\n3;\"0001-01-01 00:00:00\";\"C\""}`},
	} {
		got := runTest(t, outFn, "-connect="+*flagConnect, "TST_oracall."+todo[0], todo[1])
		if strings.Index(todo[2], "{{NOW}}") >= 0 {
			todo[2] = strings.Replace(todo[2], "{{NOW}}", time.Now().Format(time.RFC3339), -1)
		}
		if strings.TrimSpace(got) != todo[2] {
			t.Errorf("%d. awaited\n\t%s\ngot\n\t%s", i, todo[2], got)
		}
	}
}

func createStoredProc(t *testing.T) {
	cu := conn.NewCursor()
	defer cu.Close()

	err := cu.Execute(`CREATE OR REPLACE PACKAGE TST_oracall AS
TYPE num_tab_typ IS TABLE OF NUMBER INDEX BY BINARY_INTEGER;

TYPE mix_rec_typ IS RECORD (num NUMBER, dt DATE, text VARCHAR2(1000));
TYPE mix_tab_typ IS TABLE OF mix_rec_typ INDEX BY BINARY_INTEGER;

PROCEDURE simple_char_in(txt IN VARCHAR2);
FUNCTION simple_char_out RETURN VARCHAR2;
PROCEDURE simple_num_in(num IN NUMBER);
FUNCTION simple_num_out RETURN NUMBER;
PROCEDURE simple_date_in(dat IN DATE);
FUNCTION simple_date_out RETURN DATE;
FUNCTION simple_char_in_char_ret(txt IN VARCHAR2) RETURN VARCHAR2;
PROCEDURE simple_all_inout(
    txt1 IN VARCHAR2, int1 IN PLS_INTEGER, num1 IN NUMBER, dt1 IN DATE,
    txt2 OUT VARCHAR2, int2 OUT PLS_INTEGER, num2 OUT NUMBER, dt2 OUT DATE,
    txt3 IN OUT VARCHAR2, int3 IN OUT PLS_INTEGER, num3 IN OUT NUMBER, dt3 IN OUT DATE);

FUNCTION simple_nums_count(nums IN num_tab_typ) RETURN PLS_INTEGER;
FUNCTION simple_sum_nums(nums IN num_tab_typ, outnums OUT num_tab_typ) RETURN NUMBER;

FUNCTION rec_in(rec IN mix_rec_typ) RETURN VARCHAR2;
FUNCTION rec_tab_in(tab IN mix_tab_typ) RETURN VARCHAR2;
END TST_oracall;
    `, nil, nil)
	if err != nil {
		t.Fatalf("error creating package head: %v", err)
	}
	if err = cu.Execute(`CREATE OR REPLACE PACKAGE BODY TST_oracall AS
PROCEDURE simple_char_in(txt IN VARCHAR2) IS
  v_txt VARCHAR2(1000) := SUBSTR(txt, 1, 100);
BEGIN NULL; END simple_char_in;
FUNCTION simple_char_out RETURN VArCHAR2 IS BEGIN RETURN('A'); END simple_char_out;

PROCEDURE simple_num_in(num IN NUMBER) IS
  v_num NUMBER := num;
BEGIN NULL; END simple_num_in;

PROCEDURE simple_date_in(dat IN DATE) IS
  v_dat DATE := dat;
BEGIN NULL; END simple_date_in;

FUNCTION simple_char_in_char_ret(txt IN VARCHAR2) RETURN VARCHAR2 IS
  v_txt CONSTANT VARCHAR2(4000) := SUBSTR(txt, 1, 4000);
  v_ret VARCHAR2(4000);
BEGIN
  SELECT DUMP(txt) INTO v_ret FROM DUAL;
  RETURN v_ret;
END simple_char_in_char_ret;

FUNCTION simple_date_out RETURN DATE IS BEGIN RETURN SYSDATE; END simple_date_out;
FUNCTION simple_num_out RETURN NUMBER IS BEGIN RETURN 2/3; END simple_num_out;

PROCEDURE simple_all_inout(
    txt1 IN VARCHAR2, int1 IN PLS_INTEGER, num1 IN NUMBER, dt1 IN DATE,
    txt2 OUT VARCHAR2, int2 OUT PLS_INTEGER, num2 OUT NUMBER, dt2 OUT DATE,
    txt3 IN OUT VARCHAR2,
    int3 IN OUT PLS_INTEGER, num3 IN OUT NUMBER, dt3 IN OUT DATE) IS
BEGIN
  txt2 := txt1||'#'; int2 := NVL(int1, 0) + 1;
  num2 := NVL(num1, 0) + 1/3; dt2 := ADD_MONTHS(NVL(dt1, SYSDATE), 1);
  txt3 := txt3||'#'; int3 := NVL(int3, 0) + 1;
  num3 := NVL(num3, 0) + 1; dt3 := ADD_MONTHS(NVL(dt3, SYSDATE), 1);
END simple_all_inout;

FUNCTION simple_nums_count(nums IN num_tab_typ) RETURN PLS_INTEGER IS
BEGIN
  RETURN nums.COUNT;
END simple_nums_count;

FUNCTION simple_sum_nums(nums IN num_tab_typ, outnums OUT num_tab_typ) RETURN NUMBER IS
  v_idx PLS_INTEGER;
  s NUMBER := 0;
BEGIN
  outnums.DELETE;
  v_idx := nums.FIRST;
  WHILE v_idx IS NOT NULL LOOP
    s := NVL(s, 0) + NVL(nums(v_idx), 0);
    outnums(v_idx) := NVL(nums(v_idx), 0) * 2;
    v_idx := nums.NEXT(v_idx);
  END LOOP;
  RETURN(s);
END simple_sum_nums;

FUNCTION rec_in(rec IN mix_rec_typ) RETURN VARCHAR2 IS
BEGIN
  RETURN rec.num||';"'||TO_CHAR(rec.dt, 'YYYY-MM-DD HH24:MI:SS')||'";"'||rec.text||'"';
END rec_in;

FUNCTION rec_tab_in(tab IN mix_tab_typ) RETURN VARCHAR2 IS
  i PLS_INTEGER;
  text VARCHAR2(32767);
BEGIN
  i := tab.FIRST;
  WHILE i IS NOT NULL LOOP
    text := text||CHR(10)||SUBSTR(
              tab(i).num||';"'||TO_CHAR(tab(i).dt, 'YYYY-MM-DD HH24:MI:SS')
              ||'";"'||tab(i).text||'"',
              1, GREATEST(0, 32767-NVL(LENGTH(text), 0)-1));
    EXIT WHEN LENGTH(text) >= 32767;
    i := tab.NEXT(i);
  END LOOP;
  RETURN(text);
END rec_tab_in;

FUNCTION nums_count(nums IN num_tab_typ) RETURN PLS_INTEGER IS
BEGIN
  RETURN nums.COUNT;
END nums_count;

FUNCTION sum_nums(nums IN num_tab_typ, outnums OUT num_tab_typ) RETURN NUMBER IS
  v_idx PLS_INTEGER;
  s NUMBER := 0;
BEGIN
  outnums.DELETE;
  v_idx := nums.FIRST;
  WHILE v_idx IS NOT NULL LOOP
    s := NVL(s, 0) + NVL(nums(v_idx), 0);
    outnums(v_idx) := NVL(nums(v_idx), 0) * 2;
    v_idx := nums.NEXT(v_idx);
  END LOOP;
  RETURN(s);
END sum_nums;

END TST_oracall;
    `, nil, nil); err != nil {
		t.Fatalf("error creating package body: %v", err)
	}
	if err = cu.Execute("SELECT text FROM user_errors WHERE name = :1", []interface{}{"TST_ORACALL"}, nil); err != nil {
		t.Fatalf("error querying errors: %v", err)
	}
	rows, err := cu.FetchAll()
	if err != nil {
		t.Fatalf("error fetching errors: %v", err)
	}
	if len(rows) > 0 {
		errTexts := make([]string, len(rows))
		for i := range rows {
			errTexts[i] = rows[i][0].(string)
		}
		t.Fatalf("error with package: %s", strings.Join(errTexts, "\n"))
	}
}
func runCommand(t *testing.T, prog string, args ...string) {
	out, err := exec.Command(prog, args...).CombinedOutput()
	if err != nil {
		t.Errorf("error '%q %s': %v\n%s", prog, args, err, out)
		t.FailNow()
	} else {
		t.Logf("%q %s:\n%s", prog, args, out)
	}
}

func build(t *testing.T) {
	buildOnce.Do(func() {
		createStoredProc(t)
		runCommand(t, "go", "build")
	})
}

func generateAndBuild(t *testing.T, prefix string) (outFn string) {
	runCommand(t, "sh", "-c",
		"./oracall -F -connect='"+*flagConnect+
			"' TST_ORACALL."+strings.ToUpper(prefix)+"%"+
			" >./testdata/integration_test/generated_functions.go")

	if outFh, err := ioutil.TempFile("", "oracall-integration_test"); err != nil {
		t.Errorf("cannot create temp file: %v", err)
		t.FailNow()
	} else {
		outFn = outFh.Name()
		outFh.Close()
	}
	os.Remove(outFn)
	runCommand(t, "go", "build", "-o="+outFn, "./testdata/integration_test")
	return
}

var errBuf = bytes.NewBuffer(make([]byte, 0, 512))

func runTest(t *testing.T, prog string, args ...string) string {
	c := exec.Command(prog, args...)
	errBuf.Reset()
	c.Stderr = errBuf
	out, err := c.Output()
	if err != nil {
		t.Errorf("error '%q %s': %v\n%s", prog, args, err, errBuf)
		t.FailNow()
	} else {
		t.Logf("%q %s:\n%s\n%s", prog, args, out, errBuf)
	}
	return string(out)
}

//var dsn = flag.String("connect", "", "Oracle DSN (user/passw@sid)")
var dbg = flag.Bool("debug", false, "print debug messages?")
var buildOnce sync.Once

func init() {
	flag.Parse()
}

var conn oracle.Connection

func getConnection(t *testing.T) oracle.Connection {
	if conn.IsConnected() {
		return conn
	}

	if !(flagConnect != nil && *flagConnect != "") {
		t.Logf("cannot test connection without dsn!")
		t.FailNow()
	}
	user, passw, sid := oracle.SplitDSN(*flagConnect)
	var err error
	conn, err = oracle.NewConnection(user, passw, sid, false)
	if err != nil {
		log.Panicf("error creating connection to %s: %s", *flagConnect, err)
	}
	if err = conn.Connect(0, false); err != nil {
		log.Panicf("error connecting: %s", err)
	}
	return conn
}
<|endoftext|>


<commit_before>package main

type benchmarkConfig struct {
	LogFile            string             `toml:"log_file"`
	StatsServer        statsServer        `toml:"stats_server"`
	Servers            []server           `toml:"servers"`
	ClusterCredentials clusterCredentials `toml:"cluster_credentials"`
	LoadSettings       loadSettings       `toml:"load_settings"`
	LoadDefinitions    []loadDefinition   `toml:"load_definitions"`
	Log                *os.File
}

type statsServer struct {
	Host     string `toml:"host"`
	User     string `toml:"user"`
	Password string `toml:"password"`
	Database string `toml:"database"`
}

type clusterCredentials struct {
	Database string `toml:"database"`
	User     string `toml:"user"`
	Password string `toml:"password"`
}

type server struct {
	ConnectionString string `toml:"connection_string"`
}

type loadSettings struct {
	ConcurrentConnections int `toml:"concurrent_connections"`
	RunPerLoadDefinition  int `toml:"runs_per_load_definition"`
}

type loadDefinition struct {
	BaseSeriesName string         `toml:"base_series_name"`
	SeriesCount    int            `toml:"series_count"`
	WriteSettings  writeSettings  `toml:"write_settings"`
	IntColumns     []intColumn    `toml:"int_columns"`
	StringColumns  []stringColumn `toml:"string_columns"`
	FloatColumns   []floatColumn  `toml:"float_columns"`
	BoolColumns    []boolColumn   `toml:"bool_columns"`
	Queries        []query        `toml:"queries"`
}

type writeSettings struct {
	BatchSeriesSize   int    `toml:"batch_series_size"`
	BatchPointsSize   int    `toml:"batch_points_size"`
	DelayBetweenPosts string `toml:"delay_between_posts"`
}

type query struct {
	FullQuery    string `toml:"full_query"`
	QueryStart   string `toml:"query_start"`
	QueryEnd     string `toml:"query_end"`
	PerformEvery string `toml:"perform_every"`
}

type intColumn struct {
	Name     string `toml:"name"`
	MinValue int    `toml:"min_value"`
	MaxValue int    `toml:"max_value"`
}

type floatColumn struct {
	Name     string  `toml:"name"`
	MinValue float64 `toml:"min_value"`
	MaxValue float64 `toml:"max_value"`
}

type boolColumn struct {
	Name string `toml:"name"`
}

type stringColumn struct {
	Name         string   `toml:"name"`
	Values       []string `toml:"values"`
	RandomLength int      `toml:"random_length"`
}
<commit_msg>Remove extraneous benchmark config file. It's in the influxdb_benchmark.go<commit_after><|endoftext|>


<commit_before>package messaging

import (
	"fmt"
	"os"
	"os/signal"
	"syscall"

	"github.com/Shopify/sarama"
	libs "github.com/k8guard/k8guardlibs"
	"github.com/k8guard/k8guardlibs/messaging/kafka"

	"encoding/json"
	"reflect"

	"github.com/k8guard/k8guard-action/actions"

	"github.com/k8guard/k8guard-action/db"

	"github.com/k8guard/k8guardlibs/violations"
)

func ConsumeMessages() {

	signalChannel := make(chan os.Signal, 2)
	signal.Notify(signalChannel, syscall.SIGTERM)

	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt)

	topic := libs.Cfg.KafkaActionTopic

	master, err := kafka.NewConsumer(kafka.ACTION_CLIENTID, libs.Cfg)
	if err != nil {
		panic(err)

	}

	defer func() {
		if err := master.Close(); err != nil {
			panic(err)
		}
	}()

	partitions, _ := master.Partitions(topic)

	messages := make(chan *sarama.ConsumerMessage)
	for _, partition := range partitions {

		libs.Log.Info("Creating Consumer ", topic, " on partition ", partition)
		consumer, err := master.ConsumePartition(topic, partition, sarama.OffsetNewest)
		if err != nil {
			panic(err)
		}

		go func(consumer sarama.PartitionConsumer) {
			for {
				select {
				case err := <-consumer.Errors():
					fmt.Println(err)
				case msg := <-consumer.Messages():
					messages <- msg
				case <-signals:
					libs.Log.Debug("Interrupt is detected")
				}
			}
		}(consumer)
	}

	libs.Log.Info("Waiting for messages")
	for {
		message := <-messages
		parseViolationMessage(message)
	}
}

func parseViolationMessage(msg *sarama.ConsumerMessage) {
	libs.Log.Info("Taking Action ...")

	messageData := map[string]interface{}{}
	err := json.Unmarshal(msg.Value, &messageData)
	if err != nil {
		libs.Log.Fatal(err)
	}

	dataBytes, _ := json.Marshal(messageData["data"])

	entityViolations := []violations.Violation{}
	var actionableEntity actions.ActionableEntity

	switch messageData["kind"] {
	case string(kafka.POD_MESSAGE):
		libs.Log.Debug("Parsing Pod Message")

		pod := actions.ActionPod{}
		json.Unmarshal(dataBytes, &pod)
		actionableEntity = pod

		entityViolations = append(entityViolations, pod.Violations...)

		break
	case string(kafka.NAMESPACE_MESSAGE):
		libs.Log.Debug("Parsing Namespace Message")

		namespace := actions.ActionNamespace{}
		json.Unmarshal(dataBytes, &namespace)
		actionableEntity = namespace

		entityViolations = append(entityViolations, namespace.Violations...)

		break
	case string(kafka.DEPLOYMENT_MESSAGE):
		libs.Log.Debug("Parsing Deployment Message")

		deployment := actions.ActionDeployment{}
		json.Unmarshal(dataBytes, &deployment)
		actionableEntity = deployment

		entityViolations = append(entityViolations, deployment.Violations...)

		break
	case string(kafka.DAEMONSET_MESSAGE):
		libs.Log.Debug("Parsing DaemonSet Message")

		daemonSet := actions.ActionDaemonSet{}
		json.Unmarshal(dataBytes, &daemonSet)
		actionableEntity = daemonSet

		entityViolations = append(entityViolations, daemonSet.Violations...)

		break
	case string(kafka.INGRESS_MESSAGE):
		libs.Log.Debug("Parsing Ingress Message")

		ingress := actions.ActionIngress{}
		json.Unmarshal(dataBytes, &ingress)
		actionableEntity = ingress

		entityViolations = append(entityViolations, ingress.Violations...)

		break
	case string(kafka.JOB_MESSAGE):
		libs.Log.Debug("Parsing Job Message")

		job := actions.ActionJob{}
		json.Unmarshal(dataBytes, &job)
		actionableEntity = job

		entityViolations = append(entityViolations, job.Violations...)

		break
	case string(kafka.CRONJOB_MESSAGE):
		libs.Log.Debug("Parsing CronJob Message")

		cronjob := actions.ActionCronJob{}
		json.Unmarshal(dataBytes, &cronjob)
		actionableEntity = cronjob

		entityViolations = append(entityViolations, cronjob.Violations...)
		break
	default:
		libs.Log.Error("Unknown Message Kind: ", messageData["kind"])
		return
	}

	for _, violation := range entityViolations {

		vEntity, err := actions.ConvertActionableEntityToViolatableEntity(actionableEntity)
		if err != nil {
			libs.Log.Fatal(err)
		}

		// Insert violation into log
		db.InsertVLOGRow(vEntity, violation, reflect.TypeOf(actionableEntity).Name())
		action := createAction(violation)
		vActionRow := db.SelectVActionRow(vEntity, violation, reflect.TypeOf(actionableEntity).Name())
		doneActions := actions.DoAction(action, actionableEntity, vEntity, vActionRow.Actions, libs.Cfg.ActionDryRun)

		if len(doneActions) == 0 {
			// If we did no actions don't insert anything
			return
		}

		for actionName, t := range doneActions {

			// Insert action into log
			db.InsertActionLogRow(vEntity.Namespace, reflect.TypeOf(actionableEntity).Name(), vEntity.Name, string(violation.Type), violation.Source, actionName)

			if _, ok := vActionRow.Actions[actionName]; ok {
				vActionRow.Actions[actionName] = append(vActionRow.Actions[actionName], t...)
			} else {
				vActionRow.Actions[actionName] = t
			}
		}

		// Insert violation state
		db.InsertVactionRow(vEntity.Namespace, reflect.TypeOf(actionableEntity).Name(), vEntity.Name, string(violation.Type), violation.Source, vActionRow.Actions)

	}

}

func createAction(violation violations.Violation) actions.Action {
	switch vType := violation.Type; vType {
	case violations.SINGLE_REPLICA_TYPE:
		return actions.SingleReplicaAction{Violation: violation}
	case violations.IMAGE_SIZE_TYPE:
		return actions.ImageSizeAction{Violation: violation}
	case violations.IMAGE_REPO_TYPE:
		return actions.ImageRepoAction{Violation: violation}
	case violations.INGRESS_HOST_INVALID_TYPE:
		return actions.IngressAction{Violation: violation}
	case violations.CAPABILITIES_TYPE:
		return actions.CapabilitiesAction{Violation: violation}
	case violations.PRIVILEGED_TYPE:
		return actions.PrivilegedAction{Violation: violation}
	case violations.HOST_VOLUMES_TYPE:
		return actions.HostVolumesAction{Violation: violation}
	case violations.REQUIRED_NAMESPACES_TYPE:
		return actions.RequiredNamespaceAction{Violation: violation}
	case violations.REQUIRED_NAMESPACE_ANNOTATIONS_TYPE:
		return actions.RequiredNamespaceAnnotationAction{Violation: violation}
	case violations.REQUIRED_NAMESPACE_LABELS_TYPE:
		return actions.RequiredNamespaceLabelAction{Violation: violation}
	case violations.REQUIRED_DEPLOYMENTS_TYPE:
		return actions.RequiredDeploymentAction{Violation: violation}
	case violations.REQUIRED_DEPLOYMENT_ANNOTATIONS_TYPE:
		return actions.RequiredDeploymentAnnotationAction{Violation: violation}
	case violations.REQUIRED_DEPLOYMENT_LABELS_TYPE:
		return actions.RequiredDeploymentLabelAction{Violation: violation}
	case violations.REQUIRED_PODS_TYPE:
		return actions.RequiredPodAction{Violation: violation}
	case violations.REQUIRED_POD_ANNOTATIONS_TYPE:
		return actions.RequiredPodAnnotationAction{Violation: violation}
	case violations.REQUIRED_POD_LABELS_TYPE:
		return actions.RequiredPodLabelAction{Violation: violation}
	case violations.REQUIRED_DAEMONSETS_TYPE:
		return actions.RequiredDaemonSetAction{Violation: violation}
	case violations.REQUIRED_DAEMONSET_ANNOTATIONS_TYPE:
		return actions.RequiredDaemonSetAnnotationAction{Violation: violation}
	case violations.REQUIRED_DAEMONSET_LABELS_TYPE:
		return actions.RequiredDaemonSetLabelAction{Violation: violation}
	case violations.REQUIRED_RESOURCEQUOTA_TYPE:
		return actions.RequiredResourceQuotaAction{Violation: violation}
	case violations.NO_OWNER_ANNOTATION_TYPE:
		return actions.NoOwnerAction{Violation: violation}
	default:
		libs.Log.Fatal("Unknown Violation Type ", vType)
	}

	return nil
}
<commit_msg>initial check in of supporting lighter weight dependencies<commit_after>package messaging

import (
	libs "github.com/k8guard/k8guardlibs"
	"github.com/k8guard/k8guardlibs/messaging"
	"github.com/k8guard/k8guardlibs/messaging/types"

	"encoding/json"
	"reflect"

	"github.com/k8guard/k8guard-action/actions"

	"github.com/k8guard/k8guard-action/db"

	"github.com/k8guard/k8guardlibs/violations"
)

func ConsumeMessages() {

	c, err := messaging.CreateMessageConsumer(
		types.MessageBrokerType(libs.Cfg.MessageBroker), types.ACTION_CLIENTID, libs.Cfg)
	if err != nil {
		panic(err)
	}

	defer func() {
		c.Close()
	}()

	messages := make(chan []byte)
	c.ConsumeMessages(messages)

	libs.Log.Info("Waiting for messages")
	for {
		message := <-messages
		parseViolationMessage(message)
	}
}

func parseViolationMessage(msg []byte) {
	libs.Log.Info("Processing violation message ...")

	messageData := map[string]interface{}{}
	err := json.Unmarshal(msg, &messageData)
	if err != nil {
		libs.Log.Fatal(err)
	}

	dataBytes, _ := json.Marshal(messageData["data"])

	entityViolations := []violations.Violation{}
	var actionableEntity actions.ActionableEntity

	switch messageData["kind"] {
	case string(types.POD_MESSAGE):
		libs.Log.Debug("Parsing Pod Message")

		pod := actions.ActionPod{}
		json.Unmarshal(dataBytes, &pod)
		actionableEntity = pod

		entityViolations = append(entityViolations, pod.Violations...)

		break
	case string(types.NAMESPACE_MESSAGE):
		libs.Log.Debug("Parsing Namespace Message")

		namespace := actions.ActionNamespace{}
		json.Unmarshal(dataBytes, &namespace)
		actionableEntity = namespace

		entityViolations = append(entityViolations, namespace.Violations...)

		break
	case string(types.DEPLOYMENT_MESSAGE):
		libs.Log.Debug("Parsing Deployment Message")

		deployment := actions.ActionDeployment{}
		json.Unmarshal(dataBytes, &deployment)
		actionableEntity = deployment

		entityViolations = append(entityViolations, deployment.Violations...)

		break
	case string(types.DAEMONSET_MESSAGE):
		libs.Log.Debug("Parsing DaemonSet Message")

		daemonSet := actions.ActionDaemonSet{}
		json.Unmarshal(dataBytes, &daemonSet)
		actionableEntity = daemonSet

		entityViolations = append(entityViolations, daemonSet.Violations...)

		break
	case string(types.INGRESS_MESSAGE):
		libs.Log.Debug("Parsing Ingress Message")

		ingress := actions.ActionIngress{}
		json.Unmarshal(dataBytes, &ingress)
		actionableEntity = ingress

		entityViolations = append(entityViolations, ingress.Violations...)

		break
	case string(types.JOB_MESSAGE):
		libs.Log.Debug("Parsing Job Message")

		job := actions.ActionJob{}
		json.Unmarshal(dataBytes, &job)
		actionableEntity = job

		entityViolations = append(entityViolations, job.Violations...)

		break
	case string(types.CRONJOB_MESSAGE):
		libs.Log.Debug("Parsing CronJob Message")

		cronjob := actions.ActionCronJob{}
		json.Unmarshal(dataBytes, &cronjob)
		actionableEntity = cronjob

		entityViolations = append(entityViolations, cronjob.Violations...)
		break
	default:
		libs.Log.Error("Unknown Message Kind: ", messageData["kind"])
		return
	}

	for _, violation := range entityViolations {

		vEntity, err := actions.ConvertActionableEntityToViolatableEntity(actionableEntity)
		if err != nil {
			libs.Log.Fatal(err)
		}

		// Insert violation into log
		db.InsertVLOGRow(vEntity, violation, reflect.TypeOf(actionableEntity).Name())
		action := createAction(violation)
		vActionRow := db.SelectVActionRow(vEntity, violation, reflect.TypeOf(actionableEntity).Name())
		doneActions := actions.DoAction(action, actionableEntity, vEntity, vActionRow.Actions, libs.Cfg.ActionDryRun)

		if len(doneActions) == 0 {
			// If we did no actions don't insert anything
			return
		}

		for actionName, t := range doneActions {

			// Insert action into log
			db.InsertActionLogRow(vEntity.Namespace, reflect.TypeOf(actionableEntity).Name(), vEntity.Name, string(violation.Type), violation.Source, actionName)

			if _, ok := vActionRow.Actions[actionName]; ok {
				vActionRow.Actions[actionName] = append(vActionRow.Actions[actionName], t...)
			} else {
				vActionRow.Actions[actionName] = t
			}
		}

		// Insert violation state
		db.InsertVactionRow(vEntity.Namespace, reflect.TypeOf(actionableEntity).Name(), vEntity.Name, string(violation.Type), violation.Source, vActionRow.Actions)

	}

}

func createAction(violation violations.Violation) actions.Action {
	switch vType := violation.Type; vType {
	case violations.SINGLE_REPLICA_TYPE:
		return actions.SingleReplicaAction{Violation: violation}
	case violations.IMAGE_SIZE_TYPE:
		return actions.ImageSizeAction{Violation: violation}
	case violations.IMAGE_REPO_TYPE:
		return actions.ImageRepoAction{Violation: violation}
	case violations.INGRESS_HOST_INVALID_TYPE:
		return actions.IngressAction{Violation: violation}
	case violations.CAPABILITIES_TYPE:
		return actions.CapabilitiesAction{Violation: violation}
	case violations.PRIVILEGED_TYPE:
		return actions.PrivilegedAction{Violation: violation}
	case violations.HOST_VOLUMES_TYPE:
		return actions.HostVolumesAction{Violation: violation}
	case violations.REQUIRED_NAMESPACES_TYPE:
		return actions.RequiredNamespaceAction{Violation: violation}
	case violations.REQUIRED_NAMESPACE_ANNOTATIONS_TYPE:
		return actions.RequiredNamespaceAnnotationAction{Violation: violation}
	case violations.REQUIRED_NAMESPACE_LABELS_TYPE:
		return actions.RequiredNamespaceLabelAction{Violation: violation}
	case violations.REQUIRED_DEPLOYMENTS_TYPE:
		return actions.RequiredDeploymentAction{Violation: violation}
	case violations.REQUIRED_DEPLOYMENT_ANNOTATIONS_TYPE:
		return actions.RequiredDeploymentAnnotationAction{Violation: violation}
	case violations.REQUIRED_DEPLOYMENT_LABELS_TYPE:
		return actions.RequiredDeploymentLabelAction{Violation: violation}
	case violations.REQUIRED_PODS_TYPE:
		return actions.RequiredPodAction{Violation: violation}
	case violations.REQUIRED_POD_ANNOTATIONS_TYPE:
		return actions.RequiredPodAnnotationAction{Violation: violation}
	case violations.REQUIRED_POD_LABELS_TYPE:
		return actions.RequiredPodLabelAction{Violation: violation}
	case violations.REQUIRED_DAEMONSETS_TYPE:
		return actions.RequiredDaemonSetAction{Violation: violation}
	case violations.REQUIRED_DAEMONSET_ANNOTATIONS_TYPE:
		return actions.RequiredDaemonSetAnnotationAction{Violation: violation}
	case violations.REQUIRED_DAEMONSET_LABELS_TYPE:
		return actions.RequiredDaemonSetLabelAction{Violation: violation}
	case violations.REQUIRED_RESOURCEQUOTA_TYPE:
		return actions.RequiredResourceQuotaAction{Violation: violation}
	case violations.NO_OWNER_ANNOTATION_TYPE:
		return actions.NoOwnerAction{Violation: violation}
	default:
		libs.Log.Fatal("Unknown Violation Type ", vType)
	}

	return nil
}
<|endoftext|>


<commit_before>package url

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/storage"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"strings"
	"time"
)

//Resource represents a URL based resource, with enriched meta info
type Resource struct {
	URL             string   `description:"resource URL or relative or absolute path" required:"true"` //URL of resource
	Credentials     string   `description:"credentials file"`                                          //name of credential file or credential key depending on implementation
	ParsedURL       *url.URL `json:"-"`                                                                //parsed URL resource
	Cache           string   `description:"local cache path"`                                          //Cache path for the resource, if specified resource will be cached in the specified path
	CacheExpiryMs   int      //CacheExpiryMs expiry time in ms
	modificationTag int64
	init            bool
}

//Clone creates a clone of the resource
func (r *Resource) Clone() *Resource {
	return &Resource{
		init:          r.init,
		URL:           r.URL,
		Credentials:   r.Credentials,
		ParsedURL:     r.ParsedURL,
		Cache:         r.Cache,
		CacheExpiryMs: r.CacheExpiryMs,
	}
}

var defaultSchemePorts = map[string]int{
	"ssh":   22,
	"scp":   22,
	"http":  80,
	"https": 443,
}

//Host returns url's host name with user name if user name is part of url
func (r *Resource) Host() string {
	result := r.ParsedURL.Hostname() + ":" + r.Port()
	if r.ParsedURL.User != nil {
		result = r.ParsedURL.User.Username() + "@" + result
	}
	return result
}

//CredentialURL returns url's with provided credential
func (r *Resource) CredentialURL(username, password string) string {
	var urlCredential = ""
	if username != "" {
		urlCredential = username
		if password != "" {
			urlCredential += ":" + password
		}
		urlCredential += "@"
	}
	result := r.ParsedURL.Scheme + "://" + urlCredential + r.ParsedURL.Hostname() + ":" + r.Port() + r.ParsedURL.Path
	if r.ParsedURL.RawQuery != "" {
		result += "?" + r.ParsedURL.RawQuery
	}

	return result
}

//Path returns url's path  directory, assumption is that directory does not have extension, if path ends with '/' it is being stripped.
func (r *Resource) DirectoryPath() string {
	if r.ParsedURL == nil {
		return ""
	}
	var result = r.ParsedURL.Path

	parent, name := path.Split(result)
	if path.Ext(name) != "" {
		result = parent
	}
	if strings.HasSuffix(result, "/") {
		result = string(result[:len(result)-1])
	}
	return result
}

//Port returns url's port
func (r *Resource) Port() string {
	port := r.ParsedURL.Port()
	if port == "" && r.ParsedURL != nil {
		if value, ok := defaultSchemePorts[r.ParsedURL.Scheme]; ok {
			port = toolbox.AsString(value)
		}
	}
	return port
}

//Download downloads data from URL, it returns data as []byte, or error, if resource is cacheable it first look into cache
func (r *Resource) Download() ([]byte, error) {
	if r == nil {
		return nil, fmt.Errorf("Fail to download content on empty resource")
	}
	if r.Cachable() {
		content := r.readFromCache()
		if content != nil {
			return content, nil
		}
	}
	service, err := storage.NewServiceForURL(r.URL, r.Credentials)
	if err != nil {
		return nil, err
	}
	object, err := service.StorageObject(r.URL)
	if err != nil {
		return nil, err
	}
	reader, err := service.Download(object)
	if err != nil {
		return nil, err
	}
	defer reader.Close()
	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}
	if r.Cachable() {
		_ = ioutil.WriteFile(r.Cache, content, 0666)
	}
	return content, err
}

//DownloadText returns a text downloaded from url
func (r *Resource) DownloadText() (string, error) {
	var result, err = r.Download()
	if err != nil {
		return "", err
	}
	return string(result), err
}

//Decode decodes url's data into target, it support JSON and YAML exp.
func (r *Resource) Decode(target interface{}) (err error) {
	defer func() {
		if err != nil {
			err = fmt.Errorf("failed to decode: %v, %v", r.URL, err)
		}
	}()
	if r.ParsedURL == nil {
		if r.ParsedURL, err = url.Parse(r.URL); err != nil {
			return err
		}
	}
	ext := path.Ext(r.ParsedURL.Path)
	switch ext {
	case ".yaml", ".yml":
		err = r.YAMLDecode(target)
	default:
		err = r.JSONDecode(target)
	}
	return err
}

//DecoderFactory returns new decoder factory for resource
func (r *Resource) DecoderFactory() toolbox.DecoderFactory {
	ext := path.Ext(r.ParsedURL.Path)
	switch ext {
	case ".yaml", ".yml":
		return toolbox.NewYamlDecoderFactory()
	default:
		return toolbox.NewJSONDecoderFactory()
	}
}

//Decode decodes url's data into target, it takes decoderFactory which decodes data into target
func (r *Resource) DecodeWith(target interface{}, decoderFactory toolbox.DecoderFactory) error {
	if r == nil {
		return fmt.Errorf("fail to %T decode on empty resource", decoderFactory)
	}
	if decoderFactory == nil {
		return fmt.Errorf("fail to decode %v, decoderFactory was empty", r.URL)
	}
	var content, err = r.Download()
	if err != nil {
		return err
	}

	text := string(content)
	if toolbox.IsNewLineDelimitedJSON(text) {
		if aSlice, err := toolbox.NewLineDelimitedJSON(text); err == nil {
			return toolbox.DefaultConverter.AssignConverted(target, aSlice)
		}
	}
	err = decoderFactory.Create(bytes.NewReader(content)).Decode(target)
	if err != nil {
		return fmt.Errorf("failed to decode: %v, payload: %s", err, content)
	}
	return err
}

//Rename renames URI name of this resource
func (r *Resource) Rename(name string) (err error) {
	var _, currentName = toolbox.URLSplit(r.URL)
	if currentName == "" && strings.HasSuffix(r.URL, "/") {
		_, currentName = toolbox.URLSplit(r.URL[:len(r.URL)-1])
		currentName += "/"
	}

	r.URL = strings.Replace(r.URL, currentName, name, 1)
	r.ParsedURL, err = url.Parse(r.URL)
	return err
}

//JSONDecode decodes json resource into target
func (r *Resource) JSONDecode(target interface{}) error {
	return r.DecodeWith(target, toolbox.NewJSONDecoderFactory())
}

//JSONDecode decodes yaml resource into target
func (r *Resource) YAMLDecode(target interface{}) error {
	if interfacePrt, ok := target.(*interface{}); ok {
		var data interface{}
		if err := r.DecodeWith(&data, toolbox.NewYamlDecoderFactory()); err != nil {
			return err
		}
		if toolbox.IsSlice(data) {
			*interfacePrt = data
			return nil
		}
	}
	var mapSlice = yaml.MapSlice{}
	if err := r.DecodeWith(&mapSlice, toolbox.NewYamlDecoderFactory()); err != nil {
		return err
	}
	if !toolbox.IsMap(target) {
		return toolbox.DefaultConverter.AssignConverted(target, mapSlice)
	}
	resultMap := toolbox.AsMap(target)
	for _, v := range mapSlice {
		resultMap[toolbox.AsString(v.Key)] = v.Value
	}
	return nil
}

func (r *Resource) readFromCache() []byte {
	if toolbox.FileExists(r.Cache) {
		info, err := os.Stat(r.Cache)
		var isExpired = false
		if err == nil && r.CacheExpiryMs > 0 {
			elapsed := time.Now().Sub(info.ModTime())
			isExpired = elapsed > time.Second*time.Duration(r.CacheExpiryMs)
		}
		content, err := ioutil.ReadFile(r.Cache)
		if err == nil && !isExpired {
			return content
		}
	}
	return nil
}

//Cachable returns true if resource is cachable
func (r *Resource) Cachable() bool {
	return r.Cache != ""
}

func computeResourceModificationTag(resource *Resource) (int64, error) {
	service, err := storage.NewServiceForURL(resource.URL, resource.Credentials)
	if err != nil {
		return 0, err
	}
	object, err := service.StorageObject(resource.URL)
	if err != nil {
		return 0, err
	}
	var fileInfo = object.FileInfo()

	if object.IsContent() {
		return fileInfo.Size() + fileInfo.ModTime().UnixNano(), nil
	}
	var result int64 = 0
	objects, err := service.List(resource.URL)
	if err != nil {
		return 0, err
	}
	for _, object := range objects {
		objectResource := NewResource(object.URL())
		if objectResource.ParsedURL.Path == resource.ParsedURL.Path {
			continue
		}
		modificationTag, err := computeResourceModificationTag(NewResource(object.URL(), resource.Credentials))
		if err != nil {
			return 0, err
		}
		result += modificationTag

	}
	return result, nil
}

func (r *Resource) HasChanged() (changed bool, err error) {
	if r.modificationTag == 0 {
		r.modificationTag, err = computeResourceModificationTag(r)
		return false, err
	}
	var recentModificationTag int64
	recentModificationTag, err = computeResourceModificationTag(r)
	if err != nil {
		return false, err
	}
	if recentModificationTag != r.modificationTag {
		changed = true
		r.modificationTag = recentModificationTag
	}
	return changed, err
}

func normalizeURL(URL string) string {
	if strings.Contains(URL, "://") {
		var protoPosition = strings.Index(URL, "://")
		if protoPosition != -1 {
			var urlSuffix = string(URL[protoPosition+3:])
			urlSuffix = strings.Replace(urlSuffix, "//", "/", len(urlSuffix))
			URL = string(URL[:protoPosition+3]) + urlSuffix
		}
		return URL
	}
	if !strings.HasPrefix(URL, "/") {
		currentDirectory, _ := os.Getwd()

		if strings.Contains(URL, "..") {
			fragments := strings.Split(URL, "/")
			var index = 0
			var offset = 0
			if fragments[0] == "." {
				offset = 1
			}

			for index = offset; index < len(fragments); index++ {
				var fragment = fragments[index]
				if fragment == ".." {
					currentDirectory, _ = path.Split(currentDirectory)
					if strings.HasSuffix(currentDirectory, "/") {
						currentDirectory = string(currentDirectory[:len(currentDirectory)-1])
					}
					continue
				}
				break
			}
			return toolbox.FileSchema + path.Join(currentDirectory, strings.Join(fragments[index:], "/"))
		}

		currentDirectory, err := os.Getwd()
		if err == nil {
			candidate := path.Join(currentDirectory, URL)
			URL = candidate
		}
	}
	return toolbox.FileSchema + URL
}

func (r *Resource) Init() (err error) {
	if r.init {
		return nil
	}
	r.init = true
	r.URL = normalizeURL(r.URL)
	r.ParsedURL, err = url.Parse(r.URL)
	return err
}

//DownloadBase64 loads base64 resource content
func (r *Resource) DownloadBase64() (string, error) {
	storageService, err := storage.NewServiceForURL(r.URL, r.Credentials)
	if err != nil {
		return "", err
	}
	reader, err := storage.Download(storageService, r.URL)
	if err != nil {
		return "", err
	}
	defer reader.Close()
	data, err := ioutil.ReadAll(reader)
	if err != nil {
		return "", err
	}
	_, err = base64.StdEncoding.DecodeString(string(data))
	if err == nil {
		return string(data), nil
	}
	return base64.StdEncoding.EncodeToString(data), nil
}

//NewResource returns a new resource for provided URL, followed by optional credential, cache and cache expiryMs.
func NewResource(Params ...interface{}) *Resource {
	if len(Params) == 0 {
		return nil
	}
	var URL = toolbox.AsString(Params[0])
	URL = normalizeURL(URL)

	var credential string
	if len(Params) > 1 {
		credential = toolbox.AsString(Params[1])
	}
	var cache string
	if len(Params) > 2 {
		cache = toolbox.AsString(Params[2])
	}
	var cacheExpiryMs int
	if len(Params) > 3 {
		cacheExpiryMs = toolbox.AsInt(Params[3])
	}
	parsedURL, _ := url.Parse(URL)
	return &Resource{
		init:          true,
		ParsedURL:     parsedURL,
		URL:           URL,
		Credentials:   credential,
		Cache:         cache,
		CacheExpiryMs: cacheExpiryMs,
	}
}
<commit_msg>added custom key<commit_after>package url

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"github.com/viant/afs/option"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/storage"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"strings"
	"time"
)

//Resource represents a URL based resource, with enriched meta info
type Resource struct {
	URL             string            `description:"resource URL or relative or absolute path" required:"true"` //URL of resource
	Credentials     string            `description:"credentials file"`                                          //name of credential file or credential key depending on implementation
	ParsedURL       *url.URL          `json:"-"`                                                                //parsed URL resource
	Cache           string            `description:"local cache path"`                                          //Cache path for the resource, if specified resource will be cached in the specified path
	CustomKey       *option.AES256Key `description:" content encryption key"`
	CacheExpiryMs   int               //CacheExpiryMs expiry time in ms
	modificationTag int64
	init            bool
}

//Clone creates a clone of the resource
func (r *Resource) Clone() *Resource {
	return &Resource{
		init:          r.init,
		URL:           r.URL,
		Credentials:   r.Credentials,
		ParsedURL:     r.ParsedURL,
		Cache:         r.Cache,
		CacheExpiryMs: r.CacheExpiryMs,
	}
}

var defaultSchemePorts = map[string]int{
	"ssh":   22,
	"scp":   22,
	"http":  80,
	"https": 443,
}

//Host returns url's host name with user name if user name is part of url
func (r *Resource) Host() string {
	result := r.ParsedURL.Hostname() + ":" + r.Port()
	if r.ParsedURL.User != nil {
		result = r.ParsedURL.User.Username() + "@" + result
	}
	return result
}

//CredentialURL returns url's with provided credential
func (r *Resource) CredentialURL(username, password string) string {
	var urlCredential = ""
	if username != "" {
		urlCredential = username
		if password != "" {
			urlCredential += ":" + password
		}
		urlCredential += "@"
	}
	result := r.ParsedURL.Scheme + "://" + urlCredential + r.ParsedURL.Hostname() + ":" + r.Port() + r.ParsedURL.Path
	if r.ParsedURL.RawQuery != "" {
		result += "?" + r.ParsedURL.RawQuery
	}

	return result
}

//Path returns url's path  directory, assumption is that directory does not have extension, if path ends with '/' it is being stripped.
func (r *Resource) DirectoryPath() string {
	if r.ParsedURL == nil {
		return ""
	}
	var result = r.ParsedURL.Path

	parent, name := path.Split(result)
	if path.Ext(name) != "" {
		result = parent
	}
	if strings.HasSuffix(result, "/") {
		result = string(result[:len(result)-1])
	}
	return result
}

//Port returns url's port
func (r *Resource) Port() string {
	port := r.ParsedURL.Port()
	if port == "" && r.ParsedURL != nil {
		if value, ok := defaultSchemePorts[r.ParsedURL.Scheme]; ok {
			port = toolbox.AsString(value)
		}
	}
	return port
}

//Download downloads data from URL, it returns data as []byte, or error, if resource is cacheable it first look into cache
func (r *Resource) Download() ([]byte, error) {
	if r == nil {
		return nil, fmt.Errorf("Fail to download content on empty resource")
	}
	if r.Cachable() {
		content := r.readFromCache()
		if content != nil {
			return content, nil
		}
	}
	service, err := storage.NewServiceForURL(r.URL, r.Credentials)
	if err != nil {
		return nil, err
	}
	object, err := service.StorageObject(r.URL)
	if err != nil {
		return nil, err
	}
	reader, err := service.Download(object)
	if err != nil {
		return nil, err
	}
	defer reader.Close()
	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}
	if r.Cachable() {
		_ = ioutil.WriteFile(r.Cache, content, 0666)
	}
	return content, err
}

//DownloadText returns a text downloaded from url
func (r *Resource) DownloadText() (string, error) {
	var result, err = r.Download()
	if err != nil {
		return "", err
	}
	return string(result), err
}

//Decode decodes url's data into target, it support JSON and YAML exp.
func (r *Resource) Decode(target interface{}) (err error) {
	defer func() {
		if err != nil {
			err = fmt.Errorf("failed to decode: %v, %v", r.URL, err)
		}
	}()
	if r.ParsedURL == nil {
		if r.ParsedURL, err = url.Parse(r.URL); err != nil {
			return err
		}
	}
	ext := path.Ext(r.ParsedURL.Path)
	switch ext {
	case ".yaml", ".yml":
		err = r.YAMLDecode(target)
	default:
		err = r.JSONDecode(target)
	}
	return err
}

//DecoderFactory returns new decoder factory for resource
func (r *Resource) DecoderFactory() toolbox.DecoderFactory {
	ext := path.Ext(r.ParsedURL.Path)
	switch ext {
	case ".yaml", ".yml":
		return toolbox.NewYamlDecoderFactory()
	default:
		return toolbox.NewJSONDecoderFactory()
	}
}

//Decode decodes url's data into target, it takes decoderFactory which decodes data into target
func (r *Resource) DecodeWith(target interface{}, decoderFactory toolbox.DecoderFactory) error {
	if r == nil {
		return fmt.Errorf("fail to %T decode on empty resource", decoderFactory)
	}
	if decoderFactory == nil {
		return fmt.Errorf("fail to decode %v, decoderFactory was empty", r.URL)
	}
	var content, err = r.Download()
	if err != nil {
		return err
	}

	text := string(content)
	if toolbox.IsNewLineDelimitedJSON(text) {
		if aSlice, err := toolbox.NewLineDelimitedJSON(text); err == nil {
			return toolbox.DefaultConverter.AssignConverted(target, aSlice)
		}
	}
	err = decoderFactory.Create(bytes.NewReader(content)).Decode(target)
	if err != nil {
		return fmt.Errorf("failed to decode: %v, payload: %s", err, content)
	}
	return err
}

//Rename renames URI name of this resource
func (r *Resource) Rename(name string) (err error) {
	var _, currentName = toolbox.URLSplit(r.URL)
	if currentName == "" && strings.HasSuffix(r.URL, "/") {
		_, currentName = toolbox.URLSplit(r.URL[:len(r.URL)-1])
		currentName += "/"
	}

	r.URL = strings.Replace(r.URL, currentName, name, 1)
	r.ParsedURL, err = url.Parse(r.URL)
	return err
}

//JSONDecode decodes json resource into target
func (r *Resource) JSONDecode(target interface{}) error {
	return r.DecodeWith(target, toolbox.NewJSONDecoderFactory())
}

//JSONDecode decodes yaml resource into target
func (r *Resource) YAMLDecode(target interface{}) error {
	if interfacePrt, ok := target.(*interface{}); ok {
		var data interface{}
		if err := r.DecodeWith(&data, toolbox.NewYamlDecoderFactory()); err != nil {
			return err
		}
		if toolbox.IsSlice(data) {
			*interfacePrt = data
			return nil
		}
	}
	var mapSlice = yaml.MapSlice{}
	if err := r.DecodeWith(&mapSlice, toolbox.NewYamlDecoderFactory()); err != nil {
		return err
	}
	if !toolbox.IsMap(target) {
		return toolbox.DefaultConverter.AssignConverted(target, mapSlice)
	}
	resultMap := toolbox.AsMap(target)
	for _, v := range mapSlice {
		resultMap[toolbox.AsString(v.Key)] = v.Value
	}
	return nil
}

func (r *Resource) readFromCache() []byte {
	if toolbox.FileExists(r.Cache) {
		info, err := os.Stat(r.Cache)
		var isExpired = false
		if err == nil && r.CacheExpiryMs > 0 {
			elapsed := time.Now().Sub(info.ModTime())
			isExpired = elapsed > time.Second*time.Duration(r.CacheExpiryMs)
		}
		content, err := ioutil.ReadFile(r.Cache)
		if err == nil && !isExpired {
			return content
		}
	}
	return nil
}

//Cachable returns true if resource is cachable
func (r *Resource) Cachable() bool {
	return r.Cache != ""
}

func computeResourceModificationTag(resource *Resource) (int64, error) {
	service, err := storage.NewServiceForURL(resource.URL, resource.Credentials)
	if err != nil {
		return 0, err
	}
	object, err := service.StorageObject(resource.URL)
	if err != nil {
		return 0, err
	}
	var fileInfo = object.FileInfo()

	if object.IsContent() {
		return fileInfo.Size() + fileInfo.ModTime().UnixNano(), nil
	}
	var result int64 = 0
	objects, err := service.List(resource.URL)
	if err != nil {
		return 0, err
	}
	for _, object := range objects {
		objectResource := NewResource(object.URL())
		if objectResource.ParsedURL.Path == resource.ParsedURL.Path {
			continue
		}
		modificationTag, err := computeResourceModificationTag(NewResource(object.URL(), resource.Credentials))
		if err != nil {
			return 0, err
		}
		result += modificationTag

	}
	return result, nil
}

func (r *Resource) HasChanged() (changed bool, err error) {
	if r.modificationTag == 0 {
		r.modificationTag, err = computeResourceModificationTag(r)
		return false, err
	}
	var recentModificationTag int64
	recentModificationTag, err = computeResourceModificationTag(r)
	if err != nil {
		return false, err
	}
	if recentModificationTag != r.modificationTag {
		changed = true
		r.modificationTag = recentModificationTag
	}
	return changed, err
}

func normalizeURL(URL string) string {
	if strings.Contains(URL, "://") {
		var protoPosition = strings.Index(URL, "://")
		if protoPosition != -1 {
			var urlSuffix = string(URL[protoPosition+3:])
			urlSuffix = strings.Replace(urlSuffix, "//", "/", len(urlSuffix))
			URL = string(URL[:protoPosition+3]) + urlSuffix
		}
		return URL
	}
	if !strings.HasPrefix(URL, "/") {
		currentDirectory, _ := os.Getwd()

		if strings.Contains(URL, "..") {
			fragments := strings.Split(URL, "/")
			var index = 0
			var offset = 0
			if fragments[0] == "." {
				offset = 1
			}

			for index = offset; index < len(fragments); index++ {
				var fragment = fragments[index]
				if fragment == ".." {
					currentDirectory, _ = path.Split(currentDirectory)
					if strings.HasSuffix(currentDirectory, "/") {
						currentDirectory = string(currentDirectory[:len(currentDirectory)-1])
					}
					continue
				}
				break
			}
			return toolbox.FileSchema + path.Join(currentDirectory, strings.Join(fragments[index:], "/"))
		}

		currentDirectory, err := os.Getwd()
		if err == nil {
			candidate := path.Join(currentDirectory, URL)
			URL = candidate
		}
	}
	return toolbox.FileSchema + URL
}

func (r *Resource) Init() (err error) {
	if r.init {
		return nil
	}
	r.init = true
	r.URL = normalizeURL(r.URL)
	r.ParsedURL, err = url.Parse(r.URL)
	return err
}

//DownloadBase64 loads base64 resource content
func (r *Resource) DownloadBase64() (string, error) {
	storageService, err := storage.NewServiceForURL(r.URL, r.Credentials)
	if err != nil {
		return "", err
	}
	reader, err := storage.Download(storageService, r.URL)
	if err != nil {
		return "", err
	}
	defer reader.Close()
	data, err := ioutil.ReadAll(reader)
	if err != nil {
		return "", err
	}
	_, err = base64.StdEncoding.DecodeString(string(data))
	if err == nil {
		return string(data), nil
	}
	return base64.StdEncoding.EncodeToString(data), nil
}

//NewResource returns a new resource for provided URL, followed by optional credential, cache and cache expiryMs.
func NewResource(Params ...interface{}) *Resource {
	if len(Params) == 0 {
		return nil
	}
	var URL = toolbox.AsString(Params[0])
	URL = normalizeURL(URL)

	var credential string
	if len(Params) > 1 {
		credential = toolbox.AsString(Params[1])
	}
	var cache string
	if len(Params) > 2 {
		cache = toolbox.AsString(Params[2])
	}
	var cacheExpiryMs int
	if len(Params) > 3 {
		cacheExpiryMs = toolbox.AsInt(Params[3])
	}
	parsedURL, _ := url.Parse(URL)
	return &Resource{
		init:          true,
		ParsedURL:     parsedURL,
		URL:           URL,
		Credentials:   credential,
		Cache:         cache,
		CacheExpiryMs: cacheExpiryMs,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"syscall"
	"time"

	"code.google.com/p/goplan9/plan9/acme"
)

var args []string
var win *acme.Win
var needrun = make(chan bool, 1)

var kq struct {
	fd int
	dir *os.File
	m map[string]*os.File
	name map[int]string
}

func kadd(fd int) {
	kbuf := make([]syscall.Kevent_t, 1)
	kbuf[0] = syscall.Kevent_t{
		Ident: uint64(fd),
		Filter: syscall.EVFILT_VNODE,
		Flags: syscall.EV_ADD | syscall.EV_RECEIPT | syscall.EV_ONESHOT,
		Fflags: syscall.NOTE_DELETE | syscall.NOTE_EXTEND | syscall.NOTE_WRITE,
	}
	n, err := syscall.Kevent(kq.fd, kbuf[:1], kbuf[:1], nil)
	if err != nil {
		log.Fatalf("kevent: %v", err)
	}
	ev := &kbuf[0]
	if n != 1 || (ev.Flags&syscall.EV_ERROR) == 0 || int(ev.Ident) != int(fd) || int(ev.Filter) != syscall.EVFILT_VNODE {
		log.Fatal("kqueue phase error")
	}
	if ev.Data != 0 {
		log.Fatalf("kevent: kqueue error %s", syscall.Errno(ev.Data))
	}
}

func main() {
	flag.Parse()
	args = flag.Args()

	var err error
	win, err = acme.New()
	if err != nil {
		log.Fatal(err)
	}
	pwd, _ := os.Getwd()
	win.Name(pwd + "/+watch")
	win.Ctl("clean")
	win.Fprintf("tag", "Get ")
	needrun <- true
	go events()
	go runner()

	kq.fd, err = syscall.Kqueue()
	if err != nil {
		log.Fatal(err)
	}
	kq.m = make(map[string]*os.File)
	kq.name = make(map[int]string)

	dir, err := os.Open(".")
	if err != nil {
		log.Fatal(err)
	}
	kq.dir = dir
	kadd(int(dir.Fd()))
	readdir := true

	for {
		if readdir {
		kq.dir.Seek(0, 0)
		names, err := kq.dir.Readdirnames(-1)
		if err != nil {
			log.Fatalf("readdir: %v", err)
		}
		for _, name := range names {
			if kq.m[name] != nil {
				continue
			}
			f, err := os.Open(name)
			if err != nil {
				continue
			}
			kq.m[name] = f
			fd := int(f.Fd())
			kq.name[fd] = name
			kadd(fd)
		}
		}

		kbuf := make([]syscall.Kevent_t, 1)
		var n int
		for {
			n, err = syscall.Kevent(kq.fd, nil, kbuf[:1], nil)
			if err == syscall.EINTR {
				continue
			}
			break
		}
		if err != nil {
			log.Fatalf("kevent wait: %v", err)
		}
		ev := &kbuf[0]
		if n != 1 || int(ev.Filter) != syscall.EVFILT_VNODE {
			log.Fatal("kqueue phase error")
		}

		select {
		case needrun <- true:
		default:
		}

		fd := int(ev.Ident)
		readdir = fd == int(kq.dir.Fd())
		time.Sleep(100*time.Millisecond)
		kadd(fd)
	}
}

func events() {
	for e := range win.EventChan() {
		switch e.C2 {
		case 'x', 'X': // execute
			if string(e.Text) == "Get" {
				select {
				case needrun <- true:
				default:
				}
				continue
			}
			if string(e.Text) == "Del" {
				win.Ctl("delete")
			}
		}
		win.WriteEvent(e)
	}
	os.Exit(0)
}

var run struct {
	sync.Mutex
	id int
}

func runner() {
	var lastcmd *exec.Cmd
	for _ = range needrun {
		run.Lock()
		run.id++
		id := run.id
		run.Unlock()
		if lastcmd != nil {
			lastcmd.Process.Kill()
		}
		lastcmd = nil
		cmd := exec.Command(args[0], args[1:]...)
		r, w, err := os.Pipe()
		if err != nil {
			log.Fatal(err)
		}
		win.Addr(",")
		win.Write("data", nil)
		win.Ctl("clean")
		win.Fprintf("body", "$ %s\n", strings.Join(args, " "))
		cmd.Stdout = w
		cmd.Stderr = w
		if err := cmd.Start(); err != nil {
			r.Close()
			w.Close()
			win.Fprintf("body", "%s: %s\n", strings.Join(args, " "), err)
			continue
		}
		lastcmd = cmd
		w.Close()
		go func() {
			buf := make([]byte, 4096)
			for {
				n, err := r.Read(buf)
				if err != nil {
					break
				}
				run.Lock()
				if id == run.id {
					win.Write("body", buf[:n])
				}
				run.Unlock()
			}
			if err := cmd.Wait(); err != nil {
				run.Lock()
				if id == run.id {
					win.Fprintf("body", "%s: %s\n", strings.Join(args, " "), err)
				}
				run.Unlock()
			}
			win.Fprintf("body", "$\n")
			win.Fprintf("addr", "#0")
			win.Ctl("dot=addr")
			win.Ctl("show")
			win.Ctl("clean")
		}()
	}
}
<commit_msg>cmd/Watch: bug fixes<commit_after>package main

import (
	"flag"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"syscall"
	"time"

	"code.google.com/p/goplan9/plan9/acme"
)

var args []string
var win *acme.Win
var needrun = make(chan bool, 1)

var kq struct {
	fd   int
	dir  *os.File
	m    map[string]*os.File
	name map[int]string
}

func kadd(fd int) {
	kbuf := make([]syscall.Kevent_t, 1)
	kbuf[0] = syscall.Kevent_t{
		Ident:  uint64(fd),
		Filter: syscall.EVFILT_VNODE,
		Flags:  syscall.EV_ADD | syscall.EV_RECEIPT | syscall.EV_ONESHOT,
		Fflags: syscall.NOTE_DELETE | syscall.NOTE_EXTEND | syscall.NOTE_WRITE,
	}
	n, err := syscall.Kevent(kq.fd, kbuf[:1], kbuf[:1], nil)
	if err != nil {
		log.Fatalf("kevent: %v", err)
	}
	ev := &kbuf[0]
	if n != 1 || (ev.Flags&syscall.EV_ERROR) == 0 || int(ev.Ident) != int(fd) || int(ev.Filter) != syscall.EVFILT_VNODE {
		log.Fatal("kqueue phase error")
	}
	if ev.Data != 0 {
		log.Fatalf("kevent: kqueue error %s", syscall.Errno(ev.Data))
	}
}

func main() {
	flag.Parse()
	args = flag.Args()

	var err error
	win, err = acme.New()
	if err != nil {
		log.Fatal(err)
	}
	pwd, _ := os.Getwd()
	win.Name(pwd + "/+watch")
	win.Ctl("clean")
	win.Fprintf("tag", "Get ")
	needrun <- true
	go events()
	go runner()

	kq.fd, err = syscall.Kqueue()
	if err != nil {
		log.Fatal(err)
	}
	kq.m = make(map[string]*os.File)
	kq.name = make(map[int]string)

	dir, err := os.Open(".")
	if err != nil {
		log.Fatal(err)
	}
	kq.dir = dir
	kadd(int(dir.Fd()))
	readdir := true

	for {
		if readdir {
			kq.dir.Seek(0, 0)
			names, err := kq.dir.Readdirnames(-1)
			if err != nil {
				log.Fatalf("readdir: %v", err)
			}
			for _, name := range names {
				if kq.m[name] != nil {
					continue
				}
				f, err := os.Open(name)
				if err != nil {
					continue
				}
				kq.m[name] = f
				fd := int(f.Fd())
				kq.name[fd] = name
				kadd(fd)
			}
		}

		kbuf := make([]syscall.Kevent_t, 1)
		var n int
		for {
			n, err = syscall.Kevent(kq.fd, nil, kbuf[:1], nil)
			if err == syscall.EINTR {
				continue
			}
			break
		}
		if err != nil {
			log.Fatalf("kevent wait: %v", err)
		}
		ev := &kbuf[0]
		if n != 1 || int(ev.Filter) != syscall.EVFILT_VNODE {
			log.Fatal("kqueue phase error")
		}

		select {
		case needrun <- true:
		default:
		}

		fd := int(ev.Ident)
		readdir = fd == int(kq.dir.Fd())
		time.Sleep(100 * time.Millisecond)
		kadd(fd)
	}
}

func events() {
	for e := range win.EventChan() {
		switch e.C2 {
		case 'x', 'X': // execute
			if string(e.Text) == "Get" {
				select {
				case needrun <- true:
				default:
				}
				continue
			}
			if string(e.Text) == "Del" {
				win.Ctl("delete")
			}
		}
		win.WriteEvent(e)
	}
	os.Exit(0)
}

var run struct {
	sync.Mutex
	id int
}

func runner() {
	var lastcmd *exec.Cmd
	for _ = range needrun {
		run.Lock()
		run.id++
		id := run.id
		run.Unlock()
		if lastcmd != nil {
			lastcmd.Process.Kill()
		}
		lastcmd = nil
		cmd := exec.Command(args[0], args[1:]...)
		r, w, err := os.Pipe()
		if err != nil {
			log.Fatal(err)
		}
		win.Addr(",")
		win.Write("data", nil)
		win.Ctl("clean")
		win.Fprintf("body", "$ %s\n", strings.Join(args, " "))
		cmd.Stdout = w
		cmd.Stderr = w
		if err := cmd.Start(); err != nil {
			r.Close()
			w.Close()
			win.Fprintf("body", "%s: %s\n", strings.Join(args, " "), err)
			continue
		}
		lastcmd = cmd
		w.Close()
		go func() {
			buf := make([]byte, 4096)
			for {
				n, err := r.Read(buf)
				if err != nil {
					break
				}
				run.Lock()
				if id == run.id {
					win.Write("body", buf[:n])
				}
				run.Unlock()
			}
			if err := cmd.Wait(); err != nil {
				run.Lock()
				if id == run.id {
					win.Fprintf("body", "%s: %s\n", strings.Join(args, " "), err)
				}
				run.Unlock()
			}
			win.Fprintf("body", "$\n")
			win.Fprintf("addr", "#0")
			win.Ctl("dot=addr")
			win.Ctl("show")
			win.Ctl("clean")
		}()
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package bsp offers binary space partitioning algorithm.
package bsp

const (
	MaxSize = 2048
	minSize = 1
)

type Page struct {
	root *Node
}

func (p *Page) IsEmpty() bool {
	if p.root == nil {
		return true
	}
	return !p.root.used && p.root.child0 == nil && p.root.child1 == nil
}

type Node struct {
	x      int
	y      int
	width  int
	height int

	used   bool
	parent *Node
	child0 *Node
	child1 *Node
}

func (n *Node) canFree() bool {
	if n.used {
		return false
	}
	if n.child0 == nil && n.child1 == nil {
		return true
	}
	return n.child0.canFree() && n.child1.canFree()
}

func (n *Node) Region() (x, y, width, height int) {
	return n.x, n.y, n.width, n.height
}

// square returns a float value indicating how much the given rectangle is close to a square.
// If the given rectangle is square, this return 1 (maximum value).
// Otherwise, this returns a value in [0, 1).
func square(width, height int) float64 {
	if width == 0 && height == 0 {
		return 0
	}
	if width <= height {
		return float64(width) / float64(height)
	}
	return float64(height) / float64(width)
}

func (n *Node) alloc(width, height int) *Node {
	if n.width < width || n.height < height {
		return nil
	}
	if n.used {
		return nil
	}
	if n.child0 == nil && n.child1 == nil {
		if n.width == width && n.height == height {
			n.used = true
			return n
		}
		if square(n.width-width, n.height) >= square(n.width, n.height-height) {
			// Split vertically
			n.child0 = &Node{
				x:      n.x,
				y:      n.y,
				width:  width,
				height: n.height,
				parent: n,
			}
			n.child1 = &Node{
				x:      n.x + width,
				y:      n.y,
				width:  n.width - width,
				height: n.height,
				parent: n,
			}
		} else {
			// Split holizontally
			n.child0 = &Node{
				x:      n.x,
				y:      n.y,
				width:  n.width,
				height: height,
				parent: n,
			}
			n.child1 = &Node{
				x:      n.x,
				y:      n.y + height,
				width:  n.width,
				height: n.height - height,
				parent: n,
			}
		}
		return n.child0.alloc(width, height)
	}
	if n.child0 == nil || n.child1 == nil {
		panic("not reached")
	}
	if node := n.child0.alloc(width, height); node != nil {
		return node
	}
	if node := n.child1.alloc(width, height); node != nil {
		return node
	}
	return nil
}

func (p *Page) Alloc(width, height int) *Node {
	if width <= 0 || height <= 0 {
		panic("bsp: width and height must > 0")
	}
	if p.root == nil {
		p.root = &Node{
			width:  MaxSize,
			height: MaxSize,
		}
	}
	if width < minSize {
		width = minSize
	}
	if height < minSize {
		height = minSize
	}
	return p.root.alloc(width, height)
}

func (p *Page) Free(node *Node) {
	if node.child0 != nil || node.child1 != nil {
		panic("bsp: can't free the node including children")
	}
	node.used = false
	if node.parent == nil {
		return
	}
	if node.parent.child0 == nil || node.parent.child1 == nil {
		panic("not reached")
	}
	if node.parent.child0.canFree() && node.parent.child1.canFree() {
		node.parent.child0 = nil
		node.parent.child1 = nil
		p.Free(node.parent)
	}
}
<commit_msg>bsp: Make bsp concurrent safe<commit_after>// Copyright 2018 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package bsp offers binary space partitioning algorithm.
package bsp

import (
	"github.com/hajimehoshi/ebiten/internal/sync"
)

const (
	MaxSize = 2048
	minSize = 1
)

type Page struct {
	root *Node
	m    sync.Mutex
}

func (p *Page) IsEmpty() bool {
	p.m.Lock()
	if p.root == nil {
		p.m.Unlock()
		return true
	}
	r := !p.root.used && p.root.child0 == nil && p.root.child1 == nil
	p.m.Unlock()
	return r
}

type Node struct {
	x      int
	y      int
	width  int
	height int

	used   bool
	parent *Node
	child0 *Node
	child1 *Node
}

func (n *Node) canFree() bool {
	if n.used {
		return false
	}
	if n.child0 == nil && n.child1 == nil {
		return true
	}
	return n.child0.canFree() && n.child1.canFree()
}

func (n *Node) Region() (x, y, width, height int) {
	return n.x, n.y, n.width, n.height
}

// square returns a float value indicating how much the given rectangle is close to a square.
// If the given rectangle is square, this return 1 (maximum value).
// Otherwise, this returns a value in [0, 1).
func square(width, height int) float64 {
	if width == 0 && height == 0 {
		return 0
	}
	if width <= height {
		return float64(width) / float64(height)
	}
	return float64(height) / float64(width)
}

func (n *Node) alloc(width, height int) *Node {
	if n.width < width || n.height < height {
		return nil
	}
	if n.used {
		return nil
	}
	if n.child0 == nil && n.child1 == nil {
		if n.width == width && n.height == height {
			n.used = true
			return n
		}
		if square(n.width-width, n.height) >= square(n.width, n.height-height) {
			// Split vertically
			n.child0 = &Node{
				x:      n.x,
				y:      n.y,
				width:  width,
				height: n.height,
				parent: n,
			}
			n.child1 = &Node{
				x:      n.x + width,
				y:      n.y,
				width:  n.width - width,
				height: n.height,
				parent: n,
			}
		} else {
			// Split holizontally
			n.child0 = &Node{
				x:      n.x,
				y:      n.y,
				width:  n.width,
				height: height,
				parent: n,
			}
			n.child1 = &Node{
				x:      n.x,
				y:      n.y + height,
				width:  n.width,
				height: n.height - height,
				parent: n,
			}
		}
		return n.child0.alloc(width, height)
	}
	if n.child0 == nil || n.child1 == nil {
		panic("not reached")
	}
	if node := n.child0.alloc(width, height); node != nil {
		return node
	}
	if node := n.child1.alloc(width, height); node != nil {
		return node
	}
	return nil
}

func (p *Page) Alloc(width, height int) *Node {
	p.m.Lock()
	if width <= 0 || height <= 0 {
		panic("bsp: width and height must > 0")
	}
	if p.root == nil {
		p.root = &Node{
			width:  MaxSize,
			height: MaxSize,
		}
	}
	if width < minSize {
		width = minSize
	}
	if height < minSize {
		height = minSize
	}
	n := p.root.alloc(width, height)
	p.m.Unlock()
	return n
}

func (p *Page) Free(node *Node) {
	p.m.Lock()
	p.free(node)
	p.m.Unlock()
}

func (p *Page) free(node *Node) {
	if node.child0 != nil || node.child1 != nil {
		panic("bsp: can't free the node including children")
	}
	node.used = false
	if node.parent == nil {
		return
	}
	if node.parent.child0 == nil || node.parent.child1 == nil {
		panic("not reached")
	}
	if node.parent.child0.canFree() && node.parent.child1.canFree() {
		node.parent.child0 = nil
		node.parent.child1 = nil
		p.free(node.parent)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

// Command bzzup uploads files to the swarm HTTP API.
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

func main() {
	var (
		bzzapiFlag    = flag.String("bzzapi", "http://127.0.0.1:8500", "Swarm HTTP endpoint")
		recursiveFlag = flag.Bool("recursive", false, "Upload directories recursively")
		manifestFlag  = flag.Bool("manifest", true, "Skip automatic manifest upload")
	)
	log.SetOutput(os.Stderr)
	log.SetFlags(0)
	flag.Parse()
	if flag.NArg() != 1 {
		log.Fatal("need filename as the first and only argument")
	}

	var (
		file   = flag.Arg(0)
		client = &client{api: *bzzapiFlag}
		mroot  manifest
	)
	fi, err := os.Stat(file)
	if err != nil {
		log.Fatal(err)
	}
	if fi.IsDir() {
		if !*recursiveFlag {
			log.Fatal("argument is a directory and recursive upload is disabled")
		}
		mroot, err = client.uploadDirectory(file)
	} else {
		mroot, err = client.uploadFile(file, fi)
		if *manifestFlag {
			// Wrap the raw file entry in a proper manifest so both hashes get printed.
			mroot = manifest{Entries: []manifest{mroot}}
		}
	}
	if err != nil {
		log.Fatalln("upload failed:", err)
	}
	if *manifestFlag {
		hash, err := client.uploadManifest(mroot)
		if err != nil {
			log.Fatalln("manifest upload failed:", err)
		}
		mroot.Hash = hash
	}

	// Print the manifest. This is the only output to stdout.
	mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
	fmt.Println(string(mrootJSON))
}

// client wraps interaction with the swarm HTTP gateway.
type client struct {
	api string
}

// manifest is the JSON representation of a swarm manifest.
type manifest struct {
	Hash        string     `json:"hash,omitempty"`
	ContentType string     `json:"contentType,omitempty"`
	Path        string     `json:"path,omitempty"`
	Entries     []manifest `json:"entries,omitempty"`
}

func (c *client) uploadFile(file string, fi os.FileInfo) (manifest, error) {
	hash, err := c.uploadFileContent(file, fi)
	m := manifest{
		Hash:        hash,
		ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
	}
	return m, err
}

func (c *client) uploadDirectory(dir string) (manifest, error) {
	dirm := manifest{}
	prefix := filepath.ToSlash(dir) + "/"
	err := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
		if err != nil || fi.IsDir() {
			return err
		}
		if !strings.HasPrefix(path, dir) {
			return fmt.Errorf("path %s outside directory %s", path, dir)
		}
		entry, err := c.uploadFile(path, fi)
		entry.Path = strings.TrimPrefix(filepath.ToSlash(path), prefix)
		dirm.Entries = append(dirm.Entries, entry)
		return err
	})
	return dirm, err
}

func (c *client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
	fd, err := os.Open(file)
	if err != nil {
		return "", err
	}
	defer fd.Close()
	log.Printf("uploading file %s (%d bytes)", file, fi.Size())
	return c.postRaw("application/octet-stream", fi.Size(), fd)
}

func (c *client) uploadManifest(m manifest) (string, error) {
	jsm, err := json.Marshal(m)
	if err != nil {
		panic(err)
	}
	log.Println("uploading manifest")
	return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
}

func (c *client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
	req, err := http.NewRequest("POST", c.api+"/bzzr:/", body)
	if err != nil {
		return "", err
	}
	req.Header.Set("content-type", mimetype)
	req.ContentLength = size
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 400 {
		return "", fmt.Errorf("bad status: %s", resp.Status)
	}
	content, err := ioutil.ReadAll(resp.Body)
	return string(content), err
}
<commit_msg>cmd/bzzup: trim directory in the manifest entry path (#3299)<commit_after>// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

// Command bzzup uploads files to the swarm HTTP API.
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

func main() {
	var (
		bzzapiFlag    = flag.String("bzzapi", "http://127.0.0.1:8500", "Swarm HTTP endpoint")
		recursiveFlag = flag.Bool("recursive", false, "Upload directories recursively")
		manifestFlag  = flag.Bool("manifest", true, "Skip automatic manifest upload")
	)
	log.SetOutput(os.Stderr)
	log.SetFlags(0)
	flag.Parse()
	if flag.NArg() != 1 {
		log.Fatal("need filename as the first and only argument")
	}

	var (
		file   = flag.Arg(0)
		client = &client{api: *bzzapiFlag}
		mroot  manifest
	)
	fi, err := os.Stat(file)
	if err != nil {
		log.Fatal(err)
	}
	if fi.IsDir() {
		if !*recursiveFlag {
			log.Fatal("argument is a directory and recursive upload is disabled")
		}
		mroot, err = client.uploadDirectory(file)
	} else {
		mroot, err = client.uploadFile(file, fi)
		if *manifestFlag {
			// Wrap the raw file entry in a proper manifest so both hashes get printed.
			mroot = manifest{Entries: []manifest{mroot}}
		}
	}
	if err != nil {
		log.Fatalln("upload failed:", err)
	}
	if *manifestFlag {
		hash, err := client.uploadManifest(mroot)
		if err != nil {
			log.Fatalln("manifest upload failed:", err)
		}
		mroot.Hash = hash
	}

	// Print the manifest. This is the only output to stdout.
	mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
	fmt.Println(string(mrootJSON))
}

// client wraps interaction with the swarm HTTP gateway.
type client struct {
	api string
}

// manifest is the JSON representation of a swarm manifest.
type manifest struct {
	Hash        string     `json:"hash,omitempty"`
	ContentType string     `json:"contentType,omitempty"`
	Path        string     `json:"path,omitempty"`
	Entries     []manifest `json:"entries,omitempty"`
}

func (c *client) uploadFile(file string, fi os.FileInfo) (manifest, error) {
	hash, err := c.uploadFileContent(file, fi)
	m := manifest{
		Hash:        hash,
		ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
	}
	return m, err
}

func (c *client) uploadDirectory(dir string) (manifest, error) {
	dirm := manifest{}
	prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
	err := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
		if err != nil || fi.IsDir() {
			return err
		}
		if !strings.HasPrefix(path, dir) {
			return fmt.Errorf("path %s outside directory %s", path, dir)
		}
		entry, err := c.uploadFile(path, fi)
		entry.Path = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
		dirm.Entries = append(dirm.Entries, entry)
		return err
	})
	return dirm, err
}

func (c *client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
	fd, err := os.Open(file)
	if err != nil {
		return "", err
	}
	defer fd.Close()
	log.Printf("uploading file %s (%d bytes)", file, fi.Size())
	return c.postRaw("application/octet-stream", fi.Size(), fd)
}

func (c *client) uploadManifest(m manifest) (string, error) {
	jsm, err := json.Marshal(m)
	if err != nil {
		panic(err)
	}
	log.Println("uploading manifest")
	return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
}

func (c *client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
	req, err := http.NewRequest("POST", c.api+"/bzzr:/", body)
	if err != nil {
		return "", err
	}
	req.Header.Set("content-type", mimetype)
	req.ContentLength = size
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 400 {
		return "", fmt.Errorf("bad status: %s", resp.Status)
	}
	content, err := ioutil.ReadAll(resp.Body)
	return string(content), err
}
<|endoftext|>


<commit_before>// Taken from https://gist.github.com/tam7t/1b45125ae4de13b3fc6fd0455954c08e

package main

import (
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
)

func main() {
	err := realMain()
	if err != nil {
		log.Fatal(err)
	}
	os.Exit(0)
}

func realMain() error {
	if len(os.Args) != 4 {
		// Ensure the empty input case is handled correctly
		return nil
	}

	// certdump <filepath> <owner> <data>
	path := os.Args[1]
	owner := os.Args[2]
	data := os.Args[3]

	// Ensure folder exists
	dir := filepath.Dir(path)
	if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil {
		return err
	}

	err := ioutil.WriteFile(path, []byte(data), 0700)
	if err != nil {
		return err
	}
	u, err := user.Lookup(owner)
	if err != nil {
		return err
	}
	uid, err := strconv.Atoi(u.Uid)
	if err != nil {
		return err
	}
	gid := os.Getgid()
	err = os.Chmod(path, 0660)
	if err != nil {
		return err
	}
	err = os.Chown(path, uid, gid)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Set gid to gid of user, not gid or running process<commit_after>// Taken from https://gist.github.com/tam7t/1b45125ae4de13b3fc6fd0455954c08e

package main

import (
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
)

func main() {
	err := realMain()
	if err != nil {
		log.Fatal(err)
	}
	os.Exit(0)
}

func realMain() error {
	if len(os.Args) != 4 {
		// Ensure the empty input case is handled correctly
		return nil
	}

	// certdump <filepath> <owner> <data>
	path := os.Args[1]
	owner := os.Args[2]
	data := os.Args[3]

	// Ensure folder exists
	dir := filepath.Dir(path)
	if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil {
		return err
	}

	err := ioutil.WriteFile(path, []byte(data), 0700)
	if err != nil {
		return err
	}
	u, err := user.Lookup(owner)
	if err != nil {
		return err
	}
	uid, err := strconv.Atoi(u.Uid)
	if err != nil {
		return err
	}
	gid, err := strconv.Atoi(u.Gid)
	if err != nil {
		return err
	}
	err = os.Chmod(path, 0660)
	if err != nil {
		return err
	}
	err = os.Chown(path, uid, gid)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/go-kit/kit/log"
	"github.com/spf13/pflag"
	"golang.org/x/net/context"
	"k8s.io/kubernetes/pkg/client/restclient"

	"github.com/weaveworks/fluxy"
	"github.com/weaveworks/fluxy/platform/kubernetes"
	"github.com/weaveworks/fluxy/registry"
)

func main() {
	// Flag domain.
	pflag.Usage = func() {
		fmt.Fprintf(os.Stderr, "DESCRIPTION\n")
		fmt.Fprintf(os.Stderr, "  fluxd is a deployment daemon.\n")
		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "FLAGS\n")
		pflag.PrintDefaults()
	}
	var (
		listenAddr              = pflag.StringP("listen", "l", ":3030", "Listen address for Flux API clients")
		registryCredentials     = pflag.StringP("registry-credentials", "", "", "Path to image registry credentials file, in the format of ~/.docker/config.json")
		kubernetesEnabled       = pflag.BoolP("kubernetes", "", false, "Enable Kubernetes platform")
		kubernetesHost          = pflag.StringP("kubernetes-host", "", "", "Kubernetes host, e.g. http://10.11.12.13:8080")
		kubernetesUsername      = pflag.StringP("kubernetes-username", "", "", "Kubernetes HTTP basic auth username")
		kubernetesPassword      = pflag.StringP("kubernetes-password", "", "", "Kubernetes HTTP basic auth password")
		kubernetesClientCert    = pflag.StringP("kubernetes-client-certificate", "", "", "Path to Kubernetes client certification file for TLS")
		kubernetesClientKey     = pflag.StringP("kubernetes-client-key", "", "", "Path to Kubernetes client key file for TLS")
		kubernetesCertAuthority = pflag.StringP("kubernetes-certificate-authority", "", "", "Path to Kubernetes cert file for certificate authority")
	)
	pflag.Parse()

	// Logger domain.
	var logger log.Logger
	{
		logger = log.NewLogfmtLogger(os.Stderr)
		logger = log.NewContext(logger).With("ts", log.DefaultTimestampUTC)
		logger = log.NewContext(logger).With("caller", log.DefaultCaller)
	}

	// Registry component.
	var reg *registry.Client
	{
		logger := log.NewContext(logger).With("component", "registry")
		creds := registry.NoCredentials()
		if *registryCredentials != "" {
			logger.Log("credentials", *registryCredentials)
			c, err := registry.CredentialsFromFile(*registryCredentials)
			if err != nil {
				logger.Log("err", err)
				os.Exit(1)
			}
			creds = c
		} else {
			logger.Log("credentials", "none")
		}
		reg = &registry.Client{
			Credentials: creds,
			Logger:      logger,
		}
	}

	// Platform component.
	var k8s *kubernetes.Cluster
	if *kubernetesEnabled {
		logger := log.NewContext(logger).With("component", "Kubernetes")
		logger.Log("host", kubernetesHost)

		var err error
		k8s, err = kubernetes.NewCluster(&restclient.Config{
			Host:     *kubernetesHost,
			Username: *kubernetesUsername,
			Password: *kubernetesPassword,
			TLSClientConfig: restclient.TLSClientConfig{
				CertFile: *kubernetesClientCert,
				KeyFile:  *kubernetesClientKey,
				CAFile:   *kubernetesCertAuthority,
			},
		}, logger)
		if err != nil {
			logger.Log("err", err)
			os.Exit(1)
		}

		if services, err := k8s.Services("default"); err != nil {
			logger.Log("services", err)
		} else {
			logger.Log("services", len(services))
		}
	}

	// Service (business logic) domain.
	var service flux.Service
	{
		service = flux.NewService(reg, k8s)
		service = flux.LoggingMiddleware(logger)(service)
	}

	// Endpoint domain.
	var endpoints flux.Endpoints
	{
		endpoints = flux.MakeServerEndpoints(service)
	}

	// Mechanical stuff.
	errc := make(chan error)
	go func() {
		c := make(chan os.Signal)
		signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
		errc <- fmt.Errorf("%s", <-c)
	}()

	// Transport domain.
	ctx := context.Background()
	go func() {
		logger := log.NewContext(logger).With("transport", "HTTP")
		logger.Log("addr", *listenAddr)
		h := flux.MakeHTTPHandler(ctx, endpoints, logger)
		errc <- http.ListenAndServe(*listenAddr, h)
	}()

	// Go!
	logger.Log("exit", <-errc)
}
<commit_msg>cmd/fluxd: log platform and registry consistently<commit_after>package main

import (
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/go-kit/kit/log"
	"github.com/spf13/pflag"
	"golang.org/x/net/context"
	"k8s.io/kubernetes/pkg/client/restclient"

	"github.com/weaveworks/fluxy"
	"github.com/weaveworks/fluxy/platform/kubernetes"
	"github.com/weaveworks/fluxy/registry"
)

func main() {
	// Flag domain.
	pflag.Usage = func() {
		fmt.Fprintf(os.Stderr, "DESCRIPTION\n")
		fmt.Fprintf(os.Stderr, "  fluxd is a deployment daemon.\n")
		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "FLAGS\n")
		pflag.PrintDefaults()
	}
	var (
		listenAddr              = pflag.StringP("listen", "l", ":3030", "Listen address for Flux API clients")
		registryCredentials     = pflag.StringP("registry-credentials", "", "", "Path to image registry credentials file, in the format of ~/.docker/config.json")
		kubernetesEnabled       = pflag.BoolP("kubernetes", "", false, "Enable Kubernetes platform")
		kubernetesHost          = pflag.StringP("kubernetes-host", "", "", "Kubernetes host, e.g. http://10.11.12.13:8080")
		kubernetesUsername      = pflag.StringP("kubernetes-username", "", "", "Kubernetes HTTP basic auth username")
		kubernetesPassword      = pflag.StringP("kubernetes-password", "", "", "Kubernetes HTTP basic auth password")
		kubernetesClientCert    = pflag.StringP("kubernetes-client-certificate", "", "", "Path to Kubernetes client certification file for TLS")
		kubernetesClientKey     = pflag.StringP("kubernetes-client-key", "", "", "Path to Kubernetes client key file for TLS")
		kubernetesCertAuthority = pflag.StringP("kubernetes-certificate-authority", "", "", "Path to Kubernetes cert file for certificate authority")
	)
	pflag.Parse()

	// Logger domain.
	var logger log.Logger
	{
		logger = log.NewLogfmtLogger(os.Stderr)
		logger = log.NewContext(logger).With("ts", log.DefaultTimestampUTC)
		logger = log.NewContext(logger).With("caller", log.DefaultCaller)
	}

	// Registry component.
	var reg *registry.Client
	{
		logger := log.NewContext(logger).With("component", "registry")
		creds := registry.NoCredentials()
		if *registryCredentials != "" {
			logger.Log("credentials", *registryCredentials)
			c, err := registry.CredentialsFromFile(*registryCredentials)
			if err != nil {
				logger.Log("err", err)
				os.Exit(1)
			}
			creds = c
		} else {
			logger.Log("credentials", "none")
		}
		reg = &registry.Client{
			Credentials: creds,
			Logger:      logger,
		}
	}

	// Platform component.
	var k8s *kubernetes.Cluster
	{
		logger := log.NewContext(logger).With("component", "platform")
		if *kubernetesEnabled {
			logger.Log("kind", "Kubernetes", "host", kubernetesHost)

			var err error
			k8s, err = kubernetes.NewCluster(&restclient.Config{
				Host:     *kubernetesHost,
				Username: *kubernetesUsername,
				Password: *kubernetesPassword,
				TLSClientConfig: restclient.TLSClientConfig{
					CertFile: *kubernetesClientCert,
					KeyFile:  *kubernetesClientKey,
					CAFile:   *kubernetesCertAuthority,
				},
			}, logger)
			if err != nil {
				logger.Log("err", err)
				os.Exit(1)
			}

			if services, err := k8s.Services("default"); err != nil {
				logger.Log("services", err)
			} else {
				logger.Log("services", len(services))
			}
		} else {
			logger.Log("kind", "none")
		}
	}

	// Service (business logic) domain.
	var service flux.Service
	{
		service = flux.NewService(reg, k8s)
		service = flux.LoggingMiddleware(logger)(service)
	}

	// Endpoint domain.
	var endpoints flux.Endpoints
	{
		endpoints = flux.MakeServerEndpoints(service)
	}

	// Mechanical stuff.
	errc := make(chan error)
	go func() {
		c := make(chan os.Signal)
		signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
		errc <- fmt.Errorf("%s", <-c)
	}()

	// Transport domain.
	ctx := context.Background()
	go func() {
		logger := log.NewContext(logger).With("transport", "HTTP")
		logger.Log("addr", *listenAddr)
		h := flux.MakeHTTPHandler(ctx, endpoints, logger)
		errc <- http.ListenAndServe(*listenAddr, h)
	}()

	// Go!
	logger.Log("exit", <-errc)
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/google/go-tpm-tools/cmd"
)

func main() {
	if cmd.RootCmd.Execute() != nil {
		os.Exit(1)
	}
}
<commit_msg>Add package comment to main package<commit_after>// Package main is a binary wrapper package around cmd.
package main

import (
	"os"

	"github.com/google/go-tpm-tools/cmd"
)

func main() {
	if cmd.RootCmd.Execute() != nil {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/kubernetes/helm/pkg/repo"
	"github.com/spf13/cobra"
)

func init() {
	RootCommand.AddCommand(fetchCmd)
}

var fetchCmd = &cobra.Command{
	Use:   "fetch [chart URL | repo/chartname]",
	Short: "Download a chart from a repository and (optionally) unpack it in local directory.",
	Long:  "",
	RunE:  fetch,
}

func fetch(cmd *cobra.Command, args []string) error {
	if len(args) == 0 {
		return fmt.Errorf("This command needs at least one argument, url or repo/name of the chart.")
	}

	f, err := repo.LoadRepositoriesFile(repositoriesFile())
	if err != nil {
		return err
	}

	// get download url
	u, err := mapRepoArg(args[0], f.Repositories)
	if err != nil {
		return err
	}

	// Grab the package name that we'll use for the name of the file to download to.
	p := strings.Split(u.String(), "/")
	chartName := p[len(p)-1]
	out, err := os.Create(chartName)
	if err != nil {
		return err
	}
	defer out.Close()
	resp, err := http.Get(u)
	// unpack file
	if err != nil {
		return err
	}

	defer resp.Body.Close()

	_, err = io.Copy(out, resp.Body)
	if err != nil {
		return err
	}
	return nil
}

// mapRepoArg figures out which format the argument is given, and creates a fetchable
// url from it.
func mapRepoArg(arg string, r map[string]string) (*url.URL, error) {
	// See if it's already a full URL.
	u, err := url.ParseRequestURI(arg)
	if err == nil {
		// If it has a scheme and host and path, it's a full URL
		if u.IsAbs() && len(u.Host) > 0 && len(u.Path) > 0 {
			return u, nil
		} else {
			return nil, fmt.Errorf("Invalid chart url format: %s", arg)
		}
	}
	// See if it's of the form: repo/path_to_chart
	p := strings.Split(arg, "/")
	if len(p) > 1 {
		if baseUrl, ok := r[p[0]]; ok {
			if !strings.HasSuffix(baseUrl, "/") {
				baseUrl = baseUrl + "/"
			}
			return url.ParseRequestURI(baseUrl + strings.Join(p[1:], "/"))
		} else {
			return nil, fmt.Errorf("No such repo: %s", p[0])
		}
	} else {
		return nil, fmt.Errorf("Invalid chart url format: %s", arg)
	}
}
<commit_msg>cleanups, add todos<commit_after>package main

import (
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/kubernetes/helm/pkg/chart"
	"github.com/kubernetes/helm/pkg/repo"
	"github.com/spf13/cobra"
)

func init() {
	RootCommand.AddCommand(fetchCmd)
}

var fetchCmd = &cobra.Command{
	Use:   "fetch [chart URL | repo/chartname]",
	Short: "Download a chart from a repository and (optionally) unpack it in local directory.",
	Long:  "",
	RunE:  fetch,
}

func fetch(cmd *cobra.Command, args []string) error {
	if len(args) == 0 {
		return fmt.Errorf("This command needs at least one argument, url or repo/name of the chart.")
	}

	f, err := repo.LoadRepositoriesFile(repositoriesFile())
	if err != nil {
		return err
	}

	// get download url
	u, err := mapRepoArg(args[0], f.Repositories)
	if err != nil {
		return err
	}

	resp, err := http.Get(u.String())
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return fmt.Errorf("Failed to fetch %s : %s", u.String(), resp.Status)
	}

	defer resp.Body.Close()
	// TODO(vaikas): Implement untar / flag
	untar := false
	if untar {
		return untarChart(resp.Body)
	}
	p := strings.Split(u.String(), "/")
	return saveChartFile(p[len(p)-1], resp.Body)
}

// mapRepoArg figures out which format the argument is given, and creates a fetchable
// url from it.
func mapRepoArg(arg string, r map[string]string) (*url.URL, error) {
	// See if it's already a full URL.
	u, err := url.ParseRequestURI(arg)
	if err == nil {
		// If it has a scheme and host and path, it's a full URL
		if u.IsAbs() && len(u.Host) > 0 && len(u.Path) > 0 {
			return u, nil
		}
		return nil, fmt.Errorf("Invalid chart url format: %s", arg)
	}
	// See if it's of the form: repo/path_to_chart
	p := strings.Split(arg, "/")
	if len(p) > 1 {
		if baseURL, ok := r[p[0]]; ok {
			if !strings.HasSuffix(baseURL, "/") {
				baseURL = baseURL + "/"
			}
			return url.ParseRequestURI(baseURL + strings.Join(p[1:], "/"))
		}
		return nil, fmt.Errorf("No such repo: %s", p[0])
	}
	return nil, fmt.Errorf("Invalid chart url format: %s", arg)
}

func untarChart(r io.Reader) error {
	c, err := chart.LoadDataFromReader(r)
	if err != nil {
		return err
	}
	if c == nil {
		fmt.Errorf("Failed to untar the chart")
	}
	return fmt.Errorf("Not implemented yeet")

}

func saveChartFile(c string, r io.Reader) error {
	// Grab the chart name that we'll use for the name of the file to download to.
	out, err := os.Create(c)
	if err != nil {
		return err
	}
	defer out.Close()

	_, err = io.Copy(out, r)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>