Feature/cleanup grids

accounts/casbin.go

@@ -24,12 +24,12 @@ func (ce *CasbinAccess) init() {
 
 func (ce *CasbinAccess) Enforce(user string, graph string, operation Operation) error {
 	ce.init()
-	fmt.Printf("Casbin request '%s' '%s' '%s'\n", user, graph, operation)
+	// fmt.Printf("Casbin request '%s' '%s' '%s'\n", user, graph, operation)
 	if res, err := ce.encforcer.Enforce(user, graph, string(operation)); res {
 		return nil
 	} else if err != nil {
 		fmt.Printf("casbin error: %s\n", err)
 	}
-	fmt.Printf("Not allowed: '%s' '%s' '%s'\n", user, graph, operation)
+	// fmt.Printf("Not allowed: '%s' '%s' '%s'\n", user, graph, operation)
 	return fmt.Errorf("action restricted")
 }

accounts/util.go

@@ -40,6 +40,16 @@ func (c *Config) init() {
 	}
 }
 
+func (c *Config) GetAuth() Authenticate {
+	c.init()
+	return c.auth
+}
+
+func (c *Config) GetAccess() Access {
+	c.init()
+	return c.access
+}
+
 func (c *Config) UnaryInterceptor() grpc.UnaryServerInterceptor {
 	c.init()
 	return unaryAuthInterceptor(c.auth, c.access)

cmd/load/main.go

@@ -103,7 +103,11 @@ var Cmd = &cobra.Command{
 					log.Infof("Loaded %d edges", count)
 				}
 				if edgeUID && e.Id == "" {
-					e.Id = util.UUID()
+					var data map[string]interface{}
+					if e.Data != nil {
+						data = e.Data.AsMap()
+					}
+					e.Id = util.DeterministicEdgeID(e.From, e.To, e.Label, data)
 				}
 				elemChan <- &gripql.GraphElement{Graph: graph, Edge: e}
 			}
@@ -142,7 +146,11 @@ var Cmd = &cobra.Command{
 							log.Infof("Loaded %d edges", edgeCount)
 						}
 						if edgeUID && e.Id == "" {
-							e.Id = util.UUID()
+							var data map[string]interface{}
+							if e.Data != nil {
+								data = e.Data.AsMap()
+							}
+							e.Id = util.DeterministicEdgeID(e.From, e.To, e.Label, data)
 						}
 						elemChan <- &gripql.GraphElement{Graph: graph, Edge: e}
 					}

cmd/mongoload/main.go

@@ -72,7 +72,11 @@ func edgeSerialize(edgeChan chan *gripql.Edge, workers int) chan []byte {
 		go func() {
 			for e := range edgeChan {
 				if edgeUID && e.Id == "" {
-					e.Id = util.UUID()
+					var data map[string]interface{}
+					if e.Data != nil {
+						data = e.Data.AsMap()
+					}
+					e.Id = util.DeterministicEdgeID(e.From, e.To, e.Label, data)
 				}
 				doc := mongo.PackEdge(gdbi.NewElementFromEdge(e))
 				rawBytes, err := bson.Marshal(doc)

cmd/server/main.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/signal"
 	"strings"
+	"syscall"
 
 	"github.com/bmeg/grip/config"
 	"github.com/bmeg/grip/log"
@@ -38,9 +39,11 @@ func Run(conf *config.Config, baseDir string) error {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+	defer signal.Stop(c)
 	go func() {
-		<-c
+		sig := <-c
+		log.Infof("Received signal %s, starting graceful shutdown", sig.String())
 		cancel()
 	}()
 

config/config.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	esql "github.com/bmeg/grip/existing-sql"
+	"github.com/bmeg/grip/grids"
 	"github.com/bmeg/grip/gripper"
 	"github.com/bmeg/grip/log"
 	"github.com/bmeg/grip/mongo"
@@ -26,7 +27,7 @@ func init() {
 }
 
 type DriverConfig struct {
-	Grids       *string
+	Grids       *grids.Config
 	Badger      *string
 	Bolt        *string
 	Level       *string
@@ -121,7 +122,8 @@ func (conf *Config) AddSqliteDefault() {
 
 func (conf *Config) AddGridsDefault() {
 	n := "grip-grids.db"
-	conf.Drivers["grids"] = DriverConfig{Grids: &n}
+	c := grids.Config{GraphDir: n, BulkLoaderWorkers: 10, Driver: "jsontable"}
+	conf.Drivers["grids"] = DriverConfig{Grids: &c}
 	conf.Default = "grids"
 }
 
@@ -135,6 +137,9 @@ func TestifyConfig(c *Config) {
 
 	c.RPCClient.ServerAddress = c.Server.RPCAddress()
 
+	if c.Default == "" {
+		return
+	}
 	d := c.Drivers[c.Default]
 
 	if d.Badger != nil {
@@ -145,6 +150,11 @@ func TestifyConfig(c *Config) {
 		a := "grip.db." + rand
 		d.Pebble = &a
 	}
+	if d.Grids != nil {
+		c := *d.Grids
+		c.GraphDir = "grip-grids.db." + rand
+		d.Grids = &c
+	}
 	if d.MongoDB != nil {
 		d.MongoDB.DBName = "gripdb-" + rand
 	}

conformance/tests/ot_aggregations.py

@@ -154,7 +154,7 @@ def test_traversal_edge_histogram_aggregation(man):
     if count < 2:
         errors.append(
             "Unexpected number of terms: %d != %d" %
-            (len(row["buckets"]), 2)
+            (count, 2)
         )
 
     return errors

conformance/tests/ot_basic.py

@@ -394,34 +394,22 @@ def test_limit(man):
 
     G = man.setGraph("swapi")
 
+    # Tests modified to only check cardinality since different database backends 
+    # (e.g., Postgres) do not guarantee deterministic result ordering without 
+    # explicit sorting.
     tests = [
-        "G.V().limit(3)",
-        "G.V().outE().limit(3)"
+        ("G.V().limit(3)", 3),
+        ("G.V().outE().limit(3)", 3)
     ]
 
-    expected_results = [
-        list(i["_id"] for i in G.V().execute())[:3],
-        list(i["_id"] for i in G.V().outE().execute())[:3]
-    ]
-
-    for test, expected in zip(tests, expected_results):
+    for test, expected_len in tests:
         results = eval(test).execute()
         actual = [x["_id"] for x in results]
 
-        # check contents
-        for x in actual:
-            if x not in expected:
-                errors.append("Fail: %s - unexpected result - %s" % (test, x))
-
         # check number of results
-        if len(actual) != len(expected):
+        if len(actual) != expected_len:
             errors.append("Fail: %s - unexpected result count - \
-            %s != %s" % (test, len(actual), len(expected)))
-
-        # check order
-        if actual != expected:
-            errors.append("Fail: %s - unexpected order - \
-            %s != %s" % (test, actual, expected))
+            %s != %s" % (test, len(actual), expected_len))
 
     return errors
 
@@ -431,32 +419,21 @@ def test_skip(man):
 
     G = man.setGraph("swapi")
 
+    # Tests modified to only check cardinality since different database backends 
+    # (e.g., Postgres) do not guarantee deterministic result ordering without 
+    # explicit sorting.
     tests = [
-        "G.V().skip(3).limit(3)",
-    ]
-
-    expected_results = [
-        list(i["_id"] for i in G.V().execute())[3:6],
+        ("G.V().skip(3).limit(3)", 3),
     ]
 
-    for test, expected in zip(tests, expected_results):
+    for test, expected_len in tests:
         results = eval(test).execute()
         actual = [x["_id"] for x in results]
 
-        # check contents
-        for x in actual:
-            if x not in expected:
-                errors.append("Fail: %s - unexpected result - %s" % (test, x))
-
         # check number of results
-        if len(actual) != len(expected):
+        if len(actual) != expected_len:
             errors.append("Fail: %s - unexpected result count - \
-            %s != %s" % (test, len(actual), len(expected)))
-
-        # check order
-        if actual != expected:
-            errors.append("Fail: %s - unexpected order - \
-            %s != %s" % (test, actual, expected))
+            %s != %s" % (test, len(actual), expected_len))
 
     return errors
 
@@ -466,33 +443,21 @@ def test_range(man):
 
     G = man.setGraph("swapi")
 
+    # Tests modified to only check cardinality since different database backends 
+    # (e.g., Postgres) do not guarantee deterministic result ordering without 
+    # explicit sorting.
     tests = [
-        "G.V().range(3, 5)",
-        "G.V().range(34, -1)",
-    ]
-
-    expected_results = [
-        list(i["_id"] for i in G.V().execute())[3:5],
-        list(i["_id"] for i in G.V().execute())[34:],
+        ("G.V().range(3, 5)", 2),
+        ("G.V().range(34, -1)", 5), # 39 Total in swapi V - 34 offset
     ]
 
-    for test, expected in zip(tests, expected_results):
+    for test, expected_len in tests:
         results = eval(test).execute()
         actual = [x["_id"] for x in results]
 
-        # check contents
-        for x in actual:
-            if x not in expected:
-                errors.append("Fail: %s - unexpected result - %s" % (test, x))
-
         # check number of results
-        if len(actual) != len(expected):
+        if len(actual) != expected_len:
             errors.append("Fail: %s - unexpected result count - \
-            %s != %s" % (test, len(actual), len(expected)))
-
-        # check order
-        if actual != expected:
-            errors.append("Fail: %s - unexpected order - \
-            %s != %s" % (test, actual, expected))
+            %s != %s" % (test, len(actual), expected_len))
 
     return errors

conformance/tests/ot_bulk.py

@@ -27,15 +27,67 @@ def test_bulkload(man):
 
     res = G.V().count().execute()[0]
     if res["count"] != 6:
-        errors.append(
-            "Bulk Add wrong number of vertices: %s != %s" %
-            (res["count"], 6))
+        errors.append("Bulk Add wrong number of vertices: %s != %s" % (res["count"], 6))
 
     res = G.V().outE().count().execute()[0]
     if res["count"] != 6:
-        errors.append(
-            "Bulk Add wrong number of edges: %s != %s" %
-            (res["count"], 6))
+        errors.append("Bulk Add wrong number of edges: %s != %s" % (res["count"], 6))
+
+    return errors
+
+
+def test_bulkload_duplicate(man):
+    errors = []
+
+    G = man.writeTest()
+
+    bulk = G.bulkAdd()
+
+    bulk.addVertex("1", "Person", {"name": "marko", "age": "29"})
+    bulk.addVertex("2", "Person", {"name": "vadas", "age": "27"})
+    bulk.addVertex("3", "Software", {"name": "lop", "lang": "java"})
+    bulk.addVertex("4", "Person", {"name": "josh", "age": "32"})
+    bulk.addVertex("5", "Software", {"name": "ripple", "lang": "java"})
+    bulk.addVertex("6", "Person", {"name": "peter", "age": "35"})
+
+    bulk.addEdge("1", "3", "created", {"weight": 0.4})
+    bulk.addEdge("1", "2", "knows", {"weight": 0.5})
+    bulk.addEdge("1", "4", "knows", {"weight": 1.0})
+    bulk.addEdge("4", "3", "created", {"weight": 0.4})
+    bulk.addEdge("6", "3", "created", {"weight": 0.2})
+    bulk.addEdge("4", "5", "created", {"weight": 1.0})
+
+    err = bulk.execute()
+
+    bulk = G.bulkAdd()
+
+    bulk.addVertex("1", "Person", {"name": "marko", "age": "29"})
+    bulk.addVertex("2", "Person", {"name": "vadas", "age": "27"})
+    bulk.addVertex("3", "Software", {"name": "lop", "lang": "java"})
+    bulk.addVertex("4", "Person", {"name": "josh", "age": "32"})
+    bulk.addVertex("5", "Software", {"name": "ripple", "lang": "java"})
+    bulk.addVertex("6", "Person", {"name": "peter", "age": "35"})
+
+    bulk.addEdge("1", "3", "created", {"weight": 0.4})
+    bulk.addEdge("1", "2", "knows", {"weight": 0.5})
+    bulk.addEdge("1", "4", "knows", {"weight": 1.0})
+    bulk.addEdge("4", "3", "created", {"weight": 0.4})
+    bulk.addEdge("6", "3", "created", {"weight": 0.2})
+    bulk.addEdge("4", "5", "created", {"weight": 1.0})
+
+    err = bulk.execute()
+
+    if err.get("errorCount", 0) != 0:
+        print(err)
+        errors.append("Bulk insertion error")
+
+    res = G.V().count().execute()[0]
+    if res["count"] != 6:
+        errors.append("Bulk Add wrong number of vertices: %s != %s" % (res["count"], 6))
+
+    res = G.V().outE().count().execute()[0]
+    if res["count"] != 6:
+        errors.append("Bulk Add wrong number of edges: %s != %s" % (res["count"], 6))
 
     return errors
 
@@ -89,9 +141,7 @@ def test_bulk_delete(man):
     G.addEdge("vertex4", "vertex5", "created", {"weight": 0.4}, id="edge8")
     G.addEdge("vertex4", "vertex6", "created", {"weight": 0.4}, id="edge9")
 
-    G.delete(vertices=["vertex1", "vertex2",
-                       "vertex3"],
-             edges=[])
+    G.delete(vertices=["vertex1", "vertex2", "vertex3"], edges=[])
 
     Ecount = G.V().outE().count().execute()[0]["count"]
     Vcount = G.V().count().execute()[0]["count"]
@@ -108,7 +158,6 @@ def test_bulk_delete(man):
     if Vcount != 3:
         errors.append(f"Wrong number of vertices {Vcount} != 3")
 
-
     G.delete(vertices=["vertex5", "vertex6"], edges=["edge9"])
     Ecount = G.V().outE().count().execute()[0]["count"]
     Vcount = G.V().count().execute()[0]["count"]

endpoints/cypher/translate/build.go

@@ -66,7 +66,7 @@ func (c *cypherListener) BuildQuery() (*gripql.Query, error) {
 			q = q.As(c.vertexPath[i].name)
 		}
 		if len(c.returns) > 0 {
-			log.Infof("Render: $" + c.returns[0] + "._data")
+			log.Infof("Render: %s", "$"+c.returns[0]+"._data")
 			r := map[string]any{}
 			for _, i := range c.returns {
 				r[i] = "$" + i + "._data"