Merge pull request #13 from mortenson/graph-perf

Move off of graph library for performance reasons, fix dependency issues with export

Author: mortenson

db.go

@@ -4,16 +4,15 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/binary"
-	"errors"
 	"fmt"
 	"log/slog"
 	"math/rand"
 	"regexp"
+	"slices"
 	"strings"
 	"time"
 
 	"github.com/brianvoe/gofakeit/v7"
-	"github.com/dominikbraun/graph"
 	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5"
 	"github.com/lib/pq"
@@ -154,7 +153,7 @@ func prepareValue(rawValue string) (string, error) {
 	return fakerResult, nil
 }
 
-func buildQueryForRow(primaryKeys PrimaryKeysResult, rowId string, row Row, dependencyGraph graph.Graph[string, string]) (string, error) {
+func buildQueryForRow(primaryKeys PrimaryKeysResult, rowId string, row Row, dependencyGraph map[string][]string) (string, error) {
 	parts := strings.Split(rowId, ":")
 	if len(parts) < 2 {
 		return "", fmt.Errorf("invalid id: %s", rowId)
@@ -203,12 +202,8 @@ func buildQueryForRow(primaryKeys PrimaryKeysResult, rowId string, row Row, depe
 			default:
 				return "", fmt.Errorf("cannot parse ~dependencies value in row %s", rowId)
 			}
-			for _, dependency := range dependencies {
-				err := dependencyGraph.AddEdge(rowId, dependency)
-				if isRealGraphError(err) {
-					return "", err
-				}
-			}
+			dependencyGraph[rowId] = append(dependencyGraph[rowId], dependencies...)
+			dependencyGraph[rowId] = slices.Compact(dependencyGraph[rowId])
 			continue
 		}
 
@@ -225,10 +220,8 @@ func buildQueryForRow(primaryKeys PrimaryKeysResult, rowId string, row Row, depe
 			addEdge := referenceRegex.MatchString(value)
 			// Don't add edges to and from the same row.
 			if addEdge && rowId != value {
-				err := dependencyGraph.AddEdge(rowId, value)
-				if isRealGraphError(err) {
-					return "", err
-				}
+				dependencyGraph[rowId] = append(dependencyGraph[rowId], value)
+				dependencyGraph[rowId] = slices.Compact(dependencyGraph[rowId])
 			}
 
 			columns = append(columns, pq.QuoteIdentifier(column))
@@ -265,15 +258,12 @@ func buildQueryForRow(primaryKeys PrimaryKeysResult, rowId string, row Row, depe
 
 // Returns a sorted array of queries to run based on a given ripoff file.
 func buildQueriesForRipoff(primaryKeys PrimaryKeysResult, totalRipoff RipoffFile) ([]string, error) {
-	dependencyGraph := graph.New(graph.StringHash, graph.Directed(), graph.Acyclic())
+	dependencyGraph := map[string][]string{}
 	queries := map[string]string{}
 
 	// Add vertexes first, since rows can be in any order.
 	for rowId := range totalRipoff.Rows {
-		err := dependencyGraph.AddVertex(rowId)
-		if err != nil {
-			return []string{}, err
-		}
+		dependencyGraph[rowId] = []string{}
 	}
 
 	// Build queries.
@@ -286,7 +276,10 @@ func buildQueriesForRipoff(primaryKeys PrimaryKeysResult, totalRipoff RipoffFile
 	}
 
 	// Sort and reverse the graph, so queries are in order of least (hopefully none) to most dependencies.
-	ordered, _ := graph.TopologicalSort(dependencyGraph)
+	ordered, err := topologicalSort(dependencyGraph)
+	if err != nil {
+		return []string{}, err
+	}
 	sortedQueries := []string{}
 	for i := len(ordered) - 1; i >= 0; i-- {
 		query, ok := queries[ordered[i]]
@@ -392,9 +385,38 @@ func getForeignKeysResult(ctx context.Context, conn pgx.Tx) (ForeignKeysResult,
 	return result, nil
 }
 
-func isRealGraphError(err error) bool {
-	if err == nil || errors.Is(err, graph.ErrEdgeAlreadyExists) {
-		return false
+// Copy of github.com/amwolff/gorder DFS topological sort implementation,
+// with the only change being allowing non-acyclic graphs (for better or worse).
+func topologicalSort(digraph map[string][]string) ([]string, error) {
+	var (
+		acyclic       = true
+		order         []string
+		permanentMark = make(map[string]bool)
+		temporaryMark = make(map[string]bool)
+		visit         func(string)
+	)
+
+	visit = func(u string) {
+		if temporaryMark[u] {
+			acyclic = false
+		} else if !(temporaryMark[u] || permanentMark[u]) {
+			temporaryMark[u] = true
+			for _, v := range digraph[u] {
+				visit(v)
+				if !acyclic {
+					slog.Debug("Ripoff file appears to have cycle", slog.String("rowId", u))
+				}
+			}
+			delete(temporaryMark, u)
+			permanentMark[u] = true
+			order = append([]string{u}, order...)
+		}
+	}
+
+	for u := range digraph {
+		if !permanentMark[u] {
+			visit(u)
+		}
 	}
-	return true
+	return order, nil
 }

export.go

@@ -101,12 +101,20 @@ func ExportToRipoff(ctx context.Context, tx pgx.Tx) (RipoffFile, error) {
 				if slices.Contains(primaryKeys, field.Name) {
 					ids = append(ids, columnVal)
 				}
+				foreignKey, isFkey := singleColumnFkeyMap[[2]string{table, field.Name}]
 				// No need to export primary keys due to inference from schema on import.
 				if len(primaryKeys) == 1 && primaryKeys[0] == field.Name {
+					// The primary key is a foreign key, we'll need explicit dependencies.
+					if isFkey && columnVal != "" {
+						dependencies, ok := ripoffRow["~dependencies"].([]string)
+						if !ok {
+							ripoffRow["~dependencies"] = []string{}
+						}
+						ripoffRow["~dependencies"] = append(dependencies, fmt.Sprintf("%s:literal(%s)", foreignKey.ToTable, columnVal))
+					}
 					continue
 				}
 				// If this is a foreign key, should ensure it uses the table:valueFunc() format.
-				foreignKey, isFkey := singleColumnFkeyMap[[2]string{table, field.Name}]
 				if isFkey && columnVal != "" {
 					// Does the referenced table have more than one primary key, or does the constraint not point to a primary key?
 					// Then is a foreign key to a non-primary key, we need to fill this info in later.

go.mod

@@ -6,7 +6,6 @@ toolchain go1.22.4
 
 require (
 	github.com/brianvoe/gofakeit/v7 v7.0.4
-	github.com/dominikbraun/graph v0.23.0
 	github.com/google/uuid v1.6.0
 	github.com/jackc/pgx/v5 v5.6.0
 	github.com/lib/pq v1.10.9

go.sum

@@ -4,8 +4,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucVPgCo=
-github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=

testdata/import/dependencies/dependencies.yml

@@ -6,3 +6,4 @@ rows:
     url: image.png
   avatar_modifiers:uuid(fooBarAvatar):
     grayscale: true
+    ~dependencies: [avatars:uuid(fooBarAvatar)]

testdata/import/templates/template_user.yml

@@ -6,3 +6,4 @@ rows:
     url: {{ .avatarUrl }}
   avatar_modifiers:uuid({{ .rowId }}):
     grayscale: {{ .avatarGrayscale }}
+    ~dependencies: [avatars:uuid({{ .rowId }})]