rtree: add support for multiple URLs per remote

This requires changing the index format, which I'm using as an
opportunity to:

- move from YAML to JSON (so I can maybe kill the YAML parser dependency
  at some point)
- move the index file into standard XDG paths

8 files changed, 167 insertions, 92 deletions

diff --git a/internal/cli/command.go b/internal/cli/command.go
index 8208aac..e2ad6f1 100644
--- a/internal/cli/command.go
+++ b/internal/cli/command.go
@@ -10,8 +10,8 @@ import (
 	"strings"
 )
 
-//Command describes a command that can be run using the methods in the
-//Implementation interface.
+// Command describes a command that can be run using the methods in the
+// Implementation interface.
 type Command struct {
 	Program []string
 	WorkDir string
@@ -34,12 +34,12 @@ func (e commandError) Error() string {
 	)
 }
 
-//CommandRunner is a function that can execute commands given to it.
-//This interface is only useful for unit tests; the default CommandRunner
-//suffices for all regular operation.
+// CommandRunner is a function that can execute commands given to it.
+// This interface is only useful for unit tests; the default CommandRunner
+// suffices for all regular operation.
 type CommandRunner func(c Command, stdin io.Reader, stdout, stderr io.Writer) error
 
-//DefaultCommandRunner is a CommandRunner that actually executes the command.
+// DefaultCommandRunner is a CommandRunner that actually executes the command.
 func DefaultCommandRunner(c Command, stdin io.Reader, stdout, stderr io.Writer) error {
 	cmd := exec.Command(c.Program[0], c.Program[1:]...)
 	cmd.Stdin = stdin
diff --git a/internal/rtree/get_test.go b/internal/rtree/get_test.go
index 4d0d8f3..222ad99 100644
--- a/internal/rtree/get_test.go
+++ b/internal/rtree/get_test.go
@@ -13,14 +13,14 @@ var testIndexWithTwoRepos = Index{
 	Repos: []*Repo{
 		{
 			CheckoutPath: "github.com/foo/bar",
-			Remotes: []Remote{
-				{Name: "origin", URL: "https://github.com/foo/bar"},
+			Remotes: map[string]Remote{
+				"origin": {URLs: []RemoteURL{"https://github.com/foo/bar"}},
 			},
 		},
 		{
 			CheckoutPath: "github.com/git/git",
-			Remotes: []Remote{
-				{Name: "origin", URL: "https://github.com/git/git"},
+			Remotes: map[string]Remote{
+				"origin": {URLs: []RemoteURL{"https://github.com/git/git"}},
 			},
 		},
 	},
@@ -55,9 +55,9 @@ func TestGetNewRepo(t *testing.T) {
 				Repos: []*Repo{
 					{
 						CheckoutPath: "github.com/another/repo",
-						Remotes: []Remote{
+						Remotes: map[string]Remote{
 							//regardless of the remote URL used, we expect the contracted form to be used
-							{Name: "origin", URL: "https://github.com/another/repo"},
+							"origin": {URLs: []RemoteURL{"https://github.com/another/repo"}},
 						},
 					},
 					testIndexWithTwoRepos.Repos[0],
@@ -90,9 +90,9 @@ func TestGetNewForkAsRemote(t *testing.T) {
 				testIndexWithTwoRepos.Repos[0],
 				{
 					CheckoutPath: "github.com/git/git",
-					Remotes: []Remote{
-						{Name: "origin", URL: "https://github.com/git/git"},
-						{Name: "myfork", URL: "https://example.com/git"},
+					Remotes: map[string]Remote{
+						"origin": {URLs: []RemoteURL{"https://github.com/git/git"}},
+						"myfork": {URLs: []RemoteURL{"https://example.com/git"}},
 					},
 				},
 			},
@@ -116,8 +116,8 @@ func TestGetNewForkAsSeparate(t *testing.T) {
 			Repos: []*Repo{
 				{
 					CheckoutPath: "example.com/git",
-					Remotes: []Remote{
-						{Name: "origin", URL: "https://example.com/git"},
+					Remotes: map[string]Remote{
+						"origin": {URLs: []RemoteURL{"https://example.com/git"}},
 					},
 				},
 				testIndexWithTwoRepos.Repos[0],
diff --git a/internal/rtree/index.go b/internal/rtree/index.go
index f1af07d..1432161 100644
--- a/internal/rtree/index.go
+++ b/internal/rtree/index.go
@@ -4,6 +4,7 @@
 package rtree
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -13,13 +14,11 @@ import (
 	"strings"
 
 	"github.com/majewsky/gofu/internal/cli"
-
-	yaml "go.yaml.in/yaml/v3"
 )
 
-// Index represents the contents of the index file.
+// Index represents the contents of the index.json file.
 type Index struct {
-	Repos []*Repo `yaml:"repos"`
+	Repos []*Repo `json:"repos"`
 }
 
 // ReadIndex reads the index file.
@@ -28,19 +27,27 @@ func ReadIndex() (*Index, []error) {
 	buf, err := os.ReadFile(IndexPath)
 	if err != nil {
 		if os.IsNotExist(err) {
+			_, err := os.Stat(OldIndexPath)
+			if !os.IsNotExist(err) {
+				err = fmt.Errorf(
+					"old index format detected: upgrade to the new index format with this command:\n\t"+
+						`yq -o json < %s | jq --sort-keys '{ repos: .repos | map({ path, remotes: .remotes | map({ key: .name, value: { urls: [.url] }}) | from_entries }) }' > %s`,
+					OldIndexPath, IndexPath,
+				)
+				return nil, []error{err}
+			}
 			return &Index{Repos: nil}, nil
 		}
 		return nil, []error{err}
 	}
 
-	//deserialize YAML
+	//deserialize JSON
 	var index Index
-	err = yaml.Unmarshal(buf, &index)
+	err = json.Unmarshal(buf, &index)
 	if err != nil {
 		return nil, []error{err}
 	}
-
-	//validate YAML
+	//validate JSON
 	var errs []error
 	missing := func(key string, args ...any) {
 		errs = append(errs, fmt.Errorf("read %s: missing \"%s\"",
@@ -54,12 +61,14 @@ func ReadIndex() (*Index, []error) {
 		if len(repo.Remotes) == 0 {
 			missing("repos[%d].remotes", idx)
 		}
-		for idx2, remote := range repo.Remotes {
+		for remoteName, remote := range repo.Remotes {
 			switch {
-			case remote.Name == "":
-				missing("repos[%d].remotes[%d].name", idx, idx2)
-			case remote.URL == "":
-				missing("repos[%d].remotes[%d].url", idx, idx2)
+			case remoteName == "":
+				errs = append(errs, fmt.Errorf("read %s: empty key in \"repos[%d].remotes\"",
+					IndexPath, idx,
+				))
+			case len(remote.URLs) == 0:
+				missing("repos[%d].remotes[%q].urls", idx, remoteName)
 			}
 		}
 	}
@@ -77,7 +86,7 @@ func (r reposByAbsPath) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }
 // Write writes the index file to disk.
 func (i *Index) Write() error {
 	sort.Sort(reposByAbsPath(i.Repos))
-	buf, err := yaml.Marshal(i)
+	buf, err := json.MarshalIndent(i, "", "  ")
 	if err != nil {
 		return err
 	}
@@ -129,12 +138,12 @@ func (i *Index) Rebuild() error {
 
 		//repo has been deleted - ask what to do
 		var remoteURLs []string
-		for _, remote := range repo.Remotes {
-			if remote.Name == "origin" {
-				remoteURLs = []string{remote.URL.CompactURL()}
-				break
+		if origin, ok := repo.Remotes["origin"]; ok {
+			remoteURLs = origin.CompactURLs()
+		} else {
+			for _, remote := range repo.Remotes {
+				remoteURLs = append(remoteURLs, remote.CompactURLs()...)
 			}
-			remoteURLs = append(remoteURLs, remote.URL.CompactURL())
 		}
 
 		repoPath := filepath.Join(RootPath, repo.CheckoutPath)
@@ -218,12 +227,14 @@ func (i *Index) FindRepo(rawRemoteURL string, allowClone bool) (*Repo, error) {
 	for _, repo := range i.Repos {
 		isCandidate := false
 		for _, remote := range repo.Remotes {
-			// be flexible about .git ending in remote
-			if remoteURL == remote.URL || remoteURL+".git" == remote.URL || remoteURL == remote.URL+".git" {
-				return repo, nil
-			}
-			if basename == path.Base(remote.URL.CanonicalURL()) {
-				isCandidate = true
+			for _, url := range remote.URLs {
+				// be flexible about .git ending in remote
+				if remoteURL == url || remoteURL+".git" == url || remoteURL == url+".git" {
+					return repo, nil
+				}
+				if basename == path.Base(url.CanonicalURL()) {
+					isCandidate = true
+				}
 			}
 		}
 		if isCandidate {
@@ -302,8 +313,8 @@ func (i *Index) FindRepo(rawRemoteURL string, allowClone bool) (*Repo, error) {
 
 	//report the existing remotes, and ask for the name of the new remote
 	prompt := "Existing remotes:\n"
-	for _, remote := range target.Remotes {
-		prompt += fmt.Sprintf("\t(%s) %s\n", remote.Name, remote.URL.CompactURL())
+	for remoteName, remote := range target.Remotes {
+		prompt += fmt.Sprintf("\t(%s) %s\n", remoteName, strings.Join(remote.CompactURLs(), " "))
 	}
 	prompt += fmt.Sprintf("Enter remote name for %s:", remoteURL)
 	remoteName, err := cli.Interface.ReadLine(prompt)
@@ -327,10 +338,9 @@ func (i *Index) FindRepo(rawRemoteURL string, allowClone bool) (*Repo, error) {
 		return nil, err
 	}
 
-	target.Remotes = append(target.Remotes, Remote{
-		Name: remoteName,
-		URL:  remoteURL,
-	})
+	target.Remotes[remoteName] = Remote{
+		URLs: []RemoteURL{remoteURL},
+	}
 	err = i.Write()
 	return target, err
 }
@@ -353,19 +363,20 @@ func (i *Index) ImportRepo(dirPath string) error {
 	}
 
 	//select the remote which determines the checkout path
-	choices := make([]cli.Choice, len(repo.Remotes))
+	choices := make([]cli.Choice, 0, len(repo.Remotes))
 	var checkoutPath string
-	for idx, remote := range repo.Remotes {
-		thisPath, err := remote.URL.CheckoutPath()
+	for remoteName, remote := range repo.Remotes {
+		// NOTE: This uses URLs[0] only because git fetches only from the first URL (the others are only for pushing).
+		thisPath, err := remote.URLs[0].CheckoutPath()
 		if err != nil {
 			return err
 		}
-		if remote.Name == "origin" {
+		if remoteName == "origin" {
 			//prefer "origin" over everything else
 			checkoutPath = thisPath
 			break
 		}
-		choices[idx] = cli.Choice{Return: thisPath, Text: thisPath}
+		choices = append(choices, cli.Choice{Return: thisPath, Text: thisPath})
 	}
 
 	//cannot decide myself -> let the user select
diff --git a/internal/rtree/init.go b/internal/rtree/init.go
index 3a8e08f..d2118c0 100644
--- a/internal/rtree/init.go
+++ b/internal/rtree/init.go
@@ -22,6 +22,11 @@ type RemoteAlias struct {
 // IndexPath is where the index file is stored.
 var IndexPath string
 
+// OldIndexPath is where the old index file was stored.
+// This is only used to detect if a system has not been upgraded to the new index format yet.
+// TODO: remove this after some time
+var OldIndexPath string
+
 // RootPath is the directory below which all repositories are located. Its value
 // is $GOPATH/src to match the repository layout created by `go get`.
 var RootPath string
@@ -44,7 +49,8 @@ func Init() bool {
 			cli.Interface.ShowError("$HOME is not set (rtree needs the HOME variable to locate its index file)")
 			ok = false //but keep going to report all errors at once
 		} else {
-			IndexPath = filepath.Join(homeDir, ".rtree/index.yaml")
+			IndexPath = filepath.Join(homeDir, ".config/rtree/index.json")
+			OldIndexPath = filepath.Join(homeDir, ".rtree/index.yaml")
 		}
 	}
 
diff --git a/internal/rtree/main.go b/internal/rtree/main.go
index 842a67b..68faa36 100644
--- a/internal/rtree/main.go
+++ b/internal/rtree/main.go
@@ -139,7 +139,9 @@ func commandRemotes(index *Index) {
 	var items []string
 	for _, repo := range index.Repos {
 		for _, remote := range repo.Remotes {
-			items = append(items, remote.URL.CompactURL())
+			for _, url := range remote.URLs {
+				items = append(items, url.CompactURL())
+			}
 		}
 	}
 	cli.Interface.ShowResultsSorted(items)
diff --git a/internal/rtree/remote.go b/internal/rtree/remote.go
index afcdf9b..8c355a8 100644
--- a/internal/rtree/remote.go
+++ b/internal/rtree/remote.go
@@ -4,6 +4,7 @@
 package rtree
 
 import (
+	"encoding/json"
 	"net/url"
 	"path/filepath"
 	"regexp"
@@ -86,16 +87,16 @@ func (u RemoteURL) CheckoutPath() (string, error) {
 	return filepath.Join(parsed.Hostname(), parsed.Path), nil
 }
 
-// MarshalYAML implements the yaml.Marshaler interface.
-func (u RemoteURL) MarshalYAML() (any, error) {
+// MarshalJSON implements the json.Marshaler interface.
+func (u RemoteURL) MarshalJSON() ([]byte, error) {
 	//store URLs in the index in the canonical format
-	return u.CanonicalURL(), nil
+	return json.Marshal(u.CanonicalURL())
 }
 
-// UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (u *RemoteURL) UnmarshalYAML(unmarshal func(any) error) error {
+// UnmarshalJSON implements the json.Unmarshaler interface.
+func (u *RemoteURL) UnmarshalJSON(buf []byte) error {
 	var s string
-	err := unmarshal(&s)
+	err := json.Unmarshal(buf, &s)
 	if err == nil {
 		*u = ParseRemoteURL(s)
 	}
diff --git a/internal/rtree/repo.go b/internal/rtree/repo.go
index c8e8ff3..92b7e74 100644
--- a/internal/rtree/repo.go
+++ b/internal/rtree/repo.go
@@ -16,17 +16,16 @@ import (
 // Repo describes the entry for a repository in the index file.
 type Repo struct {
 	//CheckoutPath shall be relative to the RootPath.
-	CheckoutPath string `yaml:"path"`
+	CheckoutPath string `json:"path"`
 	//Remotes maps remote names (as noted in the .git/config of the repo) to
 	//remote URLs (as they appear in the .git/config of the repo, i.e. possibly
 	//abbreviated).
-	Remotes []Remote `yaml:"remotes"`
+	Remotes map[string]Remote `json:"remotes"`
 }
 
 // Remote describes a remote that is configured in a Repo.
 type Remote struct {
-	Name string    `yaml:"name"`
-	URL  RemoteURL `yaml:"url"`
+	URLs []RemoteURL `json:"urls"`
 }
 
 // AbsolutePath returns the absolute CheckoutPath of this repo.
@@ -39,6 +38,15 @@ func (r Repo) GitDirPath() string {
 	return filepath.Join(r.AbsolutePath(), ".git")
 }
 
+// CompactURLs returns all URLs for this remote in their compact form.
+func (r Remote) CompactURLs() []string {
+	result := make([]string, len(r.URLs))
+	for idx, url := range r.URLs {
+		result[idx] = url.CompactURL()
+	}
+	return result
+}
+
 // NewRepoFromAbsolutePath initializes a Repo instance by scanning the existing
 // checkout at the given path.
 func NewRepoFromAbsolutePath(path string) (repo Repo, err error) {
@@ -56,15 +64,19 @@ func NewRepoFromAbsolutePath(path string) (repo Repo, err error) {
 		return
 	}
 
+	repo.Remotes = make(map[string]Remote)
 	for line := range strings.SplitSeq(out, "\n") {
 		match := remoteConfigRx.FindStringSubmatch(line)
 		if match == nil {
 			continue
 		}
-		repo.Remotes = append(repo.Remotes, Remote{
-			Name: match[1],
-			URL:  ParseRemoteURL(match[2]),
-		})
+		name, url := match[1], ParseRemoteURL(match[2])
+		if remote, ok := repo.Remotes[name]; ok {
+			remote.URLs = append(remote.URLs, url)
+			repo.Remotes[name] = remote
+		} else {
+			repo.Remotes[name] = Remote{URLs: []RemoteURL{url}}
+		}
 	}
 	return
 }
@@ -75,10 +87,9 @@ func NewRepoFromRemoteURL(remoteURL RemoteURL) (Repo, error) {
 	checkoutPath, err := remoteURL.CheckoutPath()
 	return Repo{
 		CheckoutPath: checkoutPath,
-		Remotes: []Remote{
-			{
-				Name: "origin",
-				URL:  remoteURL,
+		Remotes: map[string]Remote{
+			"origin": {
+				URLs: []RemoteURL{remoteURL},
 			},
 		},
 	}, err
@@ -121,9 +132,9 @@ func ForeachPhysicalRepo(action func(repo Repo) error) error {
 func (r Repo) Checkout() error {
 	//check if we have an "origin" remote to clone from
 	var originURL RemoteURL
-	for _, remote := range r.Remotes {
-		if remote.Name == "origin" {
-			originURL = remote.URL
+	for remoteName, remote := range r.Remotes {
+		if remoteName == "origin" {
+			originURL = remote.URLs[0]
 			break
 		}
 	}
@@ -146,16 +157,27 @@ func (r Repo) Checkout() error {
 	}
 
 	remotesAdded := false
-	for _, remote := range r.Remotes {
-		if remote.Name != "origin" {
-			err := cli.Interface.Run(cli.Command{
-				Program: []string{"git", "remote", "add", remote.Name, remote.URL.CanonicalURL()},
-				WorkDir: r.AbsolutePath(),
-			})
-			if err != nil {
-				return err
+	for remoteName, remote := range r.Remotes {
+		for idx, url := range remote.URLs {
+			if idx == 0 && remoteName != "origin" {
+				err := cli.Interface.Run(cli.Command{
+					Program: []string{"git", "remote", "add", remoteName, url.CanonicalURL()},
+					WorkDir: r.AbsolutePath(),
+				})
+				if err != nil {
+					return err
+				}
+				remotesAdded = true
+			} else if idx > 0 {
+				err := cli.Interface.Run(cli.Command{
+					Program: []string{"git", "remote", "set-url", "--add", remoteName, url.CanonicalURL()},
+					WorkDir: r.AbsolutePath(),
+				})
+				if err != nil {
+					return err
+				}
+				remotesAdded = true
 			}
-			remotesAdded = true
 		}
 	}
 	if remotesAdded {
@@ -215,16 +237,50 @@ func (r *Repo) Move(checkoutPath string, makeSymlink bool) error {
 }
 
 // ReformatRemoteURLs rewrites the remote URLs in this repo's .git/config into
-// their compact forms.
+// their canonical forms.
 func (r Repo) ReformatRemoteURLs() error {
-	for _, remote := range r.Remotes {
+	// NOTE: This is a bit convoluted because the specific case of updating URLs
+	// for a remote with multiple URLs requires multiple steps. First, we clear
+	// out all non-primary URLs, and then re-add them after updating the primary URL.
+	actualRepo, err := NewRepoFromAbsolutePath(r.AbsolutePath())
+	if err != nil {
+		return err
+	}
+
+	for remoteName, remote := range r.Remotes {
+		actualRemote := actualRepo.Remotes[remoteName]
+
+		if len(actualRemote.URLs) > 1 {
+			for _, url := range actualRemote.URLs[1:] {
+				err := cli.Interface.Run(cli.Command{
+					Program: []string{"git", "remote", "set-url", "--delete", remoteName, url.CanonicalURL()},
+					WorkDir: r.AbsolutePath(),
+				})
+				if err != nil {
+					return err
+				}
+			}
+		}
+
 		err := cli.Interface.Run(cli.Command{
-			Program: []string{"git", "remote", "set-url", remote.Name, remote.URL.CanonicalURL()},
+			Program: []string{"git", "remote", "set-url", remoteName, remote.URLs[0].CanonicalURL()},
 			WorkDir: r.AbsolutePath(),
 		})
 		if err != nil {
 			return err
 		}
+
+		if len(remote.URLs) > 1 {
+			for _, url := range remote.URLs[1:] {
+				err := cli.Interface.Run(cli.Command{
+					Program: []string{"git", "remote", "set-url", "--add", remoteName, url.CanonicalURL()},
+					WorkDir: r.AbsolutePath(),
+				})
+				if err != nil {
+					return err
+				}
+			}
+		}
 	}
 	return nil
 }
diff --git a/internal/rtree/shared_test.go b/internal/rtree/shared_test.go
index c435110..4810695 100644
--- a/internal/rtree/shared_test.go
+++ b/internal/rtree/shared_test.go
@@ -5,6 +5,7 @@ package rtree
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -14,8 +15,6 @@ import (
 	"strings"
 	"testing"
 
-	yaml "go.yaml.in/yaml/v3"
-
 	"github.com/majewsky/gofu/internal/cli"
 )
 
@@ -56,7 +55,7 @@ type Test struct {
 
 func (test Test) Run(t *testing.T) {
 	//write index file, if any
-	IndexPath = filepath.Join(indexTmpDir, t.Name()+".yaml")
+	IndexPath = filepath.Join(indexTmpDir, t.Name()+".json")
 	if test.Index.Repos != nil {
 		err := test.Index.Write()
 		if err != nil {
@@ -94,7 +93,7 @@ func (test Test) Run(t *testing.T) {
 	if test.ExpectIndex != nil {
 		idx = test.ExpectIndex
 	}
-	expectedIdxStr, err := yaml.Marshal(idx)
+	expectedIdxStr, err := json.MarshalIndent(idx, "", "  ")
 	if err != nil {
 		t.Fatal(err.Error())
 	}