rtree: add support for multiple URLs per remote

This requires changing the index format, which I'm using as an
opportunity to:

- move from YAML to JSON (so I can maybe kill the YAML parser dependency
  at some point)
- move the index file into standard XDG paths

9 files changed, 169 insertions, 94 deletions

diff --git a/README.md b/README.md
index cba0e8d..ca363c9 100644
--- a/README.md
+++ b/README.md
@@ -91,7 +91,7 @@ There are a few other subcommands in `rtree`:
 * `rtree each <COMMAND>` executes the given command in each repository. My most common usecase is `rtree each git status --short`.
 * `rtree import <PATH>` takes a path to a local Git repo, and moves it to the correct place below `$GOPATH/src`.
 
-Finally, `rtree index` rebuilds the index file (`~/.rtree/index.yaml`) that all of these operations use to find repos
+Finally, `rtree index` rebuilds the index file (`~/.config/rtree/index.json`) that all of these operations use to find repos
 and remotes. If a repo is checked out, but not yet indexed, the index entry will be added. If the repo for an index
 entry is missing, the user will be prompted about what to do:
 
@@ -104,4 +104,4 @@ repository /x/src/github.com/Masterminds/sprig has been deleted
 ```
 
 One of the intended usecases is that stuff below `$GOPATH/src` does not need to be backed up. As long as the index file
-`~/.rtree/index.yaml` is backed up, all repos can be restored in one step with `yes r | rtree index`.
+`~/.config/rtree/index.json` is backed up, all repos can be restored in one step with `yes r | rtree index`.
diff --git a/internal/cli/command.go b/internal/cli/command.go
index 8208aac..e2ad6f1 100644
--- a/internal/cli/command.go
+++ b/internal/cli/command.go
@@ -10,8 +10,8 @@ import (
 	"strings"
 )
 
-//Command describes a command that can be run using the methods in the
-//Implementation interface.
+// Command describes a command that can be run using the methods in the
+// Implementation interface.
 type Command struct {
 	Program []string
 	WorkDir string
@@ -34,12 +34,12 @@ func (e commandError) Error() string {
 	)
 }
 
-//CommandRunner is a function that can execute commands given to it.
-//This interface is only useful for unit tests; the default CommandRunner
-//suffices for all regular operation.
+// CommandRunner is a function that can execute commands given to it.
+// This interface is only useful for unit tests; the default CommandRunner
+// suffices for all regular operation.
 type CommandRunner func(c Command, stdin io.Reader, stdout, stderr io.Writer) error
 
-//DefaultCommandRunner is a CommandRunner that actually executes the command.
+// DefaultCommandRunner is a CommandRunner that actually executes the command.
 func DefaultCommandRunner(c Command, stdin io.Reader, stdout, stderr io.Writer) error {
 	cmd := exec.Command(c.Program[0], c.Program[1:]...)
 	cmd.Stdin = stdin
diff --git a/internal/rtree/get_test.go b/internal/rtree/get_test.go
index 4d0d8f3..222ad99 100644
--- a/internal/rtree/get_test.go
+++ b/internal/rtree/get_test.go
@@ -13,14 +13,14 @@ var testIndexWithTwoRepos = Index{
 	Repos: []*Repo{
 		{
 			CheckoutPath: "github.com/foo/bar",
-			Remotes: []Remote{
-				{Name: "origin", URL: "https://github.com/foo/bar"},
+			Remotes: map[string]Remote{
+				"origin": {URLs: []RemoteURL{"https://github.com/foo/bar"}},
 			},
 		},
 		{
 			CheckoutPath: "github.com/git/git",
-			Remotes: []Remote{
-				{Name: "origin", URL: "https://github.com/git/git"},
+			Remotes: map[string]Remote{
+				"origin": {URLs: []RemoteURL{"https://github.com/git/git"}},
 			},
 		},
 	},
@@ -55,9 +55,9 @@ func TestGetNewRepo(t *testing.T) {
 				Repos: []*Repo{
 					{
 						CheckoutPath: "github.com/another/repo",
-						Remotes: []Remote{
+						Remotes: map[string]Remote{
 							//regardless of the remote URL used, we expect the contracted form to be used
-							{Name: "origin", URL: "https://github.com/another/repo"},
+							"origin": {URLs: []RemoteURL{"https://github.com/another/repo"}},
 						},
 					},
 					testIndexWithTwoRepos.Repos[0],
@@ -90,9 +90,9 @@ func TestGetNewForkAsRemote(t *testing.T) {
 				testIndexWithTwoRepos.Repos[0],
 				{
 					CheckoutPath: "github.com/git/git",
-					Remotes: []Remote{
-						{Name: "origin", URL: "https://github.com/git/git"},
-						{Name: "myfork", URL: "https://example.com/git"},
+					Remotes: map[string]Remote{
+						"origin": {URLs: []RemoteURL{"https://github.com/git/git"}},
+						"myfork": {URLs: []RemoteURL{"https://example.com/git"}},
 					},
 				},
 			},
@@ -116,8 +116,8 @@ func TestGetNewForkAsSeparate(t *testing.T) {
 			Repos: []*Repo{
 				{
 					CheckoutPath: "example.com/git",
-					Remotes: []Remote{
-						{Name: "origin", URL: "https://example.com/git"},
+					Remotes: map[string]Remote{
+						"origin": {URLs: []RemoteURL{"https://example.com/git"}},
 					},
 				},
 				testIndexWithTwoRepos.Repos[0],
diff --git a/internal/rtree/index.go b/internal/rtree/index.go
index f1af07d..1432161 100644
--- a/internal/rtree/index.go
+++ b/internal/rtree/index.go
@@ -4,6 +4,7 @@
 package rtree
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -13,13 +14,11 @@ import (
 	"strings"
 
 	"github.com/majewsky/gofu/internal/cli"
-
-	yaml "go.yaml.in/yaml/v3"
 )
 
-// Index represents the contents of the index file.
+// Index represents the contents of the index.json file.
 type Index struct {
-	Repos []*Repo `yaml:"repos"`
+	Repos []*Repo `json:"repos"`
 }
 
 // ReadIndex reads the index file.
@@ -28,19 +27,27 @@ func ReadIndex() (*Index, []error) {
 	buf, err := os.ReadFile(IndexPath)
 	if err != nil {
 		if os.IsNotExist(err) {
+			_, err := os.Stat(OldIndexPath)
+			if !os.IsNotExist(err) {
+				err = fmt.Errorf(
+					"old index format detected: upgrade to the new index format with this command:\n\t"+
+						`yq -o json < %s | jq --sort-keys '{ repos: .repos | map({ path, remotes: .remotes | map({ key: .name, value: { urls: [.url] }}) | from_entries }) }' > %s`,
+					OldIndexPath, IndexPath,
+				)
+				return nil, []error{err}
+			}
 			return &Index{Repos: nil}, nil
 		}
 		return nil, []error{err}
 	}
 
-	//deserialize YAML
+	//deserialize JSON
 	var index Index
-	err = yaml.Unmarshal(buf, &index)
+	err = json.Unmarshal(buf, &index)
 	if err != nil {
 		return nil, []error{err}
 	}
-
-	//validate YAML
+	//validate JSON
 	var errs []error
 	missing := func(key string, args ...any) {
 		errs = append(errs, fmt.Errorf("read %s: missing \"%s\"",
@@ -54,12 +61,14 @@ func ReadIndex() (*Index, []error) {
 		if len(repo.Remotes) == 0 {
 			missing("repos[%d].remotes", idx)
 		}
-		for idx2, remote := range repo.Remotes {
+		for remoteName, remote := range repo.Remotes {
 			switch {
-			case remote.Name == "":
-				missing("repos[%d].remotes[%d].name", idx, idx2)
-			case remote.URL == "":
-				missing("repos[%d].remotes[%d].url", idx, idx2)
+			case remoteName == "":
+				errs = append(errs, fmt.Errorf("read %s: empty key in \"repos[%d].remotes\"",
+					IndexPath, idx,
+				))
+			case len(remote.URLs) == 0:
+				missing("repos[%d].remotes[%q].urls", idx, remoteName)
 			}
 		}
 	}
@@ -77,7 +86,7 @@ func (r reposByAbsPath) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }
 // Write writes the index file to disk.
 func (i *Index) Write() error {
 	sort.Sort(reposByAbsPath(i.Repos))
-	buf, err := yaml.Marshal(i)
+	buf, err := json.MarshalIndent(i, "", "  ")
 	if err != nil {
 		return err
 	}
@@ -129,12 +138,12 @@ func (i *Index) Rebuild() error {
 
 		//repo has been deleted - ask what to do
 		var remoteURLs []string
-		for _, remote := range repo.Remotes {
-			if remote.Name == "origin" {
-				remoteURLs = []string{remote.URL.CompactURL()}
-				break
+		if origin, ok := repo.Remotes["origin"]; ok {
+			remoteURLs = origin.CompactURLs()
+		} else {
+			for _, remote := range repo.Remotes {
+				remoteURLs = append(remoteURLs, remote.CompactURLs()...)
 			}
-			remoteURLs = append(remoteURLs, remote.URL.CompactURL())
 		}
 
 		repoPath := filepath.Join(RootPath, repo.CheckoutPath)
@@ -218,12 +227,14 @@ func (i *Index) FindRepo(rawRemoteURL string, allowClone bool) (*Repo, error) {
 	for _, repo := range i.Repos {
 		isCandidate := false
 		for _, remote := range repo.Remotes {
-			// be flexible about .git ending in remote
-			if remoteURL == remote.URL || remoteURL+".git" == remote.URL || remoteURL == remote.URL+".git" {
-				return repo, nil
-			}
-			if basename == path.Base(remote.URL.CanonicalURL()) {
-				isCandidate = true
+			for _, url := range remote.URLs {
+				// be flexible about .git ending in remote
+				if remoteURL == url || remoteURL+".git" == url || remoteURL == url+".git" {
+					return repo, nil
+				}
+				if basename == path.Base(url.CanonicalURL()) {
+					isCandidate = true
+				}
 			}
 		}
 		if isCandidate {
@@ -302,8 +313,8 @@ func (i *Index) FindRepo(rawRemoteURL string, allowClone bool) (*Repo, error) {
 
 	//report the existing remotes, and ask for the name of the new remote
 	prompt := "Existing remotes:\n"
-	for _, remote := range target.Remotes {
-		prompt += fmt.Sprintf("\t(%s) %s\n", remote.Name, remote.URL.CompactURL())
+	for remoteName, remote := range target.Remotes {
+		prompt += fmt.Sprintf("\t(%s) %s\n", remoteName, strings.Join(remote.CompactURLs(), " "))
 	}
 	prompt += fmt.Sprintf("Enter remote name for %s:", remoteURL)
 	remoteName, err := cli.Interface.ReadLine(prompt)
@@ -327,10 +338,9 @@ func (i *Index) FindRepo(rawRemoteURL string, allowClone bool) (*Repo, error) {
 		return nil, err
 	}
 
-	target.Remotes = append(target.Remotes, Remote{
-		Name: remoteName,
-		URL:  remoteURL,
-	})
+	target.Remotes[remoteName] = Remote{
+		URLs: []RemoteURL{remoteURL},
+	}
 	err = i.Write()
 	return target, err
 }
@@ -353,19 +363,20 @@ func (i *Index) ImportRepo(dirPath string) error {
 	}
 
 	//select the remote which determines the checkout path
-	choices := make([]cli.Choice, len(repo.Remotes))
+	choices := make([]cli.Choice, 0, len(repo.Remotes))
 	var checkoutPath string
-	for idx, remote := range repo.Remotes {
-		thisPath, err := remote.URL.CheckoutPath()
+	for remoteName, remote := range repo.Remotes {
+		// NOTE: This uses URLs[0] only because git fetches only from the first URL (the others are only for pushing).
+		thisPath, err := remote.URLs[0].CheckoutPath()
 		if err != nil {
 			return err
 		}
-		if remote.Name == "origin" {
+		if remoteName == "origin" {
 			//prefer "origin" over everything else
 			checkoutPath = thisPath
 			break
 		}
-		choices[idx] = cli.Choice{Return: thisPath, Text: thisPath}
+		choices = append(choices, cli.Choice{Return: thisPath, Text: thisPath})
 	}
 
 	//cannot decide myself -> let the user select
diff --git a/internal/rtree/init.go b/internal/rtree/init.go
index 3a8e08f..d2118c0 100644
--- a/internal/rtree/init.go
+++ b/internal/rtree/init.go
@@ -22,6 +22,11 @@ type RemoteAlias struct {
 // IndexPath is where the index file is stored.
 var IndexPath string
 
+// OldIndexPath is where the old index file was stored.
+// This is only used to detect if a system has not been upgraded to the new index format yet.
+// TODO: remove this after some time
+var OldIndexPath string
+
 // RootPath is the directory below which all repositories are located. Its value
 // is $GOPATH/src to match the repository layout created by `go get`.
 var RootPath string
@@ -44,7 +49,8 @@ func Init() bool {
 			cli.Interface.ShowError("$HOME is not set (rtree needs the HOME variable to locate its index file)")
 			ok = false //but keep going to report all errors at once
 		} else {
-			IndexPath = filepath.Join(homeDir, ".rtree/index.yaml")
+			IndexPath = filepath.Join(homeDir, ".config/rtree/index.json")
+			OldIndexPath = filepath.Join(homeDir, ".rtree/index.yaml")
 		}
 	}
 
diff --git a/internal/rtree/main.go b/internal/rtree/main.go
index 842a67b..68faa36 100644
--- a/internal/rtree/main.go
+++ b/internal/rtree/main.go
@@ -139,7 +139,9 @@ func commandRemotes(index *Index) {
 	var items []string
 	for _, repo := range index.Repos {
 		for _, remote := range repo.Remotes {
-			items = append(items, remote.URL.CompactURL())
+			for _, url := range remote.URLs {
+				items = append(items, url.CompactURL())
+			}
 		}
 	}
 	cli.Interface.ShowResultsSorted(items)
diff --git a/internal/rtree/remote.go b/internal/rtree/remote.go
index afcdf9b..8c355a8 100644
--- a/internal/rtree/remote.go
+++ b/internal/rtree/remote.go
@@ -4,6 +4,7 @@
 package rtree
 
 import (
+	"encoding/json"
 	"net/url"
 	"path/filepath"
 	"regexp"
@@ -86,16 +87,16 @@ func (u RemoteURL) CheckoutPath() (string, error) {
 	return filepath.Join(parsed.Hostname(), parsed.Path), nil
 }
 
-// MarshalYAML implements the yaml.Marshaler interface.
-func (u RemoteURL) MarshalYAML() (any, error) {
+// MarshalJSON implements the json.Marshaler interface.
+func (u RemoteURL) MarshalJSON() ([]byte, error) {
 	//store URLs in the index in the canonical format
-	return u.CanonicalURL(), nil
+	return json.Marshal(u.CanonicalURL())
 }
 
-// UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (u *RemoteURL) UnmarshalYAML(unmarshal func(any) error) error {
+// UnmarshalJSON implements the json.Unmarshaler interface.
+func (u *RemoteURL) UnmarshalJSON(buf []byte) error {
 	var s string
-	err := unmarshal(&s)
+	err := json.Unmarshal(buf, &s)
 	if err == nil {
 		*u = ParseRemoteURL(s)
 	}
diff --git a/internal/rtree/repo.go b/internal/rtree/repo.go
index c8e8ff3..92b7e74 100644
--- a/internal/rtree/repo.go
+++ b/internal/rtree/repo.go
@@ -16,17 +16,16 @@ import (
 // Repo describes the entry for a repository in the index file.
 type Repo struct {
 	//CheckoutPath shall be relative to the RootPath.
-	CheckoutPath string `yaml:"path"`
+	CheckoutPath string `json:"path"`
 	//Remotes maps remote names (as noted in the .git/config of the repo) to
 	//remote URLs (as they appear in the .git/config of the repo, i.e. possibly
 	//abbreviated).
-	Remotes []Remote `yaml:"remotes"`
+	Remotes map[string]Remote `json:"remotes"`
 }
 
 // Remote describes a remote that is configured in a Repo.
 type Remote struct {
-	Name string    `yaml:"name"`
-	URL  RemoteURL `yaml:"url"`
+	URLs []RemoteURL `json:"urls"`
 }
 
 // AbsolutePath returns the absolute CheckoutPath of this repo.
@@ -39,6 +38,15 @@ func (r Repo) GitDirPath() string {
 	return filepath.Join(r.AbsolutePath(), ".git")
 }
 
+// CompactURLs returns all URLs for this remote in their compact form.
+func (r Remote) CompactURLs() []string {
+	result := make([]string, len(r.URLs))
+	for idx, url := range r.URLs {
+		result[idx] = url.CompactURL()
+	}
+	return result
+}
+
 // NewRepoFromAbsolutePath initializes a Repo instance by scanning the existing
 // checkout at the given path.
 func NewRepoFromAbsolutePath(path string) (repo Repo, err error) {
@@ -56,15 +64,19 @@ func NewRepoFromAbsolutePath(path string) (repo Repo, err error) {
 		return
 	}
 
+	repo.Remotes = make(map[string]Remote)
 	for line := range strings.SplitSeq(out, "\n") {
 		match := remoteConfigRx.FindStringSubmatch(line)
 		if match == nil {
 			continue
 		}
-		repo.Remotes = append(repo.Remotes, Remote{
-			Name: match[1],
-			URL:  ParseRemoteURL(match[2]),
-		})
+		name, url := match[1], ParseRemoteURL(match[2])
+		if remote, ok := repo.Remotes[name]; ok {
+			remote.URLs = append(remote.URLs, url)
+			repo.Remotes[name] = remote
+		} else {
+			repo.Remotes[name] = Remote{URLs: []RemoteURL{url}}
+		}
 	}
 	return
 }
@@ -75,10 +87,9 @@ func NewRepoFromRemoteURL(remoteURL RemoteURL) (Repo, error) {
 	checkoutPath, err := remoteURL.CheckoutPath()
 	return Repo{
 		CheckoutPath: checkoutPath,
-		Remotes: []Remote{
-			{
-				Name: "origin",
-				URL:  remoteURL,
+		Remotes: map[string]Remote{
+			"origin": {
+				URLs: []RemoteURL{remoteURL},
 			},
 		},
 	}, err
@@ -121,9 +132,9 @@ func ForeachPhysicalRepo(action func(repo Repo) error) error {
 func (r Repo) Checkout() error {
 	//check if we have an "origin" remote to clone from
 	var originURL RemoteURL
-	for _, remote := range r.Remotes {
-		if remote.Name == "origin" {
-			originURL = remote.URL
+	for remoteName, remote := range r.Remotes {
+		if remoteName == "origin" {
+			originURL = remote.URLs[0]
 			break
 		}
 	}
@@ -146,16 +157,27 @@ func (r Repo) Checkout() error {
 	}
 
 	remotesAdded := false
-	for _, remote := range r.Remotes {
-		if remote.Name != "origin" {
-			err := cli.Interface.Run(cli.Command{
-				Program: []string{"git", "remote", "add", remote.Name, remote.URL.CanonicalURL()},
-				WorkDir: r.AbsolutePath(),
-			})
-			if err != nil {
-				return err
+	for remoteName, remote := range r.Remotes {
+		for idx, url := range remote.URLs {
+			if idx == 0 && remoteName != "origin" {
+				err := cli.Interface.Run(cli.Command{
+					Program: []string{"git", "remote", "add", remoteName, url.CanonicalURL()},
+					WorkDir: r.AbsolutePath(),
+				})
+				if err != nil {
+					return err
+				}
+				remotesAdded = true
+			} else if idx > 0 {
+				err := cli.Interface.Run(cli.Command{
+					Program: []string{"git", "remote", "set-url", "--add", remoteName, url.CanonicalURL()},
+					WorkDir: r.AbsolutePath(),
+				})
+				if err != nil {
+					return err
+				}
+				remotesAdded = true
 			}
-			remotesAdded = true
 		}
 	}
 	if remotesAdded {
@@ -215,16 +237,50 @@ func (r *Repo) Move(checkoutPath string, makeSymlink bool) error {
 }
 
 // ReformatRemoteURLs rewrites the remote URLs in this repo's .git/config into
-// their compact forms.
+// their canonical forms.
 func (r Repo) ReformatRemoteURLs() error {
-	for _, remote := range r.Remotes {
+	// NOTE: This is a bit convoluted because the specific case of updating URLs
+	// for a remote with multiple URLs requires multiple steps. First, we clear
+	// out all non-primary URLs, and then re-add them after updating the primary URL.
+	actualRepo, err := NewRepoFromAbsolutePath(r.AbsolutePath())
+	if err != nil {
+		return err
+	}
+
+	for remoteName, remote := range r.Remotes {
+		actualRemote := actualRepo.Remotes[remoteName]
+
+		if len(actualRemote.URLs) > 1 {
+			for _, url := range actualRemote.URLs[1:] {
+				err := cli.Interface.Run(cli.Command{
+					Program: []string{"git", "remote", "set-url", "--delete", remoteName, url.CanonicalURL()},
+					WorkDir: r.AbsolutePath(),
+				})
+				if err != nil {
+					return err
+				}
+			}
+		}
+
 		err := cli.Interface.Run(cli.Command{
-			Program: []string{"git", "remote", "set-url", remote.Name, remote.URL.CanonicalURL()},
+			Program: []string{"git", "remote", "set-url", remoteName, remote.URLs[0].CanonicalURL()},
 			WorkDir: r.AbsolutePath(),
 		})
 		if err != nil {
 			return err
 		}
+
+		if len(remote.URLs) > 1 {
+			for _, url := range remote.URLs[1:] {
+				err := cli.Interface.Run(cli.Command{
+					Program: []string{"git", "remote", "set-url", "--add", remoteName, url.CanonicalURL()},
+					WorkDir: r.AbsolutePath(),
+				})
+				if err != nil {
+					return err
+				}
+			}
+		}
 	}
 	return nil
 }
diff --git a/internal/rtree/shared_test.go b/internal/rtree/shared_test.go
index c435110..4810695 100644
--- a/internal/rtree/shared_test.go
+++ b/internal/rtree/shared_test.go
@@ -5,6 +5,7 @@ package rtree
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -14,8 +15,6 @@ import (
 	"strings"
 	"testing"
 
-	yaml "go.yaml.in/yaml/v3"
-
 	"github.com/majewsky/gofu/internal/cli"
 )
 
@@ -56,7 +55,7 @@ type Test struct {
 
 func (test Test) Run(t *testing.T) {
 	//write index file, if any
-	IndexPath = filepath.Join(indexTmpDir, t.Name()+".yaml")
+	IndexPath = filepath.Join(indexTmpDir, t.Name()+".json")
 	if test.Index.Repos != nil {
 		err := test.Index.Write()
 		if err != nil {
@@ -94,7 +93,7 @@ func (test Test) Run(t *testing.T) {
 	if test.ExpectIndex != nil {
 		idx = test.ExpectIndex
 	}
-	expectedIdxStr, err := yaml.Marshal(idx)
+	expectedIdxStr, err := json.MarshalIndent(idx, "", "  ")
 	if err != nil {
 		t.Fatal(err.Error())
 	}