6 $| = 1; # autoflush stdout after each print, to avoid output after die
8 my $initial_working_directory;
13 my $try_to_resume = 0;
14 my $ignore_failure = 0;
15 my $checked_out_flag = 0; # NOT the opposite of bare_flag (describes remote repo state)
17 my $bare_flag = ""; # NOT the opposite of checked_out_flag (describes local repo state)
21 # Figure out where to get the other repositories from.
25 if (defined($defaultrepo)) {
29 # Figure out where to get the other repositories from,
30 # based on where this GHC repo came from.
31 my $git_dir = $bare_flag ? "--git-dir=ghc.git" : "";
32 my $branch = `git $git_dir branch | grep "\* " | sed "s/^\* //"`; chomp $branch;
33 my $remote = `git $git_dir config branch.$branch.remote`; chomp $remote;
35 # remotes are not mandatory for branches (e.g. not recorded by default for bare repos)
38 $repo = `git $git_dir config remote.$remote.url`; chomp $repo;
44 if ($repo =~ /^...*:/) {
46 # Above regex says "at least two chars before the :", to avoid
47 # catching Win32 drives ("C:\").
50 # --checked-out is needed if you want to use a checked-out repo
52 if ($checked_out_flag) {
53 $checked_out_tree = 1;
55 $checked_out_tree = 0;
58 # Don't drop the last part of the path if specified with -r, as
59 # it expects repos of the form:
61 # http://darcs.haskell.org
65 # http://darcs.haskell.org/ghc
68 $repo_base =~ s#/[^/]+/?$##;
71 elsif ($repo =~ /^\/|\.\.\/|.:(\/|\\)/) {
72 # Local filesystem, either absolute (C:/ or /) or relative (../) path
74 if (-f "$repo/HEAD") {
75 # assume a local mirror:
76 $checked_out_tree = 0;
77 $repo_base =~ s#/[^/]+/?$##;
78 } elsif (-d "$repo/ghc.git") {
79 # assume a local mirror:
80 $checked_out_tree = 0;
82 # assume a checked-out tree:
83 $checked_out_tree = 1;
87 die "Couldn't work out repo";
90 return $repo_base, $checked_out_tree;
97 open IN, "< packages.conf"
98 or open IN, "< packages" # clashes with packages directory when using --bare
99 or die "Can't open packages file (or packages.conf)";
108 if (/^([^# ]+) +([^ ]+) +([^ ]+) +([^ ]+)$/) {
110 $line{"localpath"} = $1;
112 $line{"remotepath"} = $3;
114 push @packages, \%line;
116 elsif (! /^(#.*)?$/) {
117 die "Bad content on line $lineNum of packages file: $_";
123 my $filename = shift;
126 open (FH, $filename) or return "";
129 return join('', @lines);
140 print "warning: @_\n";
147 my $target_dir = "$target/$dir";
150 message "== running git-new-workdir . $target_dir @_";
152 message "== $dir: running git-new-workdir . $target_dir @_";
156 system ("git-new-workdir", ".", $target_dir, @_) == 0
158 or die "git-new-workdir failed: $?";
161 chdir($initial_working_directory);
170 message "== running $scm @_";
172 message "== $dir: running $scm @_";
176 system ($scm, @_) == 0
178 or die "$scm failed: $?";
181 chdir($initial_working_directory);
204 my ($repo_base, $checked_out_tree) = getrepo();
206 my $is_github_repo = $repo_base =~ m/(git@|git:\/\/|https:\/\/)github.com/;
212 if ($command =~ /^remote$/) {
213 while (@_ > 0 && $_[0] =~ /^-/) {
216 if (@_ < 1) { help(1); }
218 if ($subcommand ne 'add' &&
219 $subcommand ne 'rm' &&
220 $subcommand ne 'set-branches' &&
221 $subcommand ne 'set-url') {
224 while (@_ > 0 && $_[0] =~ /^-/) {
227 if (($subcommand eq 'add' || $subcommand eq 'rm') && @_ < 1) {
229 } elsif (@_ < 1) { # set-url
230 $branch_name = 'origin';
232 $branch_name = shift;
234 } elsif ($command eq 'new') {
236 $branch_name = 'origin';
238 $branch_name = shift;
244 # $doing is a good enough approximation to what we are doing that
245 # we can use it to check that --resume is resuming the right command
246 $doing = join(" ", ($command, @args));
248 if ($try_to_resume && -f "resume") {
250 open RESUME, "< resume"
251 or die "Can't open resume file";
252 $start_repo = <RESUME>;
257 if ($what eq $doing) {
262 for $line (@packages) {
263 $tag = $$line{"tag"};
264 $scm = $$line{"vcs"};
265 # Use the "remote" structure for bare git repositories
266 $localpath = ($bare_flag && $scm eq "git") ?
267 $$line{"remotepath"} : $$line{"localpath"};
268 $remotepath = ($checked_out_tree) ?
269 $$line{"localpath"} : $$line{"remotepath"};
272 if ($start_repo eq $localpath) {
280 open RESUME, "> resume.tmp";
281 print RESUME "$localpath\n";
282 print RESUME "$doing\n";
284 rename "resume.tmp", "resume";
286 # Check the SCM is OK as early as possible
287 die "Unknown SCM: $scm" if (($scm ne "darcs") and ($scm ne "git"));
289 # We can't create directories on GitHub, so we translate
290 # "packages/foo" into "package-foo".
291 if ($is_github_repo) {
292 $remotepath =~ s/\//-/;
295 # Construct the path for this package in the repo we pulled from
296 $path = "$repo_base/$remotepath";
298 if ($command eq "get") {
299 next if $remotepath eq "-"; # "git submodule init/update" will get this later
301 # Skip any repositories we have not included the tag for
302 if (not defined($tags{$tag})) {
305 if ($tags{$tag} == 0) {
310 warning("$localpath already present; omitting")
311 if $localpath ne ".";
313 scm ($localpath, $scm, "config", "core.ignorecase", "true");
318 # Note that we use "." as the path, as $localpath
320 if ($scm eq "darcs") {
321 # The first time round the loop, default the get-mode
322 if (not defined($get_mode)) {
323 warning("adding --partial, to override use --complete");
324 $get_mode = "--partial";
326 scm (".", $scm, "get", $get_mode, $path, $localpath, @args);
329 my @argsWithBare = @args;
330 push @argsWithBare, $bare_flag if $bare_flag;
331 scm (".", $scm, "clone", $path, $localpath, @argsWithBare);
332 scm ($localpath, $scm, "config", "core.ignorecase", "true");
337 my $darcs_repo_present = 1 if -d "$localpath/_darcs";
338 my $git_repo_present = 1 if -e "$localpath/.git" || ($bare_flag && -d "$localpath");
339 if ($darcs_repo_present) {
340 if ($git_repo_present) {
341 die "Found both _darcs and .git in $localpath";
344 } elsif ($git_repo_present) {
346 } elsif ($tag eq "") {
347 die "Required repo $localpath is missing";
349 message "== $localpath repo not present; skipping";
353 # Work out the arguments we should give to the SCM
354 if ($command eq "status") {
355 if ($scm eq "darcs") {
356 $command = "whatsnew";
358 elsif ($scm eq "git") {
365 # Hack around 'darcs whatsnew' failing if there are no changes
367 scm ($localpath, $scm, $command, @args);
369 elsif ($command eq "commit") {
370 # git fails if there is nothing to commit, so ignore failures
372 scm ($localpath, $scm, "commit", @args);
374 elsif ($command eq "push") {
375 scm ($localpath, $scm, "push", @args);
377 elsif ($command eq "pull") {
380 if ($remotepath eq "-") {
381 # Only fetch for the submodules. "git submodule update"
382 # will take care of making us point to the right commit.
384 # we like "sync-all pull --rebase" to work:
385 @realargs = grep(!/--rebase/,@args);
391 scm ($localpath, $scm, $realcmd, @realargs);
393 elsif ($command eq "new-workdir") {
394 gitNewWorkdir ($localpath, @args);
396 elsif ($command eq "send") {
397 if ($scm eq "darcs") {
400 elsif ($scm eq "git") {
401 $command = "send-email";
406 scm ($localpath, $scm, $command, @args);
408 elsif ($command eq "fetch") {
409 scm ($localpath, $scm, "fetch", @args);
411 elsif ($command eq "new") {
412 my @scm_args = ("log", "$branch_name..");
413 scm ($localpath, $scm, @scm_args, @args);
415 elsif ($command eq "log") {
416 scm ($localpath, $scm, "log", @args);
418 elsif ($command eq "remote") {
422 if ($remotepath eq '-') {
423 $rpath = "$repo_base/$localpath";
427 if ($subcommand eq 'add') {
428 @scm_args = ("remote", "add", $branch_name, $rpath);
429 } elsif ($subcommand eq 'rm') {
430 @scm_args = ("remote", "rm", $branch_name);
431 } elsif ($subcommand eq 'set-branches') {
432 @scm_args = ("remote", "set-branches", $branch_name);
433 } elsif ($subcommand eq 'set-url') {
434 @scm_args = ("remote", "set-url", $branch_name, $rpath);
436 scm ($localpath, $scm, @scm_args, @args);
438 elsif ($command eq "checkout") {
439 # Not all repos are necessarily branched, so ignore failure
441 scm ($localpath, $scm, "checkout", @args)
442 unless $scm eq "darcs";
444 elsif ($command eq "grep") {
445 # Hack around 'git grep' failing if there are no matches
447 scm ($localpath, $scm, "grep", @args)
448 unless $scm eq "darcs";
450 elsif ($command eq "diff") {
451 scm ($localpath, $scm, "diff", @args)
452 unless $scm eq "darcs";
454 elsif ($command eq "clean") {
455 scm ($localpath, $scm, "clean", @args)
456 unless $scm eq "darcs";
458 elsif ($command eq "reset") {
459 scm ($localpath, $scm, "reset", @args)
460 unless $scm eq "darcs";
462 elsif ($command eq "branch") {
463 scm ($localpath, $scm, "branch", @args)
464 unless $scm eq "darcs";
466 elsif ($command eq "config") {
467 scm ($localpath, $scm, "config", @args)
468 unless $scm eq "darcs";
470 elsif ($command eq "repack") {
471 scm ($localpath, $scm, "repack", @args)
474 elsif ($command eq "format-patch") {
475 scm ($localpath, $scm, "format-patch", @args)
478 elsif ($command eq "gc") {
479 scm ($localpath, $scm, "gc", @args)
480 unless $scm eq "darcs";
482 elsif ($command eq "tag") {
483 scm ($localpath, $scm, "tag", @args);
486 die "Unknown command: $command";
497 # Get the built in help
501 ./sync-all [-q] [-s] [--ignore-failure] [-r repo] [--checked-out] [--bare]
502 [--nofib] [--extra] [--testsuite] [--no-dph] [--resume]
505 Applies the command "cmd" to each repository in the tree.
507 A full repository tree is obtained by first cloning the ghc
508 repository, then getting the subrepositories with "sync-all get":
510 \$ git clone http://darcs.haskell.org/ghc.git
514 After this, "./sync-all pull" will pull from the original repository
517 A remote pointing to another local repository tree can be added like
520 \$ ./sync-all -r /path/to/ghc remote add otherlocal
522 and then we can pull from this other tree with
524 \$ ./sync-all pull otherlocal
526 -------------- Commands -----------------
529 Clones all sub-repositories from the same place that the ghc
530 repository was cloned from. See "which repos to use" below
531 for details of how the subrepositories are laid out.
533 There are various --<package-tag> options that can be given
534 before "get" that enable extra repositories. The full list is
535 given at the end of this help. For example:
537 ./sync-all --testsuite get
539 would get the testsuite repository in addition to the usual set of
542 remote add <remote-name>
543 remote rm <remote-name>
544 remote set-url [--push] <remote-name>
546 Runs a "git remote" command on each subrepository, adjusting the
547 repository location in each case appropriately. For example, to
548 add a new remote pointing to the upstream repositories:
550 ./sync-all -r http://darcs.haskell.org/ remote add upstream
552 The -r flag points to the root of the repository tree (see "which
553 repos to use" below). For a repository on the local filesystem it
554 would point to the ghc repository, and for a remote repository it
555 points to the directory containing "ghc.git".
557 These commands just run the equivalent git command on each repository, passing
558 any extra arguments to git:
581 -------------- Flags -------------------
582 These flags are given *before* the command and modify the way sync-all behaves.
583 Flags given *after* the command are passed to git.
585 -q says to be quiet, and -s to be silent.
587 --resume will restart a command that failed, from the repo at which it
588 failed. This means you don't need to wait while, e.g., "pull" goes through
589 all the repos it's just pulled, and tries to pull them again.
591 --ignore-failure says to ignore errors and move on to the next repository
593 -r repo says to use repo as the location of package repositories
595 --checked-out says that the remote repo is in checked-out layout, as opposed
596 to the layout used for the main repo. By default a repo on the local
597 filesystem is assumed to be checked-out, and repos accessed via HTTP or SSH
598 are assumed to be in the main repo layout; use --checked-out to override the
601 --bare says that the local repo is in bare layout, same as the main repo. It
602 also means that these repos are bare. You only have to use this flag if you
603 don't have a bare ghc.git in the current directory and would like to 'get'
604 all of the repos bare. Requires packages.conf to be present in the current
605 directory (a renamed packages file from the main ghc repo).
607 Note: --checked-out and --bare flags are NOT the opposite of each other.
608 --checked-out: describes the layout of the remote repository tree.
609 --bare: describes the layout of the local repository tree.
611 --nofib also clones the nofib benchmark suite
613 --testsuite also clones the ghc testsuite
615 --extra also clone some extra library packages
617 --no-dph avoids cloning the dph pacakges
620 ------------ Checking out a branch -------------
621 To check out a branch you can run the following command:
623 \$ ./sync-all checkout ghc-7.4
626 ------------ Which repos to use -------------
627 sync-all uses the following algorithm to decide which remote repos to use
629 It always computes the remote repos from a single base, <repo_base> How is
630 <repo_base> set? If you say "-r repo", then that's <repo_base> otherwise
631 <repo_base> is set by asking git where the ghc repo came from, and removing the
632 last component (e.g. /ghc.git/ or /ghc/).
634 Then sync-all iterates over the package found in the file ./packages; see that
635 file for a description of the contents.
637 If <repo_base> looks like a local filesystem path, or if you give the
638 --checked-out flag, sync-all works on repos of form:
640 <repo_base>/<local-path>
642 otherwise sync-all works on repos of form:
644 <repo_base>/<remote-path>
646 This logic lets you say
647 both sync-all -r http://darcs.haskell.org/ghc-6.12 remote add ghc-6.12
648 and sync-all -r ../working remote add working
649 The latter is called a "checked-out tree".
651 sync-all *ignores* the defaultrepo of all repos other than the root one. So the
652 remote repos must be laid out in one of the two formats given by <local-path>
653 and <remote-path> in the file 'packages'.
655 Available package-tags are:
658 # Collect all the tags in the packages file
660 open IN, "< packages.conf"
661 or open IN, "< packages" # clashes with packages directory when using --bare
662 or die "Can't open packages file (or packages.conf)";
665 if (/^([^# ]+) +(?:([^ ]+) +)?([^ ]+) +([^ ]+)/) {
666 if (defined($2) && $2 ne "-") {
667 $available_tags{$2} = 1;
670 elsif (! /^(#.*)?$/) {
676 # Show those tags and the help text
677 my @available_tags = keys %available_tags;
678 print "$help@available_tags\n\n";
689 # We handle -q here as well as lower down as we need to skip over it
690 # if it comes before the source-control command
694 elsif ($arg eq "-s") {
697 elsif ($arg eq "-r") {
698 $defaultrepo = shift;
700 elsif ($arg eq "--resume") {
703 elsif ($arg eq "--ignore-failure") {
706 elsif ($arg eq "--complete" || $arg eq "--partial") {
709 # Use --checked-out if the _remote_ repos are a checked-out tree,
710 # rather than the master trees.
711 elsif ($arg eq "--checked-out") {
712 $checked_out_flag = 1;
714 # Use --bare if the _local_ repos are bare repos,
715 # rather than a checked-out tree.
716 elsif ($arg eq "--bare") {
719 elsif ($arg eq "--help") {
722 # --<tag> says we grab the libs tagged 'tag' with
723 # 'get'. It has no effect on the other commands.
724 elsif ($arg =~ m/^--no-(.*)$/) {
727 elsif ($arg =~ m/^--(.*)$/) {
732 if (grep /^-q$/, @_) {
739 # check for ghc repositories in cwd
740 my $checked_out_found = 1 if (-d ".git" && -d "compiler");
741 my $bare_found = 1 if (-d "ghc.git");
743 if ($bare_flag && ! $bare_found && ! $defaultrepo) {
744 die "error: bare repository ghc.git not found.\n"
745 . " Either clone a bare ghc repo first or specify the repo location. E.g.:\n"
746 . " ./sync-all --bare [--testsuite --nofib --extra] -r http://darcs.haskell.org/ get\n"
748 elsif ($bare_found) {
749 $bare_flag = "--bare";
751 elsif (! $bare_flag && ! $checked_out_found) {
752 die "error: sync-all must be run from the top level of the ghc tree.";
759 # Give the command and rest of the arguments to the main loop
760 # We normalise command names here to avoid duplicating the
761 # abbreviations that we allow.
764 if ($command =~ /^(?:g|ge|get)$/) {
767 elsif ($command =~ /^(?:pus|push)$/) {
770 elsif ($command =~ /^(?:pul|pull)$/) {
773 elsif ($command =~ /^(?:s|se|sen|send)$/) {
776 elsif ($command =~ /^(?:w|wh|wha|what|whats|whatsn|whatsne|whatsnew|status)$/) {
780 scmall ($command, @_);
782 my @submodule_args = grep(/^-q/,@_);
784 if ($command eq "get") {
785 &scm(".", "git", "submodule", "init", @submodule_args);
787 if ($command eq "pull") {
788 my $gitConfig = &tryReadFile(".git/config");
789 if ($gitConfig !~ /submodule/) {
790 &scm(".", "git", "submodule", "init", @submodule_args);
793 if ($command eq "get" or $command eq "pull") {
794 my $gitConfig = &tryReadFile(".git/config");
795 if ($gitConfig !~ /submodule/) {
796 &scm(".", "git", "submodule", "init", @submodule_args);
798 &scm(".", "git", "submodule", "update", @submodule_args);
804 $initial_working_directory = getcwd();
810 chdir($initial_working_directory);
812 message "== Checking for old haddock repo";
813 if (-d "utils/haddock/.git") {
814 chdir("utils/haddock");
815 if ((system "git log -1 87e2ca11c3d1b1bc49900fba0b5c5c6f85650718 > /dev/null 2> /dev/null") == 0) {
817 ============================
820 You have an old haddock repository in your GHC tree!
822 Please remove it (e.g. "rm -r utils/haddock"), and then run
823 "./sync-all get" to get the new repository.
824 ============================
827 chdir($initial_working_directory);
830 message "== Checking for old binary repo";
831 if (-d "libraries/binary/.git") {
832 chdir("libraries/binary");
833 if ((system "git log -1 749ac0efbde3b14901417364a872796598747aaf > /dev/null 2> /dev/null") == 0) {
835 ============================
838 You have an old binary repository in your GHC tree!
840 Please remove it (e.g. "rm -r libraries/binary"), and then run
841 "./sync-all get" to get the new repository.
842 ============================
845 chdir($initial_working_directory);
848 message "== Checking for old mtl repo";
849 if (-d "libraries/mtl/.git") {
850 chdir("libraries/mtl");
851 if ((system "git log -1 c67d8f7247c612dc35242bc67e616f7ea35eadb9 > /dev/null 2> /dev/null") == 0) {
853 ============================
856 You have an old mtl repository in your GHC tree!
858 Please remove it (e.g. "rm -r libraries/mtl"), and then run
859 "./sync-all get" to get the new repository.
860 ============================
863 chdir($initial_working_directory);
866 message "== Checking for old Cabal repo";
867 if (-d "libraries/Cabal/.git") {
868 chdir("libraries/Cabal");
869 if ((system "git log -1 c8ebd66a32865f72ae03ee0663c62df3d77f08fe > /dev/null 2> /dev/null") == 0) {
871 ============================
874 You have an old Cabal repository in your GHC tree!
876 Please remove it (e.g. "rm -r libraries/Cabal"), and then run
877 "./sync-all get" to get the new repository.
878 ============================
881 chdir($initial_working_directory);
884 message "== Checking for old time from tarball";
885 if (-d "libraries/time" and ! -e "libraries/time/.git") {
887 ============================
890 You have an old time package in your GHC tree!
892 Please remove it (e.g. "rm -r libraries/time"), and then run
893 "./sync-all get" to get the new repository.
894 ============================