1 # Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
2 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
4 # *-external commands of lei
5 package PublicInbox::LeiExternal;
8 use PublicInbox::Config;
11 my ($self, $cb, @arg) = @_;
12 my $cfg = $self->_lei_cfg(0);
14 for my $sec (grep(/\Aexternal\./, @{$cfg->{-section_order}})) {
15 my $loc = substr($sec, length('external.'));
16 $boost{$loc} = $cfg->{"$sec.boost"};
18 return \%boost if !wantarray && !$cb;
20 # highest boost first, but stable for alphabetic tie break
22 my @order = sort { $boost{$b} <=> $boost{$a} } sort keys %boost;
23 if (ref($cb) eq 'CODE') {
24 for my $loc (@order) {
25 $cb->(@arg, $loc, $boost{$loc});
27 } elsif (ref($cb) eq 'HASH') {
30 @order; # scalar or array
33 sub ext_canonicalize {
35 if ($location !~ m!\Ahttps?://!) {
36 PublicInbox::Config::rel2abs_collapsed($location);
39 my $uri = URI->new($location)->canonical;
40 my $path = $uri->path . '/';
41 $path =~ tr!/!/!s; # squeeze redundant '/'
47 # TODO: we will probably extract glob2re into a separate module for
48 # PublicInbox::Filter::Base and maybe other places
49 my %re_map = ( '*' => '[^/]*?', '?' => '[^/]',
50 '[' => '[', ']' => ']', ',' => ',' );
57 my $changes = ($re =~ s!(.)!
58 $re_map{$p eq '\\' ? '' : do {
59 if ($1 eq '[') { ++$in_bracket }
60 elsif ($1 eq ']') { --$in_bracket }
61 elsif ($1 eq ',') { ++$qm } # no change
65 ($p eq '-' && $in_bracket) ? $p : (++$qm, "\Q$p")
67 # bashism (also supported by curl): {a,b,c} => (a|b|c)
68 $changes += ($re =~ s/([^\\]*)\\\{([^,]*,[^\\]*)\\\}/
69 (my $in_braces = $2) =~ tr!,!|!;
72 ($changes - $qm) ? $re : undef;
75 # get canonicalized externals list matching $loc
76 # $is_exclude denotes it's for --exclude
77 # otherwise it's for --only/--include is assumed
79 my ($self, $loc, $is_exclude) = @_;
80 return (ext_canonicalize($loc)) if -e $loc;
82 my @cur = externals_each($self);
83 my $do_glob = !$self->{opt}->{globoff}; # glob by default
84 if ($do_glob && (my $re = glob2re($loc))) {
85 @m = grep(m!$re!, @cur);
86 return @m if scalar(@m);
87 } elsif (index($loc, '/') < 0) { # exact basename match:
88 @m = grep(m!/\Q$loc\E/?\z!, @cur);
89 return @m if scalar(@m) == 1;
90 } elsif ($is_exclude) { # URL, maybe:
91 my $canon = ext_canonicalize($loc);
92 @m = grep(m!\A\Q$canon\E\z!, @cur);
93 return @m if scalar(@m) == 1;
95 return (ext_canonicalize($loc));
97 if (scalar(@m) == 0) {
98 $self->fail("`$loc' is unknown");
100 $self->fail("`$loc' is ambiguous:\n", map { "\t$_\n" } @m);
105 # TODO: does this need JSON output?
106 sub lei_ls_external {
107 my ($self, $filter) = @_;
108 my $opt = $self->{opt};
109 my $do_glob = !$opt->{globoff}; # glob by default
110 my ($OFS, $ORS) = $opt->{z} ? ("\0", "\0\0") : (" ", "\n");
112 my $re = $do_glob ? glob2re($filter) : undef;
113 $re //= index($filter, '/') < 0 ?
114 qr!/\Q$filter\E/?\z! : # exact basename match
115 qr/\Q$filter\E/; # grep -F semantics
116 my @ext = externals_each($self, my $boost = {});
117 @ext = $opt->{'invert-match'} ? grep(!/$re/, @ext)
119 if ($opt->{'local'} && !$opt->{remote}) {
120 @ext = grep(!m!\A[a-z\+]+://!, @ext);
121 } elsif ($opt->{remote} && !$opt->{'local'}) {
122 @ext = grep(m!\A[a-z\+]+://!, @ext);
125 $self->out($loc, $OFS, 'boost=', $boost->{$loc}, $ORS);
129 sub add_external_finish {
130 my ($self, $location) = @_;
131 my $cfg = $self->_lei_cfg(1);
132 my $new_boost = $self->{opt}->{boost} // 0;
133 my $key = "external.$location.boost";
134 my $cur_boost = $cfg->{$key};
135 return if defined($cur_boost) && $cur_boost == $new_boost; # idempotent
136 $self->lei_config($key, $new_boost);
139 sub lei_add_external {
140 my ($self, $location) = @_;
141 $self->_lei_store(1)->write_prepare($self);
142 my $opt = $self->{opt};
143 my $mirror = $opt->{mirror} // do {
145 for my $sw ($self->index_opt, $self->curl_opt,
146 qw(c no-torsocks torsocks inbox-version)) {
147 my ($f) = (split(/|/, $sw, 2))[0];
148 next unless defined $opt->{$f};
149 $f = length($f) == 1 ? "-$f" : "--$f";
152 if (scalar(@fail) == 1) {
153 return $self->("@fail requires --mirror");
155 my $last = pop @fail;
156 my $fail = join(', ', @fail);
157 return $self->("@fail and $last require --mirror");
161 my $new_boost = $opt->{boost} // 0;
162 $location = ext_canonicalize($location);
163 if (defined($mirror) && -d $location) {
164 $self->fail(<<""); # TODO: did you mean "update-external?"
165 --mirror destination `$location' already exists
168 if ($location !~ m!\Ahttps?://! && !-d $location) {
169 $mirror // return $self->fail("$location not a directory");
170 $mirror = ext_canonicalize($mirror);
171 require PublicInbox::LeiMirror;
172 PublicInbox::LeiMirror->start($self, $mirror => $location);
174 add_external_finish($self, $location);
178 sub lei_forget_external {
179 my ($self, @locations) = @_;
180 my $cfg = $self->_lei_cfg(1);
181 my $quiet = $self->{opt}->{quiet};
183 for my $loc (@locations) {
184 my (@unset, @not_found);
185 for my $l ($loc, ext_canonicalize($loc)) {
187 my $key = "external.$l.boost";
188 delete($cfg->{$key});
189 $self->_config('--unset', $key);
192 } elsif (($? >> 8) == 5) {
195 $self->err("# --unset $key error");
196 return $self->x_it($?);
201 $self->err("# $_ gone") for @unset;
202 } elsif (@not_found) {
203 $self->err("# $_ not found") for @not_found;
204 } # else { already exited
208 sub _complete_url_common ($) {
210 # Workaround bash word-splitting URLs to ['https', ':', '//' ...]
211 # Maybe there's a better way to go about this in
212 # contrib/completion/lei-completion.bash
214 my $cur = pop @$argv;
217 if ($cur eq ':' && @x) {
221 while (@x > 2 && $x[0] !~ /\Ahttps?\z/ && $x[1] ne ':') {
224 if (@x >= 2) { # qw(https : hostname : 443) or qw(http :)
226 } else { # just filter out the flags and hope for the best
227 $re = join('', grep(!/^-/, @$argv));
229 $re = quotemeta($re);
234 # shell completion helper called by lei__complete
235 sub _complete_forget_external {
236 my ($self, @argv) = @_;
237 my $cfg = $self->_lei_cfg(0);
238 my ($cur, $re) = _complete_url_common(\@argv);
239 # FIXME: bash completion off "http:" or "https:" when the last
240 # character is a colon doesn't work properly even if we're
241 # returning "//$HTTP_HOST/$PATH_INFO/", not sure why, could
244 my $x = substr($_, length('external.'));
245 # only return the part specified on the CLI
246 # don't duplicate if already 100% completed
247 $x =~ /\A$re(\Q$cur\E.*)/ ? ($cur eq $1 ? () : $1) : ();
248 } grep(/\Aexternal\.$re\Q$cur/, @{$cfg->{-section_order}});
251 sub _complete_add_external { # for bash, this relies on "compopt -o nospace"
252 my ($self, @argv) = @_;
253 my $cfg = $self->_lei_cfg(0);
254 my ($cur, $re) = _complete_url_common(\@argv);
257 my $u = URI->new(substr($_, length('external.')));
258 my ($base) = ($u->path =~ m!((?:/?.*)?/)[^/]+/?\z!);
261 $u =~ /\A$re(\Q$cur\E.*)/ ? ($cur eq $1 ? () : $1) : ();
262 } grep(m!\Aexternal\.https?://!, @{$cfg->{-section_order}});