lei q: support a bunch of curl(1) options

Some of these options will make sense when on weird networks
(behind firewalls, etc.)  Some of these options may not make
sense at all.

This allows users who prefer to use the SOCKS5 proxy support in
curl rather than torsocks(1), but we'll still support torsocks
by default since some Tor instances aren't on the default
127.0.0.1:9050.

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 890be575fe3ad7799761db990c4043be319cb7d6..a9123c6e3c97603e3ee1917cac57912a1b1789ce 100644 (file)
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -84,8 +84,8 @@ our %CMD = ( # sorted in order of importance/use:
 'q' => [ 'SEARCH_TERMS...', 'search for messages matching terms', qw(
        save-as=s output|mfolder|o=s format|f=s dedupe|d=s thread|t augment|a
        sort|s=s reverse|r offset=i remote local! external! pretty mua-cmd=s
-       verbose|v
-       since|after=s until|before=s), opt_dash('limit|n=i', '[0-9]+') ],
+       torsocks=s no-torsocks verbose|v since|after=s until|before=s),
+       PublicInbox::LeiQuery::curl_opt(), opt_dash('limit|n=i', '[0-9]+') ],
 
 'show' => [ 'MID|OID', 'show a given object (Message-ID or object ID)',
        qw(type=s solve! format|f=s dedupe|d=s thread|t remote local!),

diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index eebf217bb316df88093c4628d189145f2d4b7e5d..acab3c2c65ca98801aa3cfccb47d66107da6c19b 100644 (file)
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -66,4 +66,45 @@ sub lei_q {
        $lxs->do_query($self);
 }
 
+# Stuff we may pass through to curl (as of 7.64.0), see curl manpage for
+# details, so most options which make sense for HTTP/HTTPS (including proxy
+# support for Tor and other methods of getting past weird networks).
+# Most of these are untested by us, some may not make sense for our use case
+# and typos below are likely.
+# n.b. some short options (-$NUMBER) are not supported since they conflict
+# with other "lei q" switches.
+# FIXME: Getopt::Long doesn't easily let us support support options with
+# '.' in them (e.g. --http1.1)
+sub curl_opt { qw(
+       abstract-unix-socket=s anyauth basic cacert=s capath=s
+       cert-status cert-type cert|E=s ciphers=s config|K=s@
+       connect-timeout=s connect-to=s cookie-jar|c=s cookie|b=s crlfile=s
+       digest disable dns-interface=s dns-ipv4-addr=s dns-ipv6-addr=s
+       dns-servers=s doh-url=s egd-file=s engine=s false-start
+       happy-eyeballs-timeout-ms=s haproxy-protocol header|H=s@
+       http2-prior-knowledge http2 insecure|k
+       interface=s ipv4 ipv6 junk-session-cookies
+       key-type=s key=s limit-rate=s local-port=s location-trusted location|L
+       max-redirs=i max-time=s negotiate netrc-file=s netrc-optional netrc
+       no-alpn no-buffer|N no-npn no-sessionid noproxy=s ntlm-wb ntlm
+       pass=s pinnedpubkey=s post301 post302 post303 preproxy=s
+       proxy-anyauth proxy-basic proxy-cacert=s proxy-capath=s
+       proxy-cert-type=s proxy-cert=s proxy-ciphers=s proxy-crlfile=s
+       proxy-digest proxy-header=s@ proxy-insecure
+       proxy-key-type=s proxy-key proxy-negotiate proxy-ntlm proxy-pass=s
+       proxy-pinnedpubkey=s proxy-service-name=s proxy-ssl-allow-beast
+       proxy-tls13-ciphers=s proxy-tlsauthtype=s proxy-tlspassword=s
+       proxy-tlsuser=s proxy-tlsv1 proxy-user|U=s proxy=s
+       proxytunnel=s pubkey=s random-file=s referer=s resolve=s
+       retry-connrefused retry-delay=s retry-max-time=s retry=i
+       sasl-ir service-name=s socks4=s socks4a=s socks5-basic
+       socks5-gssapi-service-name=s socks5-gssapi socks5-hostname=s socks5=s
+       speed-limit|Y speed-type|y ssl-allow-beast sslv2 sslv3
+       suppress-connect-headers tcp-fastopen tls-max=s
+       tls13-ciphers=s tlsauthtype=s tlspassword=s tlsuser=s
+       tlsv1 trace-ascii=s trace-time trace=s
+       unix-socket=s user-agent|A=s user|u=s
+)
+}
+
 1;

diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 8d36bca96b17285830beb37cf67978e9fc36dfe4..defe5e67cc2c6e905e2e8d37da7978e9321cb3f5 100644 (file)
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -193,6 +193,7 @@ sub query_remote_mboxrd {
        my $dedupe = $lei->{dedupe} // die 'BUG: {dedupe} missing';
        $dedupe->prepare_dedupe;
        my @cmd = qw(curl -XPOST -sSf);
+       $opt->{torsocks} = 'false' if $opt->{'no-torsocks'};
        my $tor = $opt->{torsocks} //= 'auto';
        if ($tor eq 'auto' && substr($uri->host, -6) eq '.onion' &&
                        (($lei->{env}->{LD_PRELOAD}//'') !~ /torsocks/)) {
@@ -202,6 +203,18 @@ sub query_remote_mboxrd {
        }
        my $verbose = $opt->{verbose};
        push @cmd, '-v' if $verbose;
+       for my $o ($lei->curl_opt) {
+               $o =~ s/\|[a-z0-9]\b//i; # remove single char short option
+               if ($o =~ s/=[is]@\z//) {
+                       my $ary = $opt->{$o} or next;
+                       push @cmd, map { ("--$o", $_) } @$ary;
+               } elsif ($o =~ s/=[is]\z//) {
+                       my $val = $opt->{$o} // next;
+                       push @cmd, "--$o", $val;
+               } elsif ($opt->{$o}) {
+                       push @cmd, "--$o";
+               }
+       }
        push @cmd, $uri->as_string;
        $lei->err("# @cmd") if $verbose;
        $? = 0;