From 3702b79a2d469f9ba8fa7e4d18502f6b2fba50e3 Mon Sep 17 00:00:00 2001
From: Eric Wong <e@80x24.org>
Date: Sun, 31 Oct 2021 09:10:16 +0000
Subject: [PATCH] doc: add lei-mail-sync-overview manpage

Mostly illustrating how clunky the process is :p
We'll also tweak some things in existing man pages around
mail synchronization.
---
 Documentation/lei-export-kw.pod          |  3 ++
 Documentation/lei-forget-mail-sync.pod   |  5 ++-
 Documentation/lei-import.pod             | 34 ++++++++++++---
 Documentation/lei-index.pod              |  5 ++-
 Documentation/lei-ls-label.pod           |  1 +
 Documentation/lei-ls-mail-source.pod     |  5 ++-
 Documentation/lei-ls-watch.pod           |  3 +-
 Documentation/lei-mail-formats.pod       |  5 ++-
 Documentation/lei-mail-sync-overview.pod | 53 ++++++++++++++++++++++++
 Documentation/lei-overview.pod           | 13 +++++-
 Documentation/lei-tag.pod                | 21 +++++++++-
 Documentation/lei.pod                    |  4 +-
 MANIFEST                                 |  1 +
 Makefile.PL                              |  4 +-
 14 files changed, 137 insertions(+), 20 deletions(-)
 create mode 100644 Documentation/lei-mail-sync-overview.pod

diff --git a/Documentation/lei-export-kw.pod b/Documentation/lei-export-kw.pod
index cf482ca0..4cb673ac 100644
--- a/Documentation/lei-export-kw.pod
+++ b/Documentation/lei-export-kw.pod
@@ -12,6 +12,9 @@ lei export-kw MFOLDER [MFOLDER...]
 
 C<lei export-kw> propagates keywords (e.g. C<seen>, C<answered>,
 C<flagged>, etc.) from lei/store to IMAP folders and/or Maildirs.
+It only works for messages lei knows about (e.g. was used as a
+C<lei q --output>, or imported via L<lei-import(1)>, or indexed
+via L<lei-index(1)>).
 
 It does not delete, write, nor modify messages themselves;
 it only sets metadata on Maildirs and IMAP folders.
diff --git a/Documentation/lei-forget-mail-sync.pod b/Documentation/lei-forget-mail-sync.pod
index e70b4d33..29ff5671 100644
--- a/Documentation/lei-forget-mail-sync.pod
+++ b/Documentation/lei-forget-mail-sync.pod
@@ -9,8 +9,9 @@ lei forget-mail-sync [OPTIONS] LOCATION [LOCATION...]
 =head1 DESCRIPTION
 
 Forget synchronization information for C<LOCATION>, an IMAP or Maildir
-folder.  Note that this won't delete any messages stored in Git,
-leaving C<lei-index(1)> users with dangling references.
+folder.  Note that this won't delete any messages on the filesystem.
+Users using L<lei-index(1)> without L<lei-import(1)> will be left
+with dangling references in search results.
 
 =head1 CONTACT
 
diff --git a/Documentation/lei-import.pod b/Documentation/lei-import.pod
index 9322dfa8..25f1daf5 100644
--- a/Documentation/lei-import.pod
+++ b/Documentation/lei-import.pod
@@ -4,7 +4,7 @@ lei-import - one-time import of messages into local store
 
 =head1 SYNOPSIS
 
-lei import [OPTIONS] LOCATION [LOCATION...]
+lei import [OPTIONS] LOCATION [LOCATION...] [+L:LABEL]
 
 lei import [OPTIONS] (--stdin|-)
 
@@ -16,8 +16,8 @@ source of messages: a directory (Maildir), a file, or a URL
 authentication use L<git-credential(1)> to
 fill in the username and password.
 
-For a regular file, the location must have a C<E<lt>formatE<gt>:>
-prefix specifying one of the following formats: C<eml>, C<mboxrd>,
+For a regular file, the C<LOCATION> must have a C<E<lt>formatE<gt>:>
+prefix specifying one of the following formats: C<mboxrd>,
 C<mboxcl2>, C<mboxcl>, or C<mboxo>.
 
 =head1 OPTIONS
@@ -42,6 +42,29 @@ C<none>.
 
 Default: fcntl,dotlock
 
+=item +L:LABEL
+
+Add the given C<LABEL> to all messages imported, where C<LABEL>
+is an arbitrary user-defined value consisting of lowercase and digits.
+See L<lei-tag(1)> for more info on labels.
+
+For example, specifying C<+L:inbox> applies the C<inbox> label
+to all messages being imported.
+
+May be specified multiple times to apply multiple labels.
+
+Default: none
+
+=item +kw:KEYWORD
+
+Apply C<KEYWORD> to all messages being imported in addition
+to any per-message keywords from the store (unless C<--no-kw>
+is specified).  See L<lei-tag(1)> for more info on keywords.
+
+May be specified multiple times to apply multiple keywords.
+
+Default: none
+
 =item --no-kw
 
 Don't import message keywords (or "flags" in IMAP terminology).
@@ -73,11 +96,10 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
 
 =head1 COPYRIGHT
 
-Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
-
 =head1 SEE ALSO
 
-L<lei-add-external(1)>
+L<lei-index(1)>
diff --git a/Documentation/lei-index.pod b/Documentation/lei-index.pod
index 9e720263..f8ff6950 100644
--- a/Documentation/lei-index.pod
+++ b/Documentation/lei-index.pod
@@ -20,6 +20,9 @@ Combined with L<lei-q(1)>, C<lei index> allows Maildir users to
 have similar functionality to L<mairix(1)> by not duplicating
 messages into C<lei/store>.
 
+Occasional invocations of C<lei-refresh-mail-sync --all=local>
+are recommended to keep indexed messages retrievable.
+
 =head1 OPTIONS
 
 =over
@@ -58,4 +61,4 @@ License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
 =head1 SEE ALSO
 
-L<lei-store-format(5)>, L<lei-import(1)>
+L<lei-refresh-mail-sync(1)>, L<lei-store-format(5)>, L<lei-import(1)>
diff --git a/Documentation/lei-ls-label.pod b/Documentation/lei-ls-label.pod
index 41aa030d..8342705a 100644
--- a/Documentation/lei-ls-label.pod
+++ b/Documentation/lei-ls-label.pod
@@ -9,6 +9,7 @@ lei ls-label [OPTIONS]
 =head1 DESCRIPTION
 
 List all known message labels ("mailboxes" in JMAP terminology).
+This is handy for writing L<lei-import(1)> invocations.
 
 =head1 OPTIONS
 
diff --git a/Documentation/lei-ls-mail-source.pod b/Documentation/lei-ls-mail-source.pod
index 926bbe2c..59d14afe 100644
--- a/Documentation/lei-ls-mail-source.pod
+++ b/Documentation/lei-ls-mail-source.pod
@@ -9,6 +9,8 @@ lei ls-mail-source [OPTIONS] URL
 =head1 DESCRIPTION
 
 List information about the IMAP or NNTP mail source at C<URL>.
+This command populates the cache used for Bash shell completion
+and is handy for writing L<lei-import(1)> invocations.
 
 =head1 OPTIONS
 
@@ -48,11 +50,10 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
 
 =head1 COPYRIGHT
 
-Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
-
 =head1 SEE ALSO
 
 L<lei-import(1)>
diff --git a/Documentation/lei-ls-watch.pod b/Documentation/lei-ls-watch.pod
index b1681ee4..8063d34d 100644
--- a/Documentation/lei-ls-watch.pod
+++ b/Documentation/lei-ls-watch.pod
@@ -8,7 +8,8 @@ lei ls-watch
 
 =head1 DESCRIPTION
 
-List locations that lei is configured to watch.
+List locations that lei is configured to watch.  This command is
+incomplete, mail-sync locations are implicitly watched.
 
 =head1 CONTACT
 
diff --git a/Documentation/lei-mail-formats.pod b/Documentation/lei-mail-formats.pod
index 3c37c880..930c5d76 100644
--- a/Documentation/lei-mail-formats.pod
+++ b/Documentation/lei-mail-formats.pod
@@ -96,7 +96,10 @@ it worth supporting.
 
 Depending on the IMAP server software and configuration, IMAP
 servers may use any (or combination) of the aforementioned
-formats or a non-standard database backend.
+formats or a non-standard database backend.  Currently, lei
+uses L<Mail::IMAPClient> which has acceptable performance
+over low-latency links.  Performance over high-latency links
+is currently poor.
 
 =head1 eml
 
diff --git a/Documentation/lei-mail-sync-overview.pod b/Documentation/lei-mail-sync-overview.pod
new file mode 100644
index 00000000..e30674bb
--- /dev/null
+++ b/Documentation/lei-mail-sync-overview.pod
@@ -0,0 +1,53 @@
+=head1 NAME
+
+lei - an overview of lei mail synchronization
+
+=head1 DESCRIPTION
+
+L<lei(1)> provides several plumbing-level commands to synchronize
+mail and keywords (flags) between lei/store and existing IMAP
+and Maildir stores.  Nothing documented in this manpage is required
+for day-to-day use against externals.
+
+Mail and keyword synchronization is currently a clunky process.
+Future work will be done to improve it and add IMAP IDLE support.
+
+=head1 TYPICAL WORKFLOW
+
+  # import mail from a user's IMAP inbox and give it the "inbox" label:
+  lei import +L:inbox imaps://user@example.com/INBOX
+
+  # dump "inbox" labeled files from the past week to a Maildir
+  lei q L:inbox rt:last.week.. -o /tmp/results
+
+  # open /tmp/results in your favorite mail agent.  If inotify or kevent
+  # works, keyword changes (e.g. marking messages as `seen') are
+  # synchronized automatically.
+
+  # If the inotify queue overflows, or if lei-daemon crashes,
+  # "lei index" will tell lei about keyword changes:
+  lei index /tmp/results
+
+  # Optional: cleanup stale entries from mail_sync.sqlite3
+  lei refresh-mail-sync /tmp/results
+
+  # to export keyword changes back to IMAP
+  lei export-kw imaps://user@example.com/INBOX
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-import(1)>, L<lei-q(1)>, L<lei-index(1)>,
+L<lei-refresh-mail-sync(1)>, L<lei-export-kw(1)>
diff --git a/Documentation/lei-overview.pod b/Documentation/lei-overview.pod
index 99fd6ef7..7095b504 100644
--- a/Documentation/lei-overview.pod
+++ b/Documentation/lei-overview.pod
@@ -4,8 +4,8 @@ lei - an overview of lei
 
 =head1 DESCRIPTION
 
-L<lei(1)> is a local email interface for public-inbox.  This document
-provides some basic examples.
+L<lei(1)> is a local email interface for public-inbox and personal mail.
+This document provides some basic examples.
 
 =head1 LEI STORE
 
@@ -49,6 +49,11 @@ For existing local paths, the external needs to be indexed with
 L<public-inbox-index(1)> (in the case of a regular inbox) or
 L<public-inbox-extindex(1)> (in the case of an external index).
 
+=head1 SYNCHRONIZATION
+
+lei currently has primitive mail synchronization abilities;
+see L<lei-mail-sync-overview(7)> for more details.
+
 =head2 EXAMPLES
 
 =over
@@ -154,3 +159,7 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
 Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-mail-sync-overview(7)>
diff --git a/Documentation/lei-tag.pod b/Documentation/lei-tag.pod
index b2509ba4..8cb9e736 100644
--- a/Documentation/lei-tag.pod
+++ b/Documentation/lei-tag.pod
@@ -43,6 +43,26 @@ Suppress feedback messages.
 
 =back
 
+=head1 LABELS
+
+Labels are user-defined values analogous to IMAP/JMAP mailbox
+names.  They must only contain lowercase characters, digits, and
+a limited amount of punctuation (e.g. C<.>, C<->, C<@>).
+
+Messages may have multiple labels.
+
+=head1 KEYWORDS
+
+Keywords are "flags" in Maildir and IMAP terminology.
+Common keywords include: C<seen>, C<answered>, C<flagged>, and
+C<draft>, though C<forwarded>, C<phishing>, C<junk>, and C<notjunk>
+are also supported.
+
+When writing to various mboxes, the common keywords will be
+mapped to the C<Status> and C<X-Status> headers.
+
+Messages may have multiple keywords.
+
 =head1 CONTACT
 
 Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
@@ -56,7 +76,6 @@ Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
-
 =head1 SEE ALSO
 
 L<lei-add-external(1)>
diff --git a/Documentation/lei.pod b/Documentation/lei.pod
index 24a585da..f01f506a 100644
--- a/Documentation/lei.pod
+++ b/Documentation/lei.pod
@@ -106,7 +106,7 @@ Other subcommands include
 
 =item * L<lei-daemon-pid(1)>
 
-=item * lei-forget-mail-sync(1)
+=item * L<lei-forget-mail-sync(1)>
 
 =item * L<lei-mail-diff(1)>
 
@@ -145,7 +145,7 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
 
 =head1 COPYRIGHT
 
-Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
diff --git a/MANIFEST b/MANIFEST
index 9fd979ef..1e8f60fb 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -47,6 +47,7 @@ Documentation/lei-ls-search.pod
 Documentation/lei-ls-watch.pod
 Documentation/lei-mail-diff.pod
 Documentation/lei-mail-formats.pod
+Documentation/lei-mail-sync-overview.pod
 Documentation/lei-overview.pod
 Documentation/lei-p2q.pod
 Documentation/lei-q.pod
diff --git a/Makefile.PL b/Makefile.PL
index b3ac59be..8c8c0235 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -1,5 +1,5 @@
 #!/usr/bin/perl -w
-# Copyright (C) 2013-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use ExtUtils::MakeMaker;
@@ -57,7 +57,7 @@ $v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
 		public-inbox-v2-format public-inbox-extindex-format
 		lei-mail-formats lei-store-format
 		) ];
-$v->{-m7} = [ qw(lei-overview lei-security
+$v->{-m7} = [ qw(lei-mail-sync-overview lei-overview lei-security
 		public-inbox-overview public-inbox-tuning
 		public-inbox-glossary) ];
 $v->{-m8} = [ qw(public-inbox-daemon lei-daemon) ];
-- 
2.44.0