lib/PublicInbox/MsgTime.pm
lib/PublicInbox/Msgmap.pm
lib/PublicInbox/MultiGit.pm
-lib/PublicInbox/NDC_PP.pm
lib/PublicInbox/NNTP.pm
lib/PublicInbox/NNTPD.pm
lib/PublicInbox/NNTPdeflate.pm
exec($x);
__DATA__
#define _GNU_SOURCE
-#include <unistd.h>
#include <sys/syscall.h>
+#include <sys/ioctl.h>
+#include <linux/fs.h>
+#include <unistd.h>
#include <stdio.h>
#define D(x) printf("$" #x " = %ld;\n", (long)x)
D(SYS_inotify_add_watch);
D(SYS_inotify_rm_watch);
D(SYS_prctl);
+ D(SYS_fstatfs);
+#ifdef FS_IOC_GETFLAGS
+ printf("FS_IOC_GETFLAGS=%#lx\nFS_IOC_SETFLAGS=%#lx\n",
+ (unsigned long)FS_IOC_GETFLAGS, (unsigned long)FS_IOC_SETFLAGS);
+#endif
+
#ifdef SYS_renameat2
D(SYS_renameat2);
#endif
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
package PublicInbox::IMAPTracker;
use strict;
}
if (!-f $dbname) {
require File::Path;
- require PublicInbox::Spawn;
+ require PublicInbox::Syscall;
my ($dir) = ($dbname =~ m!(.*?/)[^/]+\z!);
File::Path::mkpath($dir);
+ PublicInbox::Syscall::nodatacow_dir($dir);
open my $fh, '+>>', $dbname or die "failed to open $dbname: $!";
- PublicInbox::Spawn::nodatacow_fd(fileno($fh));
}
my $self = bless { lock_path => "$dbname.lock", url => $url }, $class;
$self->lock_acquire;
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# for maintaining synchronization between lei/store <=> Maildir|MH|IMAP|JMAP
my $f = $self->{filename};
my $creat = $rw && !-s $f;
if ($creat) {
- require PublicInbox::Spawn;
+ require PublicInbox::Syscall;
open my $fh, '+>>', $f or Carp::croak "open($f): $!";
- PublicInbox::Spawn::nodatacow_fd(fileno($fh));
+ PublicInbox::Syscall::nodatacow_fh($fh);
}
my $dbh = DBI->connect("dbi:SQLite:dbname=$f",'','', {
AutoCommit => 1,
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# like PublicInbox::SearchIdx, but for searching for non-mail messages.
use PublicInbox::InboxWritable;
use PublicInbox::Search; # for SWIG Xapian and Search::Xapian compat
use PublicInbox::SearchIdx qw(index_text term_generator add_val);
-use PublicInbox::Spawn qw(nodatacow_dir);
use Carp qw(croak);
use File::Path ();
use PublicInbox::MiscSearch;
use PublicInbox::Config;
+use PublicInbox::Syscall;
my $json;
sub new {
PublicInbox::SearchIdx::load_xapian_writable();
my $mi_dir = "$eidx->{xpfx}/misc";
File::Path::mkpath($mi_dir);
- nodatacow_dir($mi_dir);
+ PublicInbox::Syscall::nodatacow_dir($mi_dir);
my $flags = $PublicInbox::SearchIdx::DB_CREATE_OR_OPEN;
$flags |= $PublicInbox::SearchIdx::DB_NO_SYNC if $eidx->{-no_fsync};
$json //= PublicInbox::Config::json();
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# bidirectional Message-ID <-> Article Number mapping for the NNTP
use DBI;
use DBD::SQLite;
use PublicInbox::Over;
-use PublicInbox::Spawn;
use Scalar::Util qw(blessed);
sub new_file {
require File::Temp;
my $tmp = "mm_tmp-$$-XXXX";
my ($fh, $fn) = File::Temp::tempfile($tmp, EXLOCK => 0, DIR => $dir);
- PublicInbox::Spawn::nodatacow_fd(fileno($fh));
+ require PublicInbox::Syscall;
+ PublicInbox::Syscall::nodatacow_fh($fh);
$self->{dbh}->sqlite_backup_to_file($fn);
$tmp = ref($self)->new_file($fn, 2);
$tmp->{dbh}->do('PRAGMA journal_mode = MEMORY');
+++ /dev/null
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
-# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-
-# Pure-perl class for Linux non-Inline::C users to disable COW for btrfs
-package PublicInbox::NDC_PP;
-use strict;
-use v5.10.1;
-
-sub nodatacow_dir ($) {
- my ($path) = @_;
- open my $mh, '<', '/proc/self/mounts' or return;
- for (grep(/ btrfs /, <$mh>)) {
- my (undef, $mnt_path, $type) = split(/ /);
- next if $type ne 'btrfs'; # in case of false-positive from grep
-
- # weird chars are escaped as octal
- $mnt_path =~ s/\\(0[0-9]{2})/chr(oct($1))/egs;
- $mnt_path .= '/' unless $mnt_path =~ m!/\z!;
- if (index($path, $mnt_path) == 0) {
- # error goes to stderr, but non-fatal for us
- system('chattr', '+C', $path);
- last;
- }
- }
-}
-
-sub nodatacow_fd ($) {
- my ($fd) = @_;
- return if $^O ne 'linux';
- defined(my $path = readlink("/proc/self/fd/$fd")) or return;
- nodatacow_dir($path);
-}
-
-1;
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# for XOVER, OVER in NNTP, and feeds/homepage/threads in PSGI
my $f = delete $self->{filename};
if (!-s $f) { # SQLite defaults mode to 0644, we want 0666
if ($rw) {
- require PublicInbox::Spawn;
+ require PublicInbox::Syscall;
my ($dir) = ($f =~ m!(.+)/[^/]+\z!);
- PublicInbox::Spawn::nodatacow_dir($dir);
+ PublicInbox::Syscall::nodatacow_dir($dir);
open my $fh, '+>>', $f or die "failed to open $f: $!";
- PublicInbox::Spawn::nodatacow_fd(fileno($fh));
} else {
$self->{filename} = $f; # die on stat() below:
}
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# based on notmuch, but with no concept of folders, files
#
use POSIX qw(strftime);
use Time::Local qw(timegm);
use PublicInbox::OverIdx;
-use PublicInbox::Spawn qw(spawn nodatacow_dir);
+use PublicInbox::Spawn qw(spawn);
use PublicInbox::Git qw(git_unquote);
use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
use PublicInbox::Address;
if (!-d $dir && (!$is_shard ||
($is_shard && need_xapian($self)))) {
File::Path::mkpath($dir);
- nodatacow_dir($dir);
+ require PublicInbox::Syscall;
+ PublicInbox::Syscall::nodatacow_dir($dir);
$self->{-set_has_threadid_once} = 1;
}
}
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# fork()-friendly key-value store. Will be used for making
my $f = $self->{filename} = "$dir/$base.sqlite3";
$self->{lock_path} = $opt->{lock_path} // "$dir/$base.flock";
unless (-s $f) {
- PublicInbox::Spawn::nodatacow_dir($dir); # for journal/shm/wal
+ require PublicInbox::Syscall;
+ PublicInbox::Syscall::nodatacow_dir($dir); # for journal/shm/wal
open my $fh, '+>>', $f or die "failed to open $f: $!";
- PublicInbox::Spawn::nodatacow_fd(fileno($fh));
}
$self;
}
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
#
# This allows vfork to be used for spawning subprocesses if
use Fcntl qw(LOCK_EX SEEK_SET);
use IO::Handle ();
use PublicInbox::ProcessPipe;
-our @EXPORT_OK = qw(which spawn popen_rd run_die nodatacow_dir);
+our @EXPORT_OK = qw(which spawn popen_rd run_die);
our @RLIMITS = qw(RLIMIT_CPU RLIMIT_CORE RLIMIT_DATA);
BEGIN {
#endif /* defined(CMSG_SPACE) && defined(CMSG_LEN) */
ALL_LIBC
-# btrfs on Linux is copy-on-write (COW) by default. As of Linux 5.7,
-# this still leads to fragmentation for SQLite and Xapian files where
-# random I/O happens, so we disable COW just for SQLite files and Xapian
-# directories. Disabling COW disables checksumming, so we only do this
-# for regeneratable files, and not canonical git storage (git doesn't
-# checksum refs, only data under $GIT_DIR/objects).
- my $set_nodatacow = $^O eq 'linux' ? <<'SET_NODATACOW' : '';
-#include <sys/ioctl.h>
-#include <sys/vfs.h>
-#include <linux/magic.h>
-#include <linux/fs.h>
-#include <dirent.h>
-
-void nodatacow_fd(int fd)
-{
- struct statfs buf;
- int val = 0;
-
- if (fstatfs(fd, &buf) < 0) {
- fprintf(stderr, "fstatfs: %s\\n", strerror(errno));
- return;
- }
-
- /* only btrfs is known to have this problem, so skip for non-btrfs */
- if (buf.f_type != BTRFS_SUPER_MAGIC)
- return;
-
- if (ioctl(fd, FS_IOC_GETFLAGS, &val) < 0) {
- fprintf(stderr, "FS_IOC_GET_FLAGS: %s\\n", strerror(errno));
- return;
- }
- val |= FS_NOCOW_FL;
- if (ioctl(fd, FS_IOC_SETFLAGS, &val) < 0)
- fprintf(stderr, "FS_IOC_SET_FLAGS: %s\\n", strerror(errno));
-}
-
-void nodatacow_dir(const char *dir)
-{
- DIR *dh = opendir(dir);
- int fd;
-
- if (!dh) croak("opendir(%s): %s", dir, strerror(errno));
- fd = dirfd(dh);
- if (fd >= 0)
- nodatacow_fd(fd);
- /* ENOTSUP probably won't happen under Linux... */
- closedir(dh);
-}
-SET_NODATACOW
-
my $inline_dir = $ENV{PERL_INLINE_DIRECTORY} //= (
$ENV{XDG_CACHE_HOME} //
( ($ENV{HOME} // '/nonexistent').'/.cache' )
).'/public-inbox/inline-c';
warn "$inline_dir exists, not writable\n" if -e $inline_dir && !-w _;
- $set_nodatacow = $all_libc = undef unless -d _ && -w _;
+ $all_libc = undef unless -d _ && -w _;
if (defined $all_libc) {
my $f = "$inline_dir/.public-inbox.lock";
open my $oldout, '>&', \*STDOUT or die "dup(1): $!";
# CentOS 7.x ships Inline 0.53, 0.64+ has built-in locking
flock($fh, LOCK_EX) or die "LOCK_EX($f): $!";
eval <<'EOM';
-use Inline C => $all_libc.$set_nodatacow, BUILD_NOISY => 1;
+use Inline C => $all_libc, BUILD_NOISY => 1;
EOM
my $err = $@;
my $ndc_err = '';
- if ($err && $set_nodatacow) { # missing Linux kernel headers
- $ndc_err = "with set_nodatacow: <\n$err\n>\n";
- undef $set_nodatacow;
- eval <<'EOM';
-use Inline C => $all_libc, BUILD_NOISY => 1;
-EOM
- };
$err = $@;
open(STDERR, '>&', $olderr) or warn "restore stderr: $!";
open(STDOUT, '>&', $oldout) or warn "restore stdout: $!";
my @msg = <$fh>;
warn "Inline::C build failed:\n",
$ndc_err, $err, "\n", @msg;
- $set_nodatacow = $all_libc = undef;
- } elsif ($ndc_err) {
- warn "Inline::C build succeeded w/o set_nodatacow\n",
- "error $ndc_err";
+ $all_libc = undef;
}
}
unless ($all_libc) {
require PublicInbox::SpawnPP;
*pi_fork_exec = \&PublicInbox::SpawnPP::pi_fork_exec
}
- unless ($set_nodatacow) {
- require PublicInbox::NDC_PP;
- no warnings 'once';
- *nodatacow_fd = \&PublicInbox::NDC_PP::nodatacow_fd;
- *nodatacow_dir = \&PublicInbox::NDC_PP::nodatacow_dir;
- }
} # /BEGIN
sub which ($) {
# This license differs from the rest of public-inbox
#
# This module is Copyright (c) 2005 Six Apart, Ltd.
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
#
# All rights reserved.
#
$SYS_renameat2,
);
+my $SYS_fstatfs; # don't need fstatfs64, just statfs.f_type
my $SFD_CLOEXEC = 02000000; # Perl does not expose O_CLOEXEC
our $no_deprecated = 0;
$SYS_epoll_wait = 256;
$SYS_signalfd4 = 327;
$SYS_renameat2 //= 353;
+ $SYS_fstatfs = 100;
} elsif ($machine eq "x86_64") {
$SYS_epoll_create = 213;
$SYS_epoll_ctl = 233;
$SYS_epoll_wait = 232;
$SYS_signalfd4 = 289;
$SYS_renameat2 //= 316;
+ $SYS_fstatfs = 138;
} elsif ($machine eq 'x32') {
$SYS_epoll_create = 1073742037;
$SYS_epoll_ctl = 1073742057;
$SYS_epoll_wait = 1073742056;
$SYS_signalfd4 = 1073742113;
$SYS_renameat2 //= 0x40000000 + 316;
+ $SYS_fstatfs = 138;
} elsif ($machine eq 'sparc64') {
$SYS_epoll_create = 193;
$SYS_epoll_ctl = 194;
$SYS_signalfd4 = 317;
$SYS_renameat2 //= 345;
$SFD_CLOEXEC = 020000000;
+ $SYS_fstatfs = 158;
} elsif ($machine =~ m/^parisc/) {
$SYS_epoll_create = 224;
$SYS_epoll_ctl = 225;
$u64_mod_8 = 1;
$SYS_signalfd4 = 313;
$SYS_renameat2 //= 357;
+ $SYS_fstatfs = 100;
} elsif ($machine eq "ppc") {
$SYS_epoll_create = 236;
$SYS_epoll_ctl = 237;
$u64_mod_8 = 1;
$SYS_signalfd4 = 313;
$SYS_renameat2 //= 357;
+ $SYS_fstatfs = 100;
} elsif ($machine =~ m/^s390/) {
$SYS_epoll_create = 249;
$SYS_epoll_ctl = 250;
$u64_mod_8 = 1;
$SYS_signalfd4 = 322;
$SYS_renameat2 //= 347;
+ $SYS_fstatfs = 100;
} elsif ($machine eq "ia64") {
$SYS_epoll_create = 1243;
$SYS_epoll_ctl = 1244;
$no_deprecated = 1;
$SYS_signalfd4 = 74;
$SYS_renameat2 //= 276;
+ $SYS_fstatfs = 44;
} elsif ($machine =~ m/arm(v\d+)?.*l/) {
# ARM OABI
$SYS_epoll_create = 250;
$u64_mod_8 = 1;
$SYS_signalfd4 = 355;
$SYS_renameat2 //= 382;
+ $SYS_fstatfs = 100;
} elsif ($machine =~ m/^mips64/) {
$SYS_epoll_create = 5207;
$SYS_epoll_ctl = 5208;
$u64_mod_8 = 1;
$SYS_signalfd4 = 5283;
$SYS_renameat2 //= 5311;
+ $SYS_fstatfs = 5135;
} elsif ($machine =~ m/^mips/) {
$SYS_epoll_create = 4248;
$SYS_epoll_ctl = 4249;
$u64_mod_8 = 1;
$SYS_signalfd4 = 4324;
$SYS_renameat2 //= 4351;
+ $SYS_fstatfs = 4100;
} else {
# as a last resort, try using the *.ph files which may not
# exist or may be wrong
}
}
+sub nodatacow_fh {
+ return if !defined($SYS_fstatfs);
+ my $buf = '';
+ vec($buf, 120 * 8 - 1, 1) = 0;
+ my ($fh) = @_;
+ syscall($SYS_fstatfs, fileno($fh), $buf) == 0 or
+ return warn("fstatfs: $!\n");
+ my $f_type = unpack('l!', $buf); # statfs.f_type is a signed word
+ return if $f_type != 0x9123683E; # BTRFS_SUPER_MAGIC
+
+ state ($FS_IOC_GETFLAGS, $FS_IOC_SETFLAGS);
+ unless (defined $FS_IOC_GETFLAGS) {
+ if (substr($Config{byteorder}, 0, 4) eq '1234') {
+ $FS_IOC_GETFLAGS = 0x80086601;
+ $FS_IOC_SETFLAGS = 0x40086602;
+ } else { # Big endian
+ $FS_IOC_GETFLAGS = 0x40086601;
+ $FS_IOC_SETFLAGS = 0x80086602;
+ }
+ }
+ ioctl($fh, $FS_IOC_GETFLAGS, $buf) //
+ return warn("FS_IOC_GET_FLAGS: $!\n");
+ my $attr = unpack('l!', $buf);
+ return if ($attr & 0x00800000); # FS_NOCOW_FL;
+ ioctl($fh, $FS_IOC_SETFLAGS, pack('l', $attr | 0x00800000)) //
+ warn("FS_IOC_SET_FLAGS: $!\n");
+}
+
+sub nodatacow_dir {
+ if (open my $fh, '<', $_[0]) { nodatacow_fh($fh) }
+}
+
1;
=head1 WARRANTY
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
package PublicInbox::Xapcmd;
use strict;
-use PublicInbox::Spawn qw(which popen_rd nodatacow_dir);
+use PublicInbox::Spawn qw(which popen_rd);
+use PublicInbox::Syscall;
use PublicInbox::Admin qw(setup_signals);
use PublicInbox::Over;
use PublicInbox::SearchIdx;
my $v = PublicInbox::Search::SCHEMA_VERSION();
my $wip = File::Temp->newdir("xapian$v-XXXX", DIR => $dir);
$tmp->{$old} = $wip;
- nodatacow_dir($wip->dirname);
+ PublicInbox::Syscall::nodatacow_dir($wip->dirname);
push @queue, [ $old, $wip ];
} elsif (defined $old) {
opendir my $dh, $old or die "Failed to opendir $old: $!\n";
same_fs_or_die($old, $wip->dirname);
my $cur = "$old/$dn";
push @queue, [ $src // $cur , $wip ];
- nodatacow_dir($wip->dirname);
+ PublicInbox::Syscall::nodatacow_dir($wip->dirname);
$tmp->{$cur} = $wip;
}
# mark old shards to be unlinked
$ft = File::Temp->newdir("$new.compact-XXXX", DIR => $dir);
setup_signals();
$tmp = $ft->dirname;
- nodatacow_dir($tmp);
+ PublicInbox::Syscall::nodatacow_dir($tmp);
} else {
$tmp = $new;
}
#!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict; use v5.10.1; use PublicInbox::TestCommon;
use File::Temp 0.19;
-use_ok 'PublicInbox::NDC_PP';
+use_ok 'PublicInbox::Syscall';
+
+# btrfs on Linux is copy-on-write (COW) by default. As of Linux 5.7,
+# this still leads to fragmentation for SQLite and Xapian files where
+# random I/O happens, so we disable COW just for SQLite files and Xapian
+# directories. Disabling COW disables checksumming, so we only do this
+# for regeneratable files, and not canonical git storage (git doesn't
+# checksum refs, only data under $GIT_DIR/objects).
SKIP: {
my $nr = 2;
skip 'test is Linux-only', $nr if $^O ne 'linux';
my $dir = $ENV{BTRFS_TESTDIR};
skip 'BTRFS_TESTDIR not defined', $nr unless defined $dir;
- require_cmd('chattr', 1) or skip 'chattr(1) not installed', $nr;
+
my $lsattr = require_cmd('lsattr', 1) or
skip 'lsattr(1) not installed', $nr;
+
my $tmp = File::Temp->newdir('nodatacow-XXXX', DIR => $dir);
my $dn = $tmp->dirname;
my $name = "$dn/pp.f";
open my $fh, '>', $name or BAIL_OUT "open($name): $!";
- my $pp_sub = \&PublicInbox::NDC_PP::nodatacow_fd;
- $pp_sub->(fileno($fh));
+ PublicInbox::Syscall::nodatacow_fh($fh);
my $res = xqx([$lsattr, $name]);
+
+ BAIL_OUT "lsattr(1) fails in $dir" if $?;
like($res, qr/C.*\Q$name\E/, "`C' attribute set on fd with pure Perl");
$name = "$dn/pp.d";
mkdir($name) or BAIL_OUT "mkdir($name) $!";
- PublicInbox::NDC_PP::nodatacow_dir($name);
+ PublicInbox::Syscall::nodatacow_dir($name);
$res = xqx([$lsattr, '-d', $name]);
like($res, qr/C.*\Q$name\E/, "`C' attribute set on dir with pure Perl");
-
- $name = "$dn/ic.f";
- my $ic_sub = \&PublicInbox::Spawn::nodatacow_fd;
- $pp_sub == $ic_sub and
- skip 'Inline::C or Linux kernel headers missing', 2;
- open $fh, '>', $name or BAIL_OUT "open($name): $!";
- $ic_sub->(fileno($fh));
- $res = xqx([$lsattr, $name]);
- like($res, qr/C.*\Q$name\E/, "`C' attribute set on fd with Inline::C");
-
- $name = "$dn/ic.d";
- mkdir($name) or BAIL_OUT "mkdir($name) $!";
- PublicInbox::Spawn::nodatacow_dir($name);
- $res = xqx([$lsattr, '-d', $name]);
- like($res, qr/C.*\Q$name\E/, "`C' attribute set on dir with Inline::C");
};
done_testing;