system/bees: Added (btrfs deduplication).

Signed-off-by: Andrew Clemons <andrew.clemons@gmail.com>

Signed-off-by: Willy Sudiarto Raharjo <willysr@slackbuilds.org>

8 files changed, 289 insertions, 0 deletions

diff --git a/system/bees/README b/system/bees/README
new file mode 100644
index 0000000000000..88041ffa13e97
--- /dev/null
+++ b/system/bees/README
@@ -0,0 +1,33 @@
+bees (Best-Effort Extent-Same) is a block-oriented userspace
+deduplication agent designed for large btrfs filesystems. It is an
+offline dedupe combined with an incremental data scan capability to
+minimize time data spends on disk from write to dedupe.
+
+Strengths:
+  * Space-efficient hash table and matching algorithms - can use as
+    little as 1 GB hash table per 10 TB unique data (0.1GB/TB)
+  * Daemon incrementally dedupes new data using btrfs tree search
+  * Works with btrfs compression - dedupe any combination of compressed
+    and uncompressed files
+  * Works around btrfs filesystem structure to free more disk space
+  * Persistent hash table for rapid restart after shutdown
+  * Whole-filesystem dedupe - including snapshots
+  * Constant hash table size - no increased RAM usage if data set
+    becomes larger
+  * Works on live data - no scheduled downtime required
+  * Automatic self-throttling based on system load
+
+Weaknesses:
+  * Whole-filesystem dedupe - has no include/exclude filters, does not
+    accept file lists
+  * Requires root privilege (or CAP_SYS_ADMIN)
+  * First run may require temporary disk space for extent reorganization
+  * First run may increase metadata space usage if many snapshots exist
+  * Constant hash table size - no decreased RAM usage if data set
+    becomes smaller
+  * btrfs only
+
+After installing, edit /etc/rc.d/rc.bees.conf, /etc/logrotate.d/bees,
+and /etc/bees/*.conf, and ensure /etc/rc.d/rc.bees is started from
+/etc/rc.d/rc.local. To drastically reduce the amount of logging it is
+recommended to add "-v 6" to OPTIONS in /etc/bees/*.conf.
diff --git a/system/bees/bees.SlackBuild b/system/bees/bees.SlackBuild
new file mode 100644
index 0000000000000..807d885a3d1e3
--- /dev/null
+++ b/system/bees/bees.SlackBuild
@@ -0,0 +1,106 @@
+#!/bin/bash
+
+# Slackware build script for bees
+
+# Copyright 2023 Erich Ritz, Jenks, Oklahoma, USA
+# All rights reserved.
+#
+# Redistribution and use of this script, with or without modification, is
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of this script must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR IMPLIED
+#  WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+#  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO
+#  EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+#  OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+#  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+#  OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+#  ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+cd $(dirname $0) ; CWD=$(pwd)
+
+PRGNAM=bees
+VERSION=${VERSION:-0.10}
+BUILD=${BUILD:-1}
+TAG=${TAG:-_SBo}
+PKGTYPE=${PKGTYPE:-tgz}
+
+if [ -z "$ARCH" ]; then
+  case "$( uname -m )" in
+    i?86) ARCH=i586 ;;
+    arm*) ARCH=arm ;;
+       *) ARCH=$( uname -m ) ;;
+  esac
+fi
+
+if [ ! -z "${PRINT_PACKAGE_NAME}" ]; then
+  echo "$PRGNAM-$VERSION-$ARCH-$BUILD$TAG.$PKGTYPE"
+  exit 0
+fi
+
+TMP=${TMP:-/tmp/SBo}
+PKG=$TMP/package-$PRGNAM
+OUTPUT=${OUTPUT:-/tmp}
+
+if [ "$ARCH" = "i586" ]; then
+  SLKCFLAGS="-O2 -march=i586 -mtune=i686"
+  LIBDIRSUFFIX=""
+elif [ "$ARCH" = "i686" ]; then
+  SLKCFLAGS="-O2 -march=i686 -mtune=i686"
+  LIBDIRSUFFIX=""
+elif [ "$ARCH" = "x86_64" ]; then
+  SLKCFLAGS="-O2 -fPIC"
+  LIBDIRSUFFIX="64"
+else
+  SLKCFLAGS="-O2"
+  LIBDIRSUFFIX=""
+fi
+
+set -e
+
+rm -rf $PKG
+mkdir -p $TMP $PKG $OUTPUT
+cd $TMP
+rm -rf $PRGNAM-$VERSION
+tar xvf $CWD/$PRGNAM-$VERSION.tar.gz
+cd $PRGNAM-$VERSION
+chown -R root:root .
+find -L . \
+ \( -perm 777 -o -perm 775 -o -perm 750 -o -perm 711 -o -perm 555 \
+  -o -perm 511 \) -exec chmod 755 {} \; -o \
+ \( -perm 666 -o -perm 664 -o -perm 640 -o -perm 600 -o -perm 444 \
+  -o -perm 440 -o -perm 400 \) -exec chmod 644 {} \;
+
+CFLAGS="$SLKCFLAGS" \
+CXXFLAGS="$SLKCFLAGS" \
+make reallyall LIBDIR=lib$LIBDIRSUFFIX BEES_VERSION=$VERSION
+
+make install LIBDIR=lib$LIBDIRSUFFIX DESTDIR=$PKG
+
+find $PKG -print0 | xargs -0 file | grep -e "executable" -e "shared object" | grep ELF \
+  | cut -f 1 -d : | xargs strip --strip-unneeded 2> /dev/null || true
+
+install -D -m0755 -oroot -groot $CWD/rc.bees $PKG/etc/rc.d/rc.bees.new
+install -D -m0644 -oroot -groot $CWD/rc.bees.conf $PKG/etc/rc.d/rc.bees.conf.new
+install -D -m0644 -oroot -groot $CWD/bees.logrotate $PKG/etc/logrotate.d/bees.new
+
+mkdir -p $PKG/usr/doc/$PRGNAM-$VERSION/docs
+cp -a \
+  README.md README.html COPYING \
+  $PKG/usr/doc/$PRGNAM-$VERSION
+cp -a \
+  docs/*.html docs/*.md \
+  $PKG/usr/doc/$PRGNAM-$VERSION/docs
+cat $CWD/$PRGNAM.SlackBuild > $PKG/usr/doc/$PRGNAM-$VERSION/$PRGNAM.SlackBuild
+
+mkdir -p $PKG/install
+cat $CWD/slack-desc > $PKG/install/slack-desc
+cat $CWD/doinst.sh > $PKG/install/doinst.sh
+
+cd $PKG
+/sbin/makepkg -l y -c n $OUTPUT/$PRGNAM-$VERSION-$ARCH-$BUILD$TAG.$PKGTYPE
diff --git a/system/bees/bees.info b/system/bees/bees.info
new file mode 100644
index 0000000000000..9757716342208
--- /dev/null
+++ b/system/bees/bees.info
@@ -0,0 +1,10 @@
+PRGNAM="bees"
+VERSION="0.10"
+HOMEPAGE="https://zygo.github.io/bees/"
+DOWNLOAD="https://github.com/Zygo/bees/archive/v0.10/bees-0.10.tar.gz"
+MD5SUM="2c90623e9867c4dcda4b4ef471372da8"
+DOWNLOAD_x86_64=""
+MD5SUM_x86_64=""
+REQUIRES=""
+MAINTAINER="Erich Ritz"
+EMAIL="erich.public@protonmail.com"
diff --git a/system/bees/bees.logrotate b/system/bees/bees.logrotate
new file mode 100644
index 0000000000000..0a29113ff31d6
--- /dev/null
+++ b/system/bees/bees.logrotate
@@ -0,0 +1,7 @@
+/var/log/bees.log {
+    copytruncate
+    compress
+    notifempty
+    missingok
+    size 10M
+}
diff --git a/system/bees/doinst.sh b/system/bees/doinst.sh
new file mode 100644
index 0000000000000..cf069623bf73c
--- /dev/null
+++ b/system/bees/doinst.sh
@@ -0,0 +1,27 @@
+config() {
+  NEW="$1"
+  OLD="$(dirname $NEW)/$(basename $NEW .new)"
+  # If there's no config file by that name, mv it over:
+  if [ ! -r $OLD ]; then
+    mv $NEW $OLD
+  elif [ "$(cat $OLD | md5sum)" = "$(cat $NEW | md5sum)" ]; then
+    # toss the redundant copy
+    rm $NEW
+  fi
+  # Otherwise, we leave the .new copy for the admin to consider...
+}
+
+preserve_perms() {
+  NEW="$1"
+  OLD="$(dirname $NEW)/$(basename $NEW .new)"
+  if [ -e $OLD ]; then
+    cp -a $OLD ${NEW}.incoming
+    cat $NEW > ${NEW}.incoming
+    mv ${NEW}.incoming $NEW
+  fi
+  config $NEW
+}
+
+preserve_perms etc/rc.d/rc.bees.new
+config etc/rc.d/rc.bees.conf.new
+config etc/logrotate.d/bees.new
diff --git a/system/bees/rc.bees b/system/bees/rc.bees
new file mode 100644
index 0000000000000..e28122e496423
--- /dev/null
+++ b/system/bees/rc.bees
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+declare -a UUIDS
+declare -a LOGFILES
+
+. /etc/rc.d/rc.bees.conf
+
+umount_uuid()
+{
+	UUID="$1"
+	MNT="/run/bees/mnt/$UUID"
+	# Try for up to 10 seconds and then bail out
+	for _ in {1..10} ; do
+		umount "$MNT" > /dev/null 2>&1
+		if mountpoint -q "$MNT" ; then
+			sleep 1
+		else
+			break
+		fi
+	done
+}
+
+bees_start()
+{
+	for i in "${!UUIDS[@]}" ; do
+		UUID="${UUIDS[$i]}"
+		LOGFILE="${LOGFILES[$i]}"
+		umount_uuid "$UUID"
+		nice -n 15 ionice -c 3 daemon --name="bees.$UUID" --output="$LOGFILE" -- beesd "$UUID"
+	done
+}
+
+bees_stop()
+{
+	for UUID in "${UUIDS[@]}" ; do
+		daemon --name="bees.$UUID" --stop
+		umount_uuid "$UUID"
+	done
+}
+
+bees_status()
+{
+	for UUID in "${UUIDS[@]}" ; do
+		daemon --name="bees.$UUID" --running --verbose
+	done
+}
+
+bees_forcestop()
+{
+	killall bees
+}
+
+case "$1" in
+	'start')
+		bees_start
+		;;
+	'stop')
+		bees_stop
+		;;
+	'status')
+		bees_status
+		;;
+	'forcestop')
+		bees_forcestop
+		;;
+	*)
+		echo "usage $0 start|stop|status|forcestop"
+		;;
+esac
diff --git a/system/bees/rc.bees.conf b/system/bees/rc.bees.conf
new file mode 100644
index 0000000000000..9bfc8aecf4ce3
--- /dev/null
+++ b/system/bees/rc.bees.conf
@@ -0,0 +1,18 @@
+# Find the UUID of btrfs filesystems by executing the command:
+#
+#   btrfs filesystem show
+#
+# For each filesystem, define UUIDS[#] and LOGFILES[#] to set the UUID
+# and log file location. Don't foget to update /etc/logrotate.d/bees
+# when adding new log files.
+#
+# Each btrfs filesystem should have a corresponding conf file in
+# /etc/bees/*.conf. See /etc/bees/beesd.conf.sample.
+
+## btrfs filesystem #1
+#UUIDS[0]=11111111-1111-1111-1111-111111111111
+#LOGFILES[0]=/var/log/bees.log
+
+## btrfs filesystem #2
+#UUIDS[1]=22222222-2222-2222-2222-222222222222
+#LOGFILES[1]=/var/log/bees_2.log
diff --git a/system/bees/slack-desc b/system/bees/slack-desc
new file mode 100644
index 0000000000000..47e2e82afeb4c
--- /dev/null
+++ b/system/bees/slack-desc
@@ -0,0 +1,19 @@
+# HOW TO EDIT THIS FILE:
+# The "handy ruler" below makes it easier to edit a package description.
+# Line up the first '|' above the ':' following the base package name, and
+# the '|' on the right side marks the last column you can put a character in.
+# You must make exactly 11 lines for the formatting to be correct.  It's also
+# customary to leave one space after the ':' except on otherwise blank lines.
+
+    |-----handy-ruler------------------------------------------------------|
+bees: bees (Best-Effort Extent-Same)
+bees:
+bees: bees is a block-oriented userspace deduplication agent designed for
+bees: large btrfs filesystems. It is an offline dedupe combined with an
+bees: incremental data scan capability to minimize time data spends on disk
+bees: from write to dedupe.
+bees:
+bees: Homepage: https://zygo.github.io/bees/
+bees:
+bees:
+bees: