aboutsummaryrefslogtreecommitdiff
path: root/system
diff options
context:
space:
mode:
authorErich Ritz <erich.public@protonmail.com>2023-09-09 10:44:30 +0900
committerWilly Sudiarto Raharjo <willysr@slackbuilds.org>2023-09-09 10:14:14 +0700
commit5522e231bbb04b8975a685576e3a0ebd3bb55a31 (patch)
treec3970cc815c00ed4f5057db8ab9c38a7d8577089 /system
parentc87531f887521d4ca804839e998f24a7b34e3295 (diff)
system/bees: Added (btrfs deduplication).
Signed-off-by: Andrew Clemons <andrew.clemons@gmail.com> Signed-off-by: Willy Sudiarto Raharjo <willysr@slackbuilds.org>
Diffstat (limited to 'system')
-rw-r--r--system/bees/README33
-rw-r--r--system/bees/bees.SlackBuild106
-rw-r--r--system/bees/bees.info10
-rw-r--r--system/bees/bees.logrotate7
-rw-r--r--system/bees/doinst.sh27
-rw-r--r--system/bees/rc.bees69
-rw-r--r--system/bees/rc.bees.conf18
-rw-r--r--system/bees/slack-desc19
8 files changed, 289 insertions, 0 deletions
diff --git a/system/bees/README b/system/bees/README
new file mode 100644
index 0000000000000..88041ffa13e97
--- /dev/null
+++ b/system/bees/README
@@ -0,0 +1,33 @@
+bees (Best-Effort Extent-Same) is a block-oriented userspace
+deduplication agent designed for large btrfs filesystems. It is an
+offline dedupe combined with an incremental data scan capability to
+minimize time data spends on disk from write to dedupe.
+
+Strengths:
+ * Space-efficient hash table and matching algorithms - can use as
+ little as 1 GB hash table per 10 TB unique data (0.1GB/TB)
+ * Daemon incrementally dedupes new data using btrfs tree search
+ * Works with btrfs compression - dedupe any combination of compressed
+ and uncompressed files
+ * Works around btrfs filesystem structure to free more disk space
+ * Persistent hash table for rapid restart after shutdown
+ * Whole-filesystem dedupe - including snapshots
+ * Constant hash table size - no increased RAM usage if data set
+ becomes larger
+ * Works on live data - no scheduled downtime required
+ * Automatic self-throttling based on system load
+
+Weaknesses:
+ * Whole-filesystem dedupe - has no include/exclude filters, does not
+ accept file lists
+ * Requires root privilege (or CAP_SYS_ADMIN)
+ * First run may require temporary disk space for extent reorganization
+ * First run may increase metadata space usage if many snapshots exist
+ * Constant hash table size - no decreased RAM usage if data set
+ becomes smaller
+ * btrfs only
+
+After installing, edit /etc/rc.d/rc.bees.conf, /etc/logrotate.d/bees,
+and /etc/bees/*.conf, and ensure /etc/rc.d/rc.bees is started from
+/etc/rc.d/rc.local. To drastically reduce the amount of logging it is
+recommended to add "-v 6" to OPTIONS in /etc/bees/*.conf.
diff --git a/system/bees/bees.SlackBuild b/system/bees/bees.SlackBuild
new file mode 100644
index 0000000000000..807d885a3d1e3
--- /dev/null
+++ b/system/bees/bees.SlackBuild
@@ -0,0 +1,106 @@
+#!/bin/bash
+
+# Slackware build script for bees
+
+# Copyright 2023 Erich Ritz, Jenks, Oklahoma, USA
+# All rights reserved.
+#
+# Redistribution and use of this script, with or without modification, is
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of this script must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+cd $(dirname $0) ; CWD=$(pwd)
+
+PRGNAM=bees
+VERSION=${VERSION:-0.10}
+BUILD=${BUILD:-1}
+TAG=${TAG:-_SBo}
+PKGTYPE=${PKGTYPE:-tgz}
+
+if [ -z "$ARCH" ]; then
+ case "$( uname -m )" in
+ i?86) ARCH=i586 ;;
+ arm*) ARCH=arm ;;
+ *) ARCH=$( uname -m ) ;;
+ esac
+fi
+
+if [ ! -z "${PRINT_PACKAGE_NAME}" ]; then
+ echo "$PRGNAM-$VERSION-$ARCH-$BUILD$TAG.$PKGTYPE"
+ exit 0
+fi
+
+TMP=${TMP:-/tmp/SBo}
+PKG=$TMP/package-$PRGNAM
+OUTPUT=${OUTPUT:-/tmp}
+
+if [ "$ARCH" = "i586" ]; then
+ SLKCFLAGS="-O2 -march=i586 -mtune=i686"
+ LIBDIRSUFFIX=""
+elif [ "$ARCH" = "i686" ]; then
+ SLKCFLAGS="-O2 -march=i686 -mtune=i686"
+ LIBDIRSUFFIX=""
+elif [ "$ARCH" = "x86_64" ]; then
+ SLKCFLAGS="-O2 -fPIC"
+ LIBDIRSUFFIX="64"
+else
+ SLKCFLAGS="-O2"
+ LIBDIRSUFFIX=""
+fi
+
+set -e
+
+rm -rf $PKG
+mkdir -p $TMP $PKG $OUTPUT
+cd $TMP
+rm -rf $PRGNAM-$VERSION
+tar xvf $CWD/$PRGNAM-$VERSION.tar.gz
+cd $PRGNAM-$VERSION
+chown -R root:root .
+find -L . \
+ \( -perm 777 -o -perm 775 -o -perm 750 -o -perm 711 -o -perm 555 \
+ -o -perm 511 \) -exec chmod 755 {} \; -o \
+ \( -perm 666 -o -perm 664 -o -perm 640 -o -perm 600 -o -perm 444 \
+ -o -perm 440 -o -perm 400 \) -exec chmod 644 {} \;
+
+CFLAGS="$SLKCFLAGS" \
+CXXFLAGS="$SLKCFLAGS" \
+make reallyall LIBDIR=lib$LIBDIRSUFFIX BEES_VERSION=$VERSION
+
+make install LIBDIR=lib$LIBDIRSUFFIX DESTDIR=$PKG
+
+find $PKG -print0 | xargs -0 file | grep -e "executable" -e "shared object" | grep ELF \
+ | cut -f 1 -d : | xargs strip --strip-unneeded 2> /dev/null || true
+
+install -D -m0755 -oroot -groot $CWD/rc.bees $PKG/etc/rc.d/rc.bees.new
+install -D -m0644 -oroot -groot $CWD/rc.bees.conf $PKG/etc/rc.d/rc.bees.conf.new
+install -D -m0644 -oroot -groot $CWD/bees.logrotate $PKG/etc/logrotate.d/bees.new
+
+mkdir -p $PKG/usr/doc/$PRGNAM-$VERSION/docs
+cp -a \
+ README.md README.html COPYING \
+ $PKG/usr/doc/$PRGNAM-$VERSION
+cp -a \
+ docs/*.html docs/*.md \
+ $PKG/usr/doc/$PRGNAM-$VERSION/docs
+cat $CWD/$PRGNAM.SlackBuild > $PKG/usr/doc/$PRGNAM-$VERSION/$PRGNAM.SlackBuild
+
+mkdir -p $PKG/install
+cat $CWD/slack-desc > $PKG/install/slack-desc
+cat $CWD/doinst.sh > $PKG/install/doinst.sh
+
+cd $PKG
+/sbin/makepkg -l y -c n $OUTPUT/$PRGNAM-$VERSION-$ARCH-$BUILD$TAG.$PKGTYPE
diff --git a/system/bees/bees.info b/system/bees/bees.info
new file mode 100644
index 0000000000000..9757716342208
--- /dev/null
+++ b/system/bees/bees.info
@@ -0,0 +1,10 @@
+PRGNAM="bees"
+VERSION="0.10"
+HOMEPAGE="https://zygo.github.io/bees/"
+DOWNLOAD="https://github.com/Zygo/bees/archive/v0.10/bees-0.10.tar.gz"
+MD5SUM="2c90623e9867c4dcda4b4ef471372da8"
+DOWNLOAD_x86_64=""
+MD5SUM_x86_64=""
+REQUIRES=""
+MAINTAINER="Erich Ritz"
+EMAIL="erich.public@protonmail.com"
diff --git a/system/bees/bees.logrotate b/system/bees/bees.logrotate
new file mode 100644
index 0000000000000..0a29113ff31d6
--- /dev/null
+++ b/system/bees/bees.logrotate
@@ -0,0 +1,7 @@
+/var/log/bees.log {
+ copytruncate
+ compress
+ notifempty
+ missingok
+ size 10M
+}
diff --git a/system/bees/doinst.sh b/system/bees/doinst.sh
new file mode 100644
index 0000000000000..cf069623bf73c
--- /dev/null
+++ b/system/bees/doinst.sh
@@ -0,0 +1,27 @@
+config() {
+ NEW="$1"
+ OLD="$(dirname $NEW)/$(basename $NEW .new)"
+ # If there's no config file by that name, mv it over:
+ if [ ! -r $OLD ]; then
+ mv $NEW $OLD
+ elif [ "$(cat $OLD | md5sum)" = "$(cat $NEW | md5sum)" ]; then
+ # toss the redundant copy
+ rm $NEW
+ fi
+ # Otherwise, we leave the .new copy for the admin to consider...
+}
+
+preserve_perms() {
+ NEW="$1"
+ OLD="$(dirname $NEW)/$(basename $NEW .new)"
+ if [ -e $OLD ]; then
+ cp -a $OLD ${NEW}.incoming
+ cat $NEW > ${NEW}.incoming
+ mv ${NEW}.incoming $NEW
+ fi
+ config $NEW
+}
+
+preserve_perms etc/rc.d/rc.bees.new
+config etc/rc.d/rc.bees.conf.new
+config etc/logrotate.d/bees.new
diff --git a/system/bees/rc.bees b/system/bees/rc.bees
new file mode 100644
index 0000000000000..e28122e496423
--- /dev/null
+++ b/system/bees/rc.bees
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+declare -a UUIDS
+declare -a LOGFILES
+
+. /etc/rc.d/rc.bees.conf
+
+umount_uuid()
+{
+ UUID="$1"
+ MNT="/run/bees/mnt/$UUID"
+ # Try for up to 10 seconds and then bail out
+ for _ in {1..10} ; do
+ umount "$MNT" > /dev/null 2>&1
+ if mountpoint -q "$MNT" ; then
+ sleep 1
+ else
+ break
+ fi
+ done
+}
+
+bees_start()
+{
+ for i in "${!UUIDS[@]}" ; do
+ UUID="${UUIDS[$i]}"
+ LOGFILE="${LOGFILES[$i]}"
+ umount_uuid "$UUID"
+ nice -n 15 ionice -c 3 daemon --name="bees.$UUID" --output="$LOGFILE" -- beesd "$UUID"
+ done
+}
+
+bees_stop()
+{
+ for UUID in "${UUIDS[@]}" ; do
+ daemon --name="bees.$UUID" --stop
+ umount_uuid "$UUID"
+ done
+}
+
+bees_status()
+{
+ for UUID in "${UUIDS[@]}" ; do
+ daemon --name="bees.$UUID" --running --verbose
+ done
+}
+
+bees_forcestop()
+{
+ killall bees
+}
+
+case "$1" in
+ 'start')
+ bees_start
+ ;;
+ 'stop')
+ bees_stop
+ ;;
+ 'status')
+ bees_status
+ ;;
+ 'forcestop')
+ bees_forcestop
+ ;;
+ *)
+ echo "usage $0 start|stop|status|forcestop"
+ ;;
+esac
diff --git a/system/bees/rc.bees.conf b/system/bees/rc.bees.conf
new file mode 100644
index 0000000000000..9bfc8aecf4ce3
--- /dev/null
+++ b/system/bees/rc.bees.conf
@@ -0,0 +1,18 @@
+# Find the UUID of btrfs filesystems by executing the command:
+#
+# btrfs filesystem show
+#
+# For each filesystem, define UUIDS[#] and LOGFILES[#] to set the UUID
+# and log file location. Don't foget to update /etc/logrotate.d/bees
+# when adding new log files.
+#
+# Each btrfs filesystem should have a corresponding conf file in
+# /etc/bees/*.conf. See /etc/bees/beesd.conf.sample.
+
+## btrfs filesystem #1
+#UUIDS[0]=11111111-1111-1111-1111-111111111111
+#LOGFILES[0]=/var/log/bees.log
+
+## btrfs filesystem #2
+#UUIDS[1]=22222222-2222-2222-2222-222222222222
+#LOGFILES[1]=/var/log/bees_2.log
diff --git a/system/bees/slack-desc b/system/bees/slack-desc
new file mode 100644
index 0000000000000..47e2e82afeb4c
--- /dev/null
+++ b/system/bees/slack-desc
@@ -0,0 +1,19 @@
+# HOW TO EDIT THIS FILE:
+# The "handy ruler" below makes it easier to edit a package description.
+# Line up the first '|' above the ':' following the base package name, and
+# the '|' on the right side marks the last column you can put a character in.
+# You must make exactly 11 lines for the formatting to be correct. It's also
+# customary to leave one space after the ':' except on otherwise blank lines.
+
+ |-----handy-ruler------------------------------------------------------|
+bees: bees (Best-Effort Extent-Same)
+bees:
+bees: bees is a block-oriented userspace deduplication agent designed for
+bees: large btrfs filesystems. It is an offline dedupe combined with an
+bees: incremental data scan capability to minimize time data spends on disk
+bees: from write to dedupe.
+bees:
+bees: Homepage: https://zygo.github.io/bees/
+bees:
+bees:
+bees: